diff --git a/docs/design/features/async-mincallcost.svg b/docs/design/features/async-mincallcost.svg
new file mode 100644
index 00000000000000..ed089be8328256
--- /dev/null
+++ b/docs/design/features/async-mincallcost.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/design/features/async-vs-async2-relative-performance.svg b/docs/design/features/async-vs-async2-relative-performance.svg
new file mode 100644
index 00000000000000..8eac56d28f95ec
--- /dev/null
+++ b/docs/design/features/async-vs-async2-relative-performance.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/design/features/async-vs-async2capture-relative-performance.svg b/docs/design/features/async-vs-async2capture-relative-performance.svg
new file mode 100644
index 00000000000000..d9dee0be0a31da
--- /dev/null
+++ b/docs/design/features/async-vs-async2capture-relative-performance.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/design/features/async2-vs-async2capture-relative-performance.svg b/docs/design/features/async2-vs-async2capture-relative-performance.svg
new file mode 100644
index 00000000000000..3143faacafe81a
--- /dev/null
+++ b/docs/design/features/async2-vs-async2capture-relative-performance.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/design/features/perf-of-throwing.svg b/docs/design/features/perf-of-throwing.svg
new file mode 100644
index 00000000000000..0658a971b015d7
--- /dev/null
+++ b/docs/design/features/perf-of-throwing.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/design/features/runtime-handled-tasks.md b/docs/design/features/runtime-handled-tasks.md
new file mode 100644
index 00000000000000..eba5d22027889b
--- /dev/null
+++ b/docs/design/features/runtime-handled-tasks.md
@@ -0,0 +1,791 @@
+# Runtime Handled Tasks Experiment
+
+The .NET Runtime has historically supported a programming model of developing async programs through use of the System.Threading.Tasks library provided by the runtime as well as the Roslyn async method support, which interacts with that tasks library to make it simple to write the callback based state machines that the runtime can use. Over the years several attempts have been made to optimize the performance and other characteristics of this scheme, but fundamentally those arrangements have relied on either tweaks to the core library's code base or adjustments to the IL code generated. This experiment seeks to identify what improvements to the async model can be achieved by moving the state machine computation and handling to being handled internally in the runtime.
+
+## Experimental Goals
+
+1. Find out if runtime generated async state machines are substantially better in performance than the existing async state machine model
+2. Find out if we can implement a variation on async which has some subtle semantic behavior changes, but remains compatible with existing code which is written in the original style
+
+## Experimental Approach
+
+1. Develop new semantics which allow the runtime to implement state machines as part of the runtime instead of relying on the existing state machine generation.
+2. Develop 1 or more implementations of the runtime which implement said semantics
+3. Develop a version of the Roslyn compiler which is capable of building code using the new semantics
+4. Develop a set of microbenchmarks for measuring the different performance characteristics of the new model
+5. Develop a set of application scenario benchmarks
+
+Throughput this document, any references to the term async2 refer exclusively to the new model.
+
+## New Semantics
+
+This section is a work in progress, but describes the current expected semantic changes.
+
+### Specification of a method as following async2 semantics
+
+To specify that a method follows async2 rules, a custom modifier to either `System.Threading.Tasks.Task`, ``System.Threading.Tasks.Task`1``, `System.Threading.Tasks.ValueTask`, or ``System.Threading.Tasks.ValueTask`1`` shall be placed as the last custom modifier before the return type in the signature of a method.
+
+### Flowing `SynchronizationContext` and `ExecutionContext` and other behavior such as AsyncLocal
+Unlike traditional C# compiler generated async, async2 methods shall not save/restore `SynchronizationContext` and `ExecutionContext` at function boundaries. Instead, changes to them can be observed by their callers (as long as the caller is not a Task to async2 thunk).
+
+#### Integration with `SynchronizationContext` and resumption
+
+A new attribute `System.Runtime.CompilerServices.ConfigureAwaitAttribute` will be defined. It can be applied at the assembly/module/type/method level. It shall be defined as
+```cs
+namespace System.Runtime.CompilerServices
+{
+ class ConfigureAwaitAttribute : Attribute
+ {
+ public ConfigureAwaitAttribute(bool continueOnCapturedContext) { ContinueOnCapturedContext = continueOnCapturedContext; }
+ public bool ContinueOnCapturedContext { get; }
+ }
+}
+```
+
+The behavior of this attribute shall be to apply configure await semantics for resuming suspended frames of execution. Notably, though, the semantics of exactly which `SynchronizationContext` to use is subtly different from that of traditional async. In traditional async since each async method saves/restores the `SynchronizationContext` an async method which suspended at an ConfigureAwait(true) await point awaiting async method `A` will always continue on the Synchronization context that existed before the call to async method `A`, where in the async2 model the context used will be that which is current just before calling into method `A`, but in the async2 model, the SynchronizationContext will be the context that was current just before the return statement of method `A`. In practice, we expect this to be a distinction without a difference, as all known intentional uses of `SyncrhonizationContext` are implemented via a pattern where the context is set in a function, and restored via a try/finally to its original state before returning from the function.
+
+#### Implications of flowing `ExecutionContext` and async locals
+
+`ExecutionContext` is used to represent the current values of the `AsyncLocal`. This feature is colloquially known as async locals. The semantics of these locals is surprising to developers today, in that while any function can modify the current state of an aync local, if an async function returns, any mutations to the async local are lost. In contrast, this proposal changes to the model such that when an async2 function returns, the async local state is **not** reverted to its previous state.
+
+### Integration with the existing api surface of Task and ValueTask based apis
+
+Any MethodDef that is an async2 based method can also be called as a `Task`/`ValueTask` returning method. Likewise any MethodDef which returns `Task` or `ValueTask` can be called via an async2 entrypoint. In addition, MethodImpl records and virtual method override rules shall be adjusted so that if an interface or base type defines a virtual method using a signature of `Task`/`ValueTask` then it can be overriden via a method that is an async2 api. The same is true vice versa. For overrides implemented via `MethodImpl` the MethodImpl must describe the Body and Decl using signatures which are either both async2 or `Task`/`ValueTask` returning, but the MethodImpl will also apply to the other async variant. These overrides are permitted to be interleaved, so for example
+
+```cs
+class Base
+{
+ public virtual async Task Method() { ... }
+}
+class Derived : Base
+{
+ public override async2 Task Method() { ... }
+}
+
+class MoreDerived : Derived
+{
+ public override async Task Method() { ... }
+}
+```
+
+Any attempt to call an entrypoint will resolve to either an implementation which matches the signature, or in case of an async variant mismatch, will resolve to a runtime generated thunk which will call into the developer provided implementation.
+
+Methods that return Task via generics do not follow this rule. For instance
+```cs
+class MyGeneric
+{
+ T DoSomething() { ... }
+}
+```
+
+`MyGeneric.DoSomething()` is not callable via an async2 entrypoint.
+
+#### Thunk from an async2 api surface to Task based implementation
+
+A thunk from an async2 api surface to a Task based implementation will have the following psuedocode, if a legal thunk can be made.
+
+```cs
+async2 Task ThunkAsync(ParameterType param1, ParameterType2 param2, ...)
+{
+ return RuntimeHelpers.UnsafeAwaitAwaiterFromRuntimeAsync, ReturnType>(TargetMethod(param1, param2, ...));
+}
+```
+
+If any of the parameter types to the target method are ref parameters, or ref structures, then the Thunk generated will be
+```cs
+async2 Task ThunkAsync(ParameterType param1, ParameterType2 param2, ...)
+{
+ throw new InvalidProgramException();
+}
+```
+
+#### Thunk from the Task api surface to async2 based implementation
+
+If any of the parameter types to the target method are ref parameters, or ref structures, then any thunk invoked will throw `InvalidProgramException`. Otherwise it will perform whatever runtime actions are necessary to transition to the async2 implementation, and return the appropriate Task object type on return.
+
+Thunks are required to save and restore the `ExecutionContext` and `SynchronizationContext` such that callers of thunks cannot observe changes to them.
+
+### Specifying custom scheduling of async2
+Async2 shall integrate with `TaskScheduler.Current`.
+
+### Semantics of async2 based IL code
+
+#### IL Semantics
+
+Async2 IL is broadly similar to the existing IL semantics with the following restrictions.
+
+1. Usage of the `localloc` instruction is forbidden
+2. The `ldloca` and `ldarga` instructions are redefined to return managed pointers instead of pointers.
+3. A pinning local cannot be validly used with a local variable.
+. As an initial restriction that is not fundamental, the `tail.` prefix is not permitted
+4. Use of byrefs and ref structs within async2 methods may or may not be acceptable. The currently JIT focussed prototype is designed with the same limitations as the existing async model, and thus does not work with byref data. However, the unwinder approach is able to tolerate byrefs.
+
+#### EH Semantics
+No call to a method with an async2 modreq will be permitted in a finally, fault, filter, or catch clause. If such a thing exists, the program is invalid.
+
+The StackTrace of a exception thrown within an async2 method shall include any async2 frames which are logically on the call stack up until the thunk which transfers execution to the root async2 method.
+
+### Api surface for interacting with non Task/ValueTask apis which async from within async2 code
+
+The runtime shall provide the following apis
+
+```
+ // public static async2 Task AwaitAwaiterFromRuntimeAsync(TAwaiter awaiter) where TAwaiter : INotifyCompletion
+ public static void modopt([System.Runtime]System.Threading.Tasks.Task) AwaitAwaiterFromRuntimeAsync(TAwaiter awaiter) where TAwaiter : INotifyCompletion
+
+ // public static async2 Task UnsafeAwaitAwaiterFromRuntimeAsync(TAwaiter awaiter) where TAwaiter : ICriticalNotifyCompletion
+ public static void modopt([System.Runtime]System.Threading.Tasks.Task) UnsafeAwaitAwaiterFromRuntimeAsync(TAwaiter awaiter) where TAwaiter : ICriticalNotifyCompletion
+```
+
+Any IL in an async2 method which needs to await will follow the existing C# model of calling `IsCompleted` and `GetResult()` and in the middle instead of using the existing patterns for suspension shall call one of the above helper methods.
+
+### Utilizing async2 based code from non-async2 based code
+
+Non-async2 code is never permitted to directly invoke async2 code. Instead any and all calls into async2 functions shall be done by means of one of the thunks.
+
+### Interaction with Reflection
+
+Async2 methods will not be visible in reflection via the `Type.GetMethod`, `TypeInfo.DeclaredMethods`, `TypeInfo.DeclaredMembers` , or `Type.GetInterfaceMap()` apis.
+
+`Type.GetMembers` and `Type.GetMethods` will be able to find the async2 methods if and only if the `BindingFlags.Async2Visible` flag is set.
+
+`Type.GetMemberWithSameMetadataDefinitionAs` will find the matching async variant method.
+
+
+## Implementation of Unwinder based async2 prototype
+
+The general design is to leverage the notion that a normal code generated function at a function call point is effectively a state machine. The index for resumption is the IP that returning to the function will set, and the stackframe + saved registers are the current state of the state machine. I believe we can achieve performance comparable to synchronous code with this scheme for non-suspended scenarios, and we may achieve acceptable overall performance as an async mechanism if suspension is relatively rare.
+
+To enable this prototype, set `DOTNET_RuntimeAsyncViaJitGeneratedStateMachines=0`.
+
+### JIT Code changes
+1. Do not allow InlinedCallFrames to be linked to the frame chain across a suspend (Or disable p/invoke inlining in these methods)
+2. Report frame pointers as ByRef
+3. Report all pointers to locals as ByRef
+4. Force the AwaitAwaiterFromRuntimeAsync to be treated as an async2 method even though it isn't marked as such (due to compiler limitations)
+5. Do not allow the JIT to hold onto a thread static base pointer across await points.
+
+### Thunk from the Task api surface to async2 based implementation
+
+A thunk from the Task api surface to an async2 based implementation will have the following psuedocode if it is not required to throw `InvalidProgramException`
+
+```
+Task ThunkAsync(ParameterType param1, ParameterType2 param2, ...)
+{
+ // A variant on this helper type will be defined for each type of `Task`/`ValueTask`/`Task`/`ValueTask`
+ System.Runtime.CompilerServices.RuntimeTaskState runtimeTaskState = new;
+
+ // If the Task is parameterized there shall be a return type to work with.
+ ReturnType result;
+
+ runtimeTaskState.Push();
+ try
+ {
+ try
+ {
+ // NOTE there is a special case for instance methods on valuetypes. For that case,
+ // the thunk will construct a boxed instance of the this value, and then invoke the method
+ // on that.
+ result = TargetMethod(param1, param2, ...);
+ }
+ catch (Exception ex)
+ {
+ return runtimeTaskState.FromException(ex);
+ }
+ return runtimeTaskState.FromResult(result);
+ }
+ finally
+ {
+ runtimeTaskState.Pop();
+ }
+}
+```
+
+### Implementation of `AwaitAwaiterFromRuntimeAsync`
+The implementation of this method captures every stack frame from itself down to the `ThunkAsync` function or `ResumptionFunc`. Each one of these stack frames is packaged up into a structure known as a `Tasklet`. These `Tasklet` objects are registered with the GC so that they will be properly reported, but that reporting is not based on normal GC reporting, but is instead a special codebase. Primarily this need to treat them as special is done so that ByRef pointers can be handled correctly. Once all of the stack frames are captured, then the set of unwound `Tasklet` and the awaiter object are placed in a thread local variable for use by the `ThunkAsync` or `ResumptionFunc` functions. Finally, the actual stack is unwound to the `ThunkAsync`/`ResumptionFunc` frame, which detects that the operation was suspended and does the appropriate thing. In the ThunkAsync implementation, this is done by the runtimeTaskState.FromResult method, which determines if it should ignore the result it acquired from the `TargetMethod` and instead just create a task to return. Finally, a delegate to `ResumptionFunc` is passed to the awaiter.
+
+`Tasklet` structures are designed to hold the data of a frame, the instruction pointer where execution should resume, the current state of any callee preserved registers that are used by the function, and layout/unwind information for finding GC data/and preserved register locations within the frame. Of particular interest is the handling for preserved registers. In the normal case of execution of JIT generated code, when a function needs to use a callee preserved register, it will generate code to store the current value of the register during the prolog of the method, and restore it during the epilog of the method. This unwinder based approach reverses that approach so that the `Tasklet` copy of the stack frame stores the "current" value of the register, and it is reported to the GC in that location when the `Tasklet` goes through GC reporting. The other detail of particular interest is that the data necessary to describe the locations of these preserved registers turns out to be exactly the data necessary to unwind a stack frame.
+
+The implementation of unwinding used to return back to the `ThunkAsync` method is currently written in a highly optimized assembly path which walks the list of preserved register locations, and resets them.
+
+Current implementation of this unwinder scheme use the standard CoreCLR unwinder and GC information reporter to build data structures that describe the unwind/GC data at a particular instruction pointer address, but in theory this should be amenable to a high performance cache.
+
+### Implementation of `ResumptionFunc`
+The dispatcher takes a collection of Tasklets, and resumes the one at the top of the stack, and if it returns, will pop it off the stack and execute the next one. It shall maintain structures such that the EH stack walker can find the list of stack frames that are still held in `Tasklet`s. In addition it shall be responsible for ensuring that the correct `ExecutionContext`/`SynchronizationContext` is maintained, and any `Tasklet` is resumed using the appropriate scheduler provided to the runtime. This implementation today is partial, but the basic structure is present to do so.
+
+#### Resumption of a Tasklet
+
+Once the `ResumptionFunc` has setup the global state to have the correct bits in it, it will call into a carefully crafted assembly stub (called `ResumeTaskletIntegerRegisterReturn` or `ResumeTaskletReferenceReturn`) which takes a `Tasklet*` as input as well as a pointer to a structure which holds the current return value that should be restored to the return value registers. This assembly stub shall copy the `Tasklet`'s stack frame onto the stack, swap any values in the preserved register locations with the current value of the preserved registers, and then tail jump to the instruction pointer held in the `Tasklet`. This resumption process was chosen to ensure that:
+1. The return address on the shadow CET stack does not need any updates upon `Tasklet` resumption.
+2. The runtime only resumes a single method at a time, which improves efficiency in the somewhat common case where there is a deep stack which has a loop in it.
+3. Once resumed, the stack appears as a normal stack to all stackwalking operations.
+
+### Expected characteristics of this approach
+1. The performance of code which does not suspend is effectively the same as normal synchronous code. (confirmed)
+2. The performance of code which suspends a lot is highly impacted by the cost of performing unwind, and somewhat by resumption cost. (under investigation)
+3. The cost of GC while `Tasklet`s exist is dependent on the number of live `Tasklet` objects. This is almost certainly a fairly high cost. (under investigation)
+4. This approach allows development of async code which uses ref parameters/locals and ByRefLike structures. (confirmed)
+
+## Implementation of prototype based on JIT generated state machines
+
+In the second prototype we leave it up to the JIT to generate the state machines for async2 functions.
+When awaiting an async2 function from within an async2 function the JIT generates suspension code to capture the live state in a continuation.
+In addition, the JIT generates code for each of these suspension points to be able to resume from the continuation that was created.
+
+This prototype is enabled by default, or when `DOTNET_RuntimeAsyncViaJitGeneratedStateMachines=1`.
+
+### Suspension
+
+Communicating suspension is done by returning a non-zero continuation, using a special calling convention.
+The continuation is a normal GC ref so special care must be taken within the JIT and within the VM to handle proper GC reporting of this return value.
+On x64 the continuation is returned in `rcx`.
+
+When a callee suspends by returning a non-zero continuation the caller itself suspends by creating its own continuation and returning it.
+Additionally, the generated suspension code links the continuations into a linked list.
+
+`Continuation` is defined as a simple class in the BCL:
+```csharp
+internal sealed unsafe class Continuation
+{
+ public Continuation? Next;
+ public delegate* Resume;
+ public uint State;
+ public CorInfoContinuationFlags Flags;
+ public byte[]? Data;
+ public object[]? GCData;
+}
+```
+
+This incurs extra cost on every call to an async2 method in the form of a test for whether the returned continuation is non-zero.
+
+### Resumption
+
+In addition to returning a continuation async2 methods also accept a continuation as an additional parameter.
+The parameter follows the normal calling convention; it is added in the parameter list after the generic context, return buffer or `this` parameter (whichever comes last).
+
+Resuming an async2 method is done by passing a non-zero continuation.
+The continuation must have been created by the same exact native code version that is being resumed.
+
+Passing a zero continuation is the same as starting the state machine.
+The JIT will automatically generate code to pass a zero continuation when calling an async2 method.
+
+When the JIT generates code for an async2 function it asks the runtime to create a resumption stub.
+The suspension code stores the pointer to the resumption stub in the `Continuation.Resume` field.
+
+The resumption stub implements the following:
+1. It calls into the original function passing the non-zero continuation
+2. It passes default values for all arguments, to make sure that the callee has the expected stack frame set up. If parameters are live, then it is expected that the resumed function restores them from the continuation state.
+3. If the function returns a value it ensures that the return value is propagated into the next continuation at the right location (either its `Data` or `GCData` arrays).
+4. If the function returned a new continuation (i.e. it suspended again), the resumption stub returns it back to the caller.
+
+In pseudo-C#:
+```csharp
+static async2 Task Foo(int a, int b)
+{
+ ...
+}
+
+static Continuation? IL_STUB_AsyncResume_Foo(Continuation continuation)
+{
+ delegate* foo = &Foo;
+ int result = foo(continuation, 0, 0);
+ Continuation? newContinuation = StubHelpers.Async2CallContinuation();
+
+ if (newContinuation == null)
+ {
+ // Foo returns an int and is saved in the Data array. If Foo returned a
+ // GC ref or struct with GC refs, it would be boxed and stored in GCData instead.
+ // Exact index depends on some factors; see the code for the details.
+ Unsafe.Write(ref continuation.Next.Data[index], result);
+ }
+
+ return newContinuation;
+}
+```
+
+### Intrinsics used for suspension/resumption
+
+To interact with the async2 calling convention the JIT/VM provides the following intrinsics:
+```csharp
+// Retrieve the continuation returned by a preceding async2 function call
+[Intrinsic]
+internal static Continuation? Async2CallContinuation() => null;
+
+// Suspend the current function by immediately returning with a specific non-zero continuation.
+[Intrinsic]
+private static void SuspendAsync2(Continuation continuation) => throw new UnreachableException();
+```
+
+These intrinsics are NOT used by user code, but they are used internally by the async1<->async2 adapter later, which will be described later.
+
+`Async2CallContinuation()` is also used by generated resumption stubs to capture the returned continuation.
+
+Since the continuation is a normal parameter no intrinsic is required to allow the resumption stub to pass the continuation to the target.
+Instead, the resumption stub calls the target by `calli` with a signature that includes the continuation, in the same way as IL instantiating stubs work.
+
+### Handling OSR
+
+There is additional complexity necessary to handle OSR correctly:
+1. OSR expects that the frame was set up by a tier-0 method so resumption directly in an OSR method is not possible.
+Instead, the JIT generates code in the tier0 version to check for an OSR continuation in which case the tier-0 method transitions immediately to the OSR method on resumption.
+The resumption stub created for an OSR method thus points to its corresponding tier-0 code version.
+This is currently implemented by storing a link between OSR methods and corresponding tier-0 methods in `PatchpointInfo`.
+2. When resuming in a tier-0 method, if we transition to the OSR method by normal means, the OSR method will see a non-zero continuation that belongs to the tier-0 method.
+The OSR method must ignore the continuation in this case.
+3. OSR continuations store the IL offset in the `Data` array which complicates propagation of return values into the next continuation, as the exact index to use depends on whether or not this IL offset is present in that continuation.
+This introduces a small amount of overhead in resumption stubs (in terms of a flag check).
+
+### Interop: the async1<->async2 adapter layer
+
+Thunks of the same shapes are provided as for the unwinding prototype.
+
+#### async2 -> async1
+
+The only way to actually begin suspension of a chain of async2 methods happens when calling into an async1 method that suspends.
+As described previously these boundaries are generally expected to be handled by Roslyn by IL codegen that involves a check for `IsCompleted` followed by a call into one of the two `AwaitAwaiterFromRuntimeAsync` variants in the asynchronous case, finally followed by a `GetResult` call.
+The runtime will automatically provide such a stub if an async1 method is called with an async2 signature.
+
+The implementation of `AwaitAwaiterFromRuntimeAsync` is provided by the BCL.
+For the JIT state machines prototype it looks as follows:
+
+```csharp
+private struct RuntimeAsyncAwaitState
+{
+ public Continuation? SentinelContinuation;
+ public INotifyCompletion? Notifier;
+}
+
+[ThreadStatic]
+private static RuntimeAsyncAwaitState t_runtimeAsyncAwaitState;
+
+public static async2 Task AwaitAwaiterFromRuntimeAsync(TAwaiter awaiter) where TAwaiter : INotifyCompletion
+{
+ ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState;
+ Continuation? sentinelContinuation = state.SentinelContinuation;
+ if (sentinelContinuation == null)
+ state.SentinelContinuation = sentinelContinuation = new Continuation();
+
+ state.Notifier = awaiter;
+ SuspendAsync2(sentinelContinuation);
+}
+
+```
+
+The helper starts the suspension by allocating a continuation into TLS, and returning it back to the caller.
+This is going to result in the JIT codegen unwinding through all calling async2 functions and linking all continuations together.
+The end result will be a linked list of continuations where the head of the continuation is available through TLS.
+
+The state stored in TLS is acted on by the async1 -> async2 adapter.
+
+#### async1 -> async2
+The async1 -> async2 adapter, which allows calling an async2 function as if it returns a `Task`, is implemented in the following pseudo-C#:
+
+```csharp
+static Task FooAdapter(int a, int b)
+{
+ int result;
+ Continuation? continuation = null;
+
+ ExecutionAndSyncBlockStore state = new();
+ state.Push();
+
+ try
+ {
+ result = Foo(a, b);
+ continuation = StubHelpers.Async2CallContinuation();
+ }
+ catch (Exception ex)
+ {
+ return Task.FromException(ex);
+ }
+ finally
+ {
+ state.Pop();
+ }
+
+ if (continuation == null)
+ return Task.FromResult(result);
+
+ return FinalizeTaskReturningThunk(continuation);
+}
+```
+
+In the suspended case where `Foo` returns a continuation, we know that the leaf called into the async2 -> async1 adapter described previously, which has made the head of the continuation chain and the leaf awaiter available in TLS.
+
+The remaining work gets handled inside `Task FinalizeTaskReturningThunk(Continuation continuation)`.
+See the code for the details, but logically two things happen:
+1. We link a synthetic continuation to the end of the continuation chain.
+When the last async2 method is resumed and finishes running, it will store its result into the synthetic continuation in the same way as other continuations, making the final result available from there.
+2. We call `OnCompleted` on the awaiter that was saved in TLS.
+The callback passed calls a dispatcher that starts resuming the continuation chain in a loop.
+
+There are complications around proper `ExecutionContext` and `SynchronizationContext` handling.
+We currently implement `FinalizeTaskReturningThunk` as a normal async1 method to get (some of) this handling automatically; see the code for the details.
+
+### Potential future improvements
+* We can avoid allocating a synthetic continuation in the async1 -> async2 case if resumption stubs instead take a pointer for where the return value should be stored.
+The dispatcher should pass the return location of the next continuation if there is one, and otherwise forward a passed-in pointer for the final result.
+* EH is currently handled by rethrowing exceptions at the suspension point.
+It should be possible for the JIT to bypass the repeated VM EH dispatching by directly running the appropriate handlers on resumption.
+* If the JIT always allocates continuations that are large enough for all suspension points within the same method then it should be possible to reuse previously allocated continuations when a resumed function suspends again.
+* With support for multiple entry points to functions we can avoid the overhead on async2 to async2 calls of passing a null continuation, and the overhead of testing this continuation at the beginning of most async2 calls.
+Instead, resumption stubs would resume at a special "resume" entry point that would handle the resumption continuation.
+* Suspending deep async call stacks result in a lot of separate allocations as the call stack unwinds.
+If we passed along the state size on async2 to async2 calls, then it would be possible to allocate the state arrays once in the leaf.
+This would trade off suspension performance for synchronous performance.
+Alternatively we could keep some large `byte[]` and `object[]` arrays in TLS that we stored state in during unwinding, after which we copied the resulting arrays.
+
+## Shared implementation between Unwinder and JIT focused implementation of runtime tasks
+
+It turns out that due to the api design of how async2 code interacts with the existing await pattern in IL, the thunk from async2 to existing Tasks and ValueTasks is the same for all designs.
+
+### Thunk from the async2 surface to Task api implementation
+
+```cs
+async2 Task Thunk(ParameterType param1, ParameterType2 param2, ...)
+{
+ var awaiter = TargetMethod(param1, param2, ...).GetAwaiter();
+ if (!awaiter.IsCompleted)
+ {
+ RuntimeHelpers.UnsafeAwaitAwaiterFromRuntimeAsync(awaiter);
+ }
+ return awaiter.GetResult();
+}
+```
+
+## C# Language changes
+
+For the purpose of the experiment we have extended the syntax and semantics of C# language to support async2 methods. It is not the goal for this syntax to become the syntax in the actual implementation, however we will record our experiences with the experimental syntax in case it is useful when designing the actual syntax.
+
+#### === Syntax v2 (current)
+
+The syntax that is currently in use in the experiment looks like:
+
+```cs
+async2 Task M1(int arg) {}
+
+// meaning in IL
+int32 modopt([System.Runtime]System.Threading.Tasks.Task`1) M1(int arg)
+```
+
+That is basically the same syntax as used by regular async, except for a different modifier. (The name of the modifier is intentionally `async2`, to indicate that the name is at best a placeholder). The syntax naturally generalizes to:
+- Nongeneric Task-returning.
+```cs
+async2 Task M1(int arg) {}
+
+// meaning in IL
+void modopt([System.Runtime]System.Threading.Tasks.Task) M1(int arg)
+```
+- ValueTask-returning.
+```cs
+async2 ValueTask M1(int arg) {}
+
+// meaning in IL
+int32 modopt([System.Runtime]System.Threading.Tasks.ValueTask`1) M1(int arg)
+```
+and combinations of the above.
+
+Internal Roslyn compiler representation of async2 methods is roughly the same as for regular `Task/ValueTask` returning methods, except that the method symbol reports that it is an async2 method thus scenarios that are different for different flavors of async/async2 (i.e. lowering of `await` operator), can work differently.
+
+In a particular case of lowering await operator, if we notice that both containing and awaited members are async2, we can lower the `await M1()` into a direct call to the `int modopt(..)M1()`
+Conversely, if we see an invocation of `M1()` without await or in a regular, non-async2 member, we lower that into a call to `Task M1()` thunk.
+And when inside an async2 function we see an `await` with an argument that is not an async2, we emit corresponding `GetAwaiter`/`IsCompleted`/`UnsafeAwaitAwaiterFromRuntimeAsync`/`GetResult` sequence.
+
+For example.
+```cs
+public async2 Task M1()
+{
+ await Task.Yield();
+ return 3;
+}
+```
+
+is lowered to an equivalent of:
+
+```cs
+public int32 modopt([System.Runtime]System.Threading.Tasks.Task`1) M1()
+{
+ YieldAwaitable.YieldAwaiter awaiter = Task.Yield().GetAwaiter();
+ if (!awaiter.IsCompleted)
+ {
+ RuntimeHelpers.UnsafeAwaitAwaiterFromRuntimeAsync(awaiter);
+ }
+ awaiter.GetResult();
+ return 3;
+}
+```
+
+Operations that are not affected by `async2`, such as generic substitution or Overriding/Hiding/Implementing end up naturally working with async2 methods. Note that these areas are some of the most complex parts of the compiler. It was very beneficial to not having to specialcase async2 in these areas.
+
+One thing to observe is that unlike regular `async`, which is a source-only concept, the async2 survives serialization/deserialization via metadata.
+Effectively we treat the `int32 modopt([System.Runtime]System.Threading.Tasks.Task'1)` as just a special encoding of `Task` return type with additional property of making the method `async2`.
+
+#### Unresolved concerns.
+None of the following appears to be unresolvable or blocking, we just did not get to these due to time constraints and the scoping of the experiment.
+
+1. There is an existing concept for `async void` methods. Does that model need to exist in the new system?
+1. It looks like there might be a need for async2 flavors of delegates.
+To the runtime these would look like delegates with async2 `Invoke` method and may "just work". They would need some kind of syntax in C#.
+1. It looks like there might be a need for async2 flavors of method pointers - to represent async2 methods.
+In the runtime this is representable via the modopt on the return type and, again, might "just work", so this is mostly a matter of picking C# syntax/semantics.
+1. Is there any impact on async enumerable/iterator?
+While async2 is interoperable with regular async and these interfaces could be used or even implemented by the means of async2 methods, there could be opportunities to employ async2 implementations in the lowering. It is not clear if that could be beneficial without defining/deriving async2 versions of `IAsyncEnumerable`/`IAsyncEnumerator`/`IAsyncDisposable`.
+
+#### ==== Syntax v1 (abandoned)
+
+Initially we assumed a different syntax for async2 methods. It ended up not working well beyond simple scenarios. We will still describe that here for future reference as something that appear to be a syntactical dead-end.
+
+The initial version of the syntax looked like:
+
+```cs
+async2 int M1(int arg) {}
+
+// meaning in IL
+int32 modopt([System.Runtime]System.Threading.Tasks.Task`1) M1(int arg)
+```
+
+While initially favored for being close to the IL representation, it was eventually abandoned due to major problems and inconsistencies.
+- When the method above is called from regular code without awaiting, user gets `Task` result. That is not what it looks in the signature.
+- For the purposes of overriding/hiding/inheriting the method above is equivalent to `Task M1(int arg)`, which is easily observable.
+For example it is illegal to add such method in the same class as invoking `Task M1(int arg)` would become ambiguous. On the other hand overloading a method like `int M1(int arg)` is completely non-conflicting.
+This was very confusing, both as a programming model and as internal implementation.
+- Internally, due to duality with `Task M1(int arg)` signature it was often necessary to "lift" the signature into the `Task` form, perform operation there (such as generic substitution or overriding resolution) and then "unlift" the result back into `async2 int` form. It was frequintly a source of subtle bugs.
+- When we got to interfacing with `ValueTask` there was no good place in the syntax to express a variant of the same method, but with `ValueTask`-based thunk.
+For some time we used an attribute and that was verbose and not symmetrical.
+Besides, it is valid for the same class to contain two async2 methods that differ only on `Task`/`ValueTask` flavor of the thunk. It would mean that in the source one could have two methods in the same class, differing only by an attribute.
+
+## Microbenchmarks
+
+# Benchmark of minimum call cost between Async and Async2 functions
+
+The performance of interaction between the Async and Async2 implementation strategies has come up as a concern in developing a replacement strategy for the existing async model.
+Given that this proposal suggests building a model which allows both to co-exist within the same application, understanding the cost of such calls is important.
+This benchmark counts the iterations per second of non-suspending calls between various methods implemented as either async or async2.
+Data for this benchmark is available for both the JIT generated state machine model, as well as the unwinder based model.
+The data was gathered using the `src/tests/Loader/async/async-mincallcost-microbench.csproj` project.
+
+This benchmark works by an async method calling into another mostly empty async method in a loop.
+The benchmark operates for a fixed period of time, measuring the number of iterations.
+The caller method is always a `Task` returning method, and the mostly empty method, reads a global variable, and then awaits the result of a call to `Task.Yield()` if that global variable is `0`. (The variable never has the value '0'.)
+
+
+
+In this graph, you can see that the performance of interaction between the async1 and async2 models are not problematically slow.
+Indeed, the performance of async2 code calling async1 functions is actually greater than the performance of async1 calling async1.
+In addition, the cost to call an async2 function from another async2 function is substantially less (even if the async2 function being called needs to save the context in order to maintain full similarity of behavior with async1.)
+Here we can see the cost of inlining vs not inlining, as there is a meaningful reduction in iteration count.
+In addition, the async2 method with Context Save is also unable to inline at the current time, and also performs work simulating the additional work that the async2 logic would need to implement to fully replicate async1 behavior around async locals, and synchronization context handling.
+
+NOTES:
+- Cost of calls from async1 method to async2 method are shown for the JIT state machine model only.
+The performance of the calls from async1 to async2 for the unwinder model were substantially slower, although if we choose that as the implementation style it is expected they could be made at least as fast as the state machine model.
+- Measurement of calls from async1 to async 2 non-inlineable and with context save were not measured.
+It is expected that in any final implementation there would be optimizations in place in order to ensure that the cost from async1 to either of these two target methods would be the same as calls to any Async2 method.
+
+# Raw data from mincallcost benchmark
+| Caller | Async1 Task returning method | Async1 ValueTask returning method | Async2 Method | Non-inlineable Async2 method | Async2 Method with context save |
+| ------ | ----- | ------ | ---- | ---- | --- |
+| Async1 Task returning method | 117,232,360 | 129,758,886 | 116,365,186 (StateMachine), 76,280,435 (Unwinder) | N/A | N/A |
+| Async2 JIT State machine | 139,952,617 | 145,961,918 | 500,753,190 | 369,495,926 | 220,148,034 |
+| Async2 Unwinder Model | 142,012,835 | 141,336,005 | 500,148,939 | 380,632,094 | 238,234,481 |
+
+
+# Benchmark the performance of EH at various depths.
+
+The performance of EH within the existing async programming model is highly problematic, and has caused customers issues.
+In particular, the impact of async EH performance tends to interfere with responsive performance of applications when failures start ocurring suddenly.
+Since EH is so much slower than standard execution performance, it is known to cause additional downtime as a server which may be capable of handling the load of succesful requests, may run out of available CPU time handling failed requests.
+In addition, EH is commonly used to implement cancellation in async code.
+Thus improving the performance of EH can dramatically improve codebases which make significant use of cancellation.
+The runtime generated async state machine model is showing some substantial improvements to performance.
+
+All of these figures are gathered by using the async-eh-microbench test located at `src/tests/Loader/async/async2-eh-microbench.csproj`.
+This benchmark works by looping until 250 ms has passed and measuring the number of iterations in that time.
+Each iteration works by recursively calling a function until the specified stack depth is achieved (optionally calling through a try/finally block), and then suspending the async method by awaiting `Task.Yield()` or not, and then finally finishing each iteration either by returning or throwing a newly created exception.
+In addition, these experiments use the JIT focused implementation of `async2`. The unwinder based implementation is both significantly slower, and does not support EH.
+
+## Notes on the following graphs
+These benchmarks are gathered by running the async2-eh-microbench in varying configurations.
+- NoSuspend vs Suspend - Suspend indicates that the async function awaits `Task.Yield()` before completing.
+- `Return` vs `Throw` - `Return` indicates that the iteration finishes by returning up the call stack via return instructions, and `Throw` indicates that each iteration terminated via a `throw new Exception()`.
+- `Task` vs `ValueTask` vs `Async2` vs `Async2Capture` - The benchmark has 4 functions which are written with the same C# code, but with different syntax using different async implementation strategies.
+With `Task`, the function signature is `async Task RunTask(int depth)`
+With `ValueTask`, the function signature is `async ValueTask RunTask(int depth)`
+With `Async2`, the function signature is `async2 long RunTask(int depth)`
+With `Async2Capture`, the function signature is `async2 long RunTask(int depth)`, and the body of the function contains a try/finally which saves/restores the Sync and ExecutionContext. This is intended to (imperfectly) measure the cost of adjusting the runtime async model to behave exactly like the existing async model.
+
+When a graph does not specify one of the config options, the graph is a relative performance gathered by dividing the iters/250ms that the benchmark produces between the two options, and then multiplying by 100 to produce a percentage.
+
+## Relative performance of throwing vs returning cleanly from a function at varying stack depths
+
+
+This graph shows the performance of throwing relative to the performance of returning cleanly from a function with varying stack depths.
+There are 2 remarkable bits of data presented here.
+1. The cost of throwing is extraordinary if the application would otherwise be able to complete synchronously.
+2. The overhead of suspending an async function and transitioning to the thread pool is rather high.
+3. The performance of runtime handled async with regards to throwing is much faster than normal EH. In fact as the stack becomes deeper, the EH model presented by runtime generated async functions does not suffer as much of a performance penalty for walking the stack as our normal EH stack walker does.
+
+## Performance of normal async `async Task` code vs runtime generated async functions
+
+In this graph the raw performance of throwing at various stack depths is shown.
+We can see that the performance of `ValueTask` and `Task` is nearly identical, and differs only between whether or not the function suspended or not before throwing.
+In addition, we can see the different performance of throwing with `Async2`.
+Since the design of EH in `Async2` is able to avoid doing expensive operations as it walks a suspended stack, the performance of throwing with various stack depths while suspended is extremely fast, and dominated by the cost of the suspend operation.
+In constrast the non-suspended performance follows the same curve as the `Task` and `ValueTask` model, but is a constant factor faster, as `Async2` produces a smaller call stack, so the runtime's stack walker has substantially less work to do.
+
+
+
+In this graph, the performance of runtime async vs classic async is presented for the particular code in use.
+What we can see is that the performance of the new runtime generated async code is always faster for this code than the traditional code generation.
+Numbers above 100% indicate speedups.
+This higher performance is likely due to the relatively large size of the async function used in this benchmark.
+In addition, in this graph, only data out to a depth of 16 is presented, as the performance of throwing with new async at higher stack depths would make looking at the data for other depths difficult.
+(It reaches a level of 3851% of the performance of the traditional async function at a stack depth of 64). Not shown is the performance of `ValueTask` based async code. It is extremely close to that of the `Task` based variant.
+
+## Impact on performance of capturing/restoring ExecutionContext and SynchronizationContext
+
+
+
+The Async2Capture variant of the testing is intended to show the impact that capturing/restoring `ExecutionContext` and `SynchronizationContext` would have on performance of the runtime generated async logic.
+In this graph, taller bars are better, and if capture had no cost, they would all be expected to be at 100%.
+The variant of code does not necessarily represent all of the costs involved in a real implementation, notably, on restore it does not attempt to ensure that the code resumes respecting the correct `SynchronizationContext` or `TaskScheduler` rules, but its probably fairly accurate for NoSuspend measurements.
+In addition, in this data we can see that the cost of the try/finally results in performance unfortunately similar to that of the `async Task` based implementation in the "Throw Suspend" scenario.
+This could likely be optimized to have performance similar to that of the higher performance runtime async model. (As the contents of the finally are well known, and would not require the entire EH machinery to be invoked.)
+
+As we can see from the graph, the performance impact on NoSuspend cases is between 20%-30% for this somewhat complex function.
+This test also fails to measure the impact of losing the ability to inline small helper functions which was expected to be one of the significant wins in practice of the new async logic.
+
+
+
+However, while this is a substantial performance loss relative to the full speed runtime async model, the overall performance of this model relative to our existing model still shows a nice performance win.
+Looking at this graph, the performance win from a fully capturing model is not terrible.
+We may wish to strongly consider implementing a variant of this where individual methods could choose to have capture or not.
+This would allow the new model to be fully drop in, but customers could opt into higher performance/logic which can support inlining.
+
+## Impact of try/finally regions on throw performance
+
+
+Much of the extraordinary performance of EH the runtime generated async prototype depends on the very fast stack walk that occurs when dispatching exceptions.
+Notably, in the runtime generated model we are able to easily skip frames which do not have any handlers in them with a simple flag check.
+This is contrary to IL compiler generated async, where the implementation generates a catch, even if the frame has no try block written in C#.
+
+We can see some very interesting details.
+1. In IL based async the cost of suspending once is dominated by the cost of throwing an exception, and so whether or not the function suspends, does not change the performance.
+2. In the NoSuspend case, for async2 we can see that the presence or abscence of a finally has a fairly small impact on throw performance.
+Most of the cost is taken up by other tasks such as stackwalking, creating the exception object, and the like.
+3. In the case where we DO suspend, the performance is quite different with async2.
+Without many finally blocks, we can see that the performance is really extraordinary, but as each frame gets a finally block, the cost of handling EH approaches that of the traditional async model.
+4. The performance of Task Throw NoSuspend and Task Throw Suspend is effectively the same.
+
+## Raw data from EH microbenchmark
+| Stack Depth | Task Return NoSuspend | ValueTask Return NoSuspend | Async2 Return NoSuspend | Task Throw NoSuspend | ValueTask Throw NoSuspend | Async2 Throw NoSuspend | Task Return Suspend | ValueTask Return Suspend | Async2 Return Suspend | Task Throw Suspend | ValueTask Throw Suspend | Async2 Throw Suspend | Async2Capture Return NoSuspend | Async2Capture Throw NoSuspend | Async2Capture Return Suspend | Async2Capture Throw Suspend |
+| ----- | --------------------- | -------------------------- | ----------------------- | -------------------- | ------------------------- | ---------------------- | ------------------- | ------------------------ | --------------------- | ------------------ | ----------------------- | -------------------- | --------------------- | ------------------ | ----------------------- | -------------------- |
+| 1 | 7,976,995.00 | 7,994,790.00 | 13,966,110.00 | 21,630.00 | 21,789.00 | 46,434.00 | 589,057.00 | 574,660.00 | 629,310.00 | 15,563.00 | 15,584.00 | 21,208.00 | 12249438.00 | 45002.00 | 624336.00 | 21005.00 |
+| 2 | 4,738,359.00 | 5,036,613.00 | 11,573,294.00 | 13,781.00 | 13,359.00 | 38,975.00 | 390,167.00 | 349,900.00 | 617,464.00 | 9,806.00 | 9,740.00 | 21,514.00 | 9306011.00 | 37769.00 | 594730.00 | 13163.00 |
+| 4 | 2,884,153.00 | 2,924,123.00 | 8,340,047.00 | 7,717.00 | 7,477.00 | 29,887.00 | 276,584.00 | 265,590.00 | 577,707.00 | 5,778.00 | 5,495.00 | 21,288.00 | 6438143.00 | 28318.00 | 543293.00 | 7253.00 |
+| 8 | 1,274,335.00 | 1,324,960.00 | 5,308,395.00 | 4,068.00 | 4,001.00 | 20,358.00 | 217,067.00 | 198,172.00 | 474,723.00 | 3,476.00 | 3,376.00 | 20,877.00 | 3836147.00 | 18871.00 | 412340.00 | 4160.00 |
+| 16 | 536,064.00 | 533,731.00 | 2,888,828.00 | 2,104.00 | 2,063.00 | 12,941.00 | 132,636.00 | 106,605.00 | 315,191.00 | 1,912.00 | 1,909.00 | 20,496.00 | 2083270.00 | 11978.00 | 275944.00 | 2234.00 |
+| 32 | 263,823.00 | 249,821.00 | 1,157,832.00 | 1,032.00 | 989.00 | 7,319.00 | 71,989.00 | 55,860.00 | 184,699.00 | 973.00 | 949.00 | 19,379.00 | 861826.00 | 6705.00 | 156610.00 | 1133.00 |
+| 64 | 113,187.00 | 114,845.00 | 488,245.00 | 484.00 | 473.00 | 3,936.00 | 37,086.00 | 28,365.00 | 98,769.00 | 458.00 | 456.00 | 17,565.00 | 393380.00 | 3654.00 | 83977.00 | 593.00 |
+
+| Finally blocks on stack | Task Throw NoSuspend | Async2 Throw NoSuspend | Task Throw Suspend | Async2 Throw Suspend |
+| ----------------------- | -------------------- | ---------------------- | ------------------ | -------------------- |
+| 1 | 1067 | 7504 | 965 | 20107 |
+| 2 | 1032 | 7441 | 965 | 12140 |
+| 4 | 1034 | 7392 | 959 | 7306 |
+| 8 | 1006 | 7287 | 928 | 4268 |
+| 16 | 972 | 7045 | 912 | 2307 |
+| 32 | 872 | 6961 | 857 | 1194 |
+
+## Scenario benchmarks
+
+1. Serialization/Deserialization of JSON data to an asynchronous stream
+2. ASP.NET servicing of requests where must of the pipeline is converted to the async2 model
+
+## On supporting span/byref variables in runtime-assisted async code.
+
+It is not a secret that inability to use span and byref with async code causes some amount of grief. The async/span impedance in particular appears to be unnecessarily constraining. That is likely because span caters to a wider audience and thus use of span often intersects with async. A simple search brings up a lot of user concerns, workarounds, and asks to relax the constraint at least partially, if possible -
+
+https://stackoverflow.com/questions/57229123/how-to-use-spanbyte-in-async-method
+
+https://stackoverflow.com/questions/63284335/spant-and-async-methods
+
+[Span in async methods not supported · Issue #27147 · dotnet/roslyn (github.com)] (https://github.com/dotnet/roslyn/issues/27147)
+
+https://stackoverflow.com/questions/20868103/ref-and-out-arguments-in-async-method
+
+[Consider relaxing restrictions of async methods in blocks that do not contain `await`. · Issue 1331 · dotnet/csharplang (github.com)](https://github.com/dotnet/csharplang/issues/1331) Lots of discussions here, including insightful comments by Vance.
+
+The current situation is that C# does not allow byrefs and byref-likes in async methods in any form – be it parameters, locals or spans. The reason for this restriction is inability to capture byrefs as fields of display types. For consistency all use is forbidden, even if it does not require capture.
+There are few cases were C# supports transient byrefs by the means of decomposing a byref-producing expression into constituent parts and capturing the parts and re-playing at use sites. Example `staticArray[i].Field += await Somehting();` That allows to handle a subset of cases, but codegen is far from great.
+
+In runtime-assisted async supporting span and byrefs is not a strict “no”, but there are implications. There are roughly two kinds of byrefs that async implementation would need to deal with. They come with distinct challenges.
+
+**Heap-referencing byrefs** are challenging because GC reporting of byrefs is only naturally supported on stack. Any alternative storage will have to implement a scheme for reporting stored variables for marking/updating purposes. The main challenge here is performance. As we expect the number of tasks to be easily in thousands at a time, O(n) algorithms inevitably result in GC pauses. There are ways to mitigate this, some of which were explored and measured as a part of this experiment.
+
+**Stack-referencing byrefs** do not have to be reported to GC, but must be adjusted when containing frame is reactivated as the referent is either in the current frame and thus now lives in a different stack location, or in one of still suspended caller frames and thus not on the stack at all. In the presence of “ref Span param” and similar, updating upon suspending is also necessary to keep chains of byrefs safely walkable.
+
+In general it would not be a tractable programming model from the user perspective, if we allow only one kind of byrefs and not another as distinction may often only be known at run time.
+Note that the stack/heap-referencing nature of a byref cannot change while the frame is not active. However that is not completely true if “ref Span param” is allowed. In such case a calee may repoint a byref in a caller frame and switch from stack-pointing to heap-pointing.
+
+**Stack-referencing byrefs, that lead into sync callers** these are unsafe as the sync caller may exit any or all its stack frames before the Task is resumed.
+
+In the current experiment we have looked into two implementing strategies:
+
+**Stack unwinding with 1:1 stack capture for suspended frames.** In this model byrefs/spans that point to heap or other async callers can be supported as there is a mechanism to report byrefs to the GC. There are some performance challenges, but not without solutions.
+
+**State machine with managed storage for captured variables.** The current implementation does not support capturing byrefs and byref-likes. On the other hand it can utilize regular GC reporting mechanisms, which has many advantages, notably not having O(n) components that need to run in GC pauses. There are some ideas on how byrefs may be captured and converted into {object, offset} at the next GC, so that not to incur continuous O(n) costs.
+
+## On unmanaged pointers.
+We cannot possibly track unmanaged pointers that refer to the stack across suspensions. Disallowing such scenarios would not be a take-back though since currently C# does not allow `await` in unsafe context, therefore await and unmanaged pointers cannot mix.
+
+## Impact on GC (heap size, pauses).
+
+We have examined the behavior of experimental async2 implementations on GC. For simplicity we looked into behavior of a simple benchmark that ensures 10000 suspended tasks at some point, each containing 100 stack frames with at least 1 object and 1 integer alive across await and thus requiring capture. At the time the tasks are suspended, the benchmark forces a number of Gen0 GCs and measures average time taken by GCs as well as the managed heap size.
+To further reduce the number of involved variables, the benchmark is single-threaded and Workstation GC was used.
+
+The benchmark produces very little garbage when the masurements are performed as the main interest is not on how GC handles allocations, but on how the presence of captured tasks impacts GC.
+
+Our observations (on x64, AMD 5950X):
+
+**===== Async1**:
+
+There is some noise, but a typical Gen0 pause in the above scenario was: 120 microseconds.
+The managed heap size (as reported by `GetGCMemoryInfo.HeapSizeBytes`) is stable at 130 Mb.
+The peak working set (as reported by `Task Manager`) is 148 Mb.
+
+**=====Async2 (unwinding/stack-capturing based)**:
+
+When GC pauses were measured as-is, the results were very disappointing - **48690 microsecond**. (**~400x worse** than async1 and generally unacceptable for a Gen0 pause)
+
+The reasons for such timings is that the implementation would go through every one of the 1000000 tasklets and report referenced objects as roots. That is time consuming and is also very redundant when GC generational model is considered. Since suspended stacks can't see assignments of new objects, once GC performs en-masse promotion, all the roots owned by these tasks become too old to be of interest for collections that target younger generations until the containing task is resumed.
+This is a unique challenge of stack-capturing implementation. In async1 suspended tasks are regular heap objects and do not require any special reporting in GC STW phases.
+
+As a solution to this issue, we implemented tasklet aging mechanism, similar to approach used by GC handles. We make note of GC promotions, and increase "min age" of suspended tasks, so that, for example, tasks that only refer to tenured objects do not need to report roots in ephemeral collections.
+
+The taklet aging was an enormous optimization and the pause time in the above benchmark went down to **480 microseconds**. That is **100x improvement** and is not too bad, considering that this is close to the worst case scenario.
+It was still **4x worse** than async1.
+
+Once tasks know their age, root reporting is no longer the dominant factor, but we still need to check age of 10000 tasks, even if the answer is "too old". A further improvement is possible by grouping the suspended tasks in groups, so that age of entire group could be examined.
+
+For the memory consumption, the managed heap is reported to be very small - **28Mb**. Unsurprisingly, since this approach uses native/malloc memory to store captured data.
+However, the peak working set was seen at **499Mb** that is **3.3x worse** than async1.
+
+The reason for higher memory consumption is that this approach captures the entire stack frame and its registers, which is less eficient than async1, which, based on data-flow analysis, may capture only variables that are live across awaits.
+
+**=====Async2 (JIT state-machine based)**:
+
+The pause times were observed at **100 microseconds**, which is **better than in async1** implementation.
+Since the root set would be roughly the same as in async1, the difference could be just second-order effects of different heap graph or different distribution of object sizes.
+
+The peak working set was seen at 328Mb.
+That is higher than async1 and most likely just additional transient allocations in the JIT. As the benchmark has just a few methods, it cannot be method bodies, but may need some follow up to confirm.
+
+Managed allocation impact in this model depends on JIT optimizations as the capture is based on liveness information, which is not available in tier-0 compiled methods. That is expected and could be acceptable as hot methods would not stay in tier-0. Besides it is possible to enable liveness analysis in tier-0 async2 methods, if needed, at about 5% of the estimated JIT cost.
+It is also possible to improve liveness analysis/information in async2 methods in general - to further improve capturing efficiency.
+
+With `set DOTNET_TieredCompilation=0`
+Managed heap size: **157Mb** - slightly more than async1
+
+With `set DOTNET_TieredCompilation=1`
+Managed heap size: **reaches 300Mb** in the first couple iterations, but then drops and **stays at 105Mb**, which is better than async1.
diff --git a/docs/design/features/throw-perf-with-different-number-of-finally-on-stack.svg b/docs/design/features/throw-perf-with-different-number-of-finally-on-stack.svg
new file mode 100644
index 00000000000000..59d27773c75f47
--- /dev/null
+++ b/docs/design/features/throw-perf-with-different-number-of-finally-on-stack.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/docs/design/features/throwing-relative-performance.svg b/docs/design/features/throwing-relative-performance.svg
new file mode 100644
index 00000000000000..521e9086fd6929
--- /dev/null
+++ b/docs/design/features/throwing-relative-performance.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/src/coreclr/System.Private.CoreLib/CompatibilitySuppressions.xml b/src/coreclr/System.Private.CoreLib/CompatibilitySuppressions.xml
new file mode 100644
index 00000000000000..e13316ffc3cfb7
--- /dev/null
+++ b/src/coreclr/System.Private.CoreLib/CompatibilitySuppressions.xml
@@ -0,0 +1,28 @@
+
+
+
+
+ CP0001
+ T:Internal.Console
+
+
+ CP0002
+ F:System.Resources.ResourceManager.BaseNameField
+
+
+ CP0002
+ F:System.Resources.ResourceSet.Reader
+
+
+ CP0002
+ M:System.String.Trim(System.ReadOnlySpan{System.Char})
+
+
+ CP0002
+ M:System.String.TrimEnd(System.ReadOnlySpan{System.Char})
+
+
+ CP0002
+ M:System.String.TrimStart(System.ReadOnlySpan{System.Char})
+
+
\ No newline at end of file
diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs
index d60b48ec0e552e..f35eb16591572c 100644
--- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs
+++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs
@@ -9,9 +9,43 @@
using System.Runtime.Serialization;
using System.Runtime.Versioning;
using System.Threading;
+using System.Threading.Tasks;
namespace System.Runtime.CompilerServices
{
+ internal struct ExecutionAndSyncBlockStore
+ {
+ // Store current ExecutionContext and SynchronizationContext as "previousXxx".
+ // This allows us to restore them and undo any Context changes made in stateMachine.MoveNext
+ // so that they won't "leak" out of the first await.
+ public ExecutionContext? _previousExecutionCtx;
+ public SynchronizationContext? _previousSyncCtx;
+ public Thread _thread;
+
+ public void Push()
+ {
+ _thread = Thread.CurrentThread;
+ _previousExecutionCtx = _thread._executionContext;
+ _previousSyncCtx = _thread._synchronizationContext;
+ }
+
+ public void Pop()
+ {
+ // The common case is that these have not changed, so avoid the cost of a write barrier if not needed.
+ if (_previousSyncCtx != _thread._synchronizationContext)
+ {
+ // Restore changed SynchronizationContext back to previous
+ _thread._synchronizationContext = _previousSyncCtx;
+ }
+
+ ExecutionContext? currentExecutionCtx = _thread._executionContext;
+ if (_previousExecutionCtx != currentExecutionCtx)
+ {
+ ExecutionContext.RestoreChangedContextToThread(_thread, _previousExecutionCtx, currentExecutionCtx);
+ }
+ }
+ }
+
public static partial class RuntimeHelpers
{
[Intrinsic]
@@ -544,6 +578,304 @@ private static unsafe void DispatchTailCalls(
}
}
+ private static Continuation AllocContinuation(Continuation prevContinuation, nuint numGCRefs, nuint dataSize)
+ {
+ Continuation newContinuation = new Continuation { Data = new byte[dataSize], GCData = new object[numGCRefs] };
+ prevContinuation.Next = newContinuation;
+ return newContinuation;
+ }
+
+ private static unsafe Continuation AllocContinuationMethod(Continuation prevContinuation, nuint numGCRefs, nuint dataSize, MethodDesc* method)
+ {
+ LoaderAllocator loaderAllocator = RuntimeMethodHandle.GetLoaderAllocator(new RuntimeMethodHandleInternal((IntPtr)method));
+ object?[] gcData;
+ if (loaderAllocator != null)
+ {
+ gcData = new object[numGCRefs + 1];
+ gcData[numGCRefs] = loaderAllocator;
+ }
+ else
+ {
+ gcData = new object[numGCRefs];
+ }
+
+ Continuation newContinuation = new Continuation { Data = new byte[dataSize], GCData = gcData };
+ prevContinuation.Next = newContinuation;
+ return newContinuation;
+ }
+
+ private static unsafe Continuation AllocContinuationClass(Continuation prevContinuation, nuint numGCRefs, nuint dataSize, MethodTable* methodTable)
+ {
+ IntPtr loaderAllocatorHandle = methodTable->GetLoaderAllocatorHandle();
+ object?[] gcData;
+ if (loaderAllocatorHandle != IntPtr.Zero)
+ {
+ gcData = new object[numGCRefs + 1];
+ gcData[numGCRefs] = GCHandle.FromIntPtr(loaderAllocatorHandle).Target;
+ }
+ else
+ {
+ gcData = new object[numGCRefs];
+ }
+
+ Continuation newContinuation = new Continuation { Data = new byte[dataSize], GCData = gcData };
+ prevContinuation.Next = newContinuation;
+ return newContinuation;
+ }
+
+ // We are allocating a box directly instead of relying on regular boxing because we want
+ // to store structs without changing layout, including nullables.
+ private static unsafe object AllocContinuationResultBox(void* ptr)
+ {
+ MethodTable* pMT = (MethodTable*)ptr;
+ Debug.Assert(pMT->IsValueType);
+ // We need no type/cctor checks since we will be storing an instance that already exists.
+ return RuntimeTypeHandle.InternalAllocNoChecks((MethodTable*)pMT);
+ }
+ private struct RuntimeAsyncAwaitState
+ {
+ public Continuation? SentinelContinuation;
+ public INotifyCompletion? Notifier;
+ }
+
+ [ThreadStatic]
+ private static RuntimeAsyncAwaitState t_runtimeAsyncAwaitState;
+
+ [Intrinsic]
+ private static void SuspendAsync2(Continuation continuation) => throw new UnreachableException();
+
+ private struct AwaitableProxy : ICriticalNotifyCompletion
+ {
+ private readonly INotifyCompletion _notifier;
+
+ public AwaitableProxy(INotifyCompletion notifier)
+ {
+ _notifier = notifier;
+ }
+
+ public bool IsCompleted => false;
+
+ public void OnCompleted(Action action)
+ {
+ _notifier!.OnCompleted(action);
+ }
+
+ public AwaitableProxy GetAwaiter() { return this; }
+
+ public void UnsafeOnCompleted(Action action)
+ {
+ if (_notifier is ICriticalNotifyCompletion criticalNotification)
+ {
+ criticalNotification.UnsafeOnCompleted(action);
+ }
+ else
+ {
+ _notifier!.OnCompleted(action);
+ }
+ }
+
+ public void GetResult() {}
+ }
+
+ private static Continuation UnlinkHeadContinuation(out AwaitableProxy awaitableProxy)
+ {
+ ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState;
+ awaitableProxy = new AwaitableProxy(state.Notifier!);
+ state.Notifier = null;
+
+ Continuation sentinelContinuation = state.SentinelContinuation!;
+ Continuation head = sentinelContinuation.Next!;
+ sentinelContinuation.Next = null;
+ return head;
+ }
+
+ private static async Task FinalizeTaskReturningThunk(Continuation continuation)
+ {
+ Continuation finalContinuation = new Continuation();
+
+ // Note that the exact location the return value is placed is tied
+ // into getAsyncResumptionStub in the VM, so do not change this
+ // without also changing that code (and the JIT).
+ if (IsReferenceOrContainsReferences())
+ {
+ finalContinuation.Flags = CorInfoContinuationFlags.CORINFO_CONTINUATION_RESULT_IN_GCDATA | CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION;
+ finalContinuation.GCData = new object[1];
+ }
+ else
+ {
+ finalContinuation.Flags = CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION;
+ finalContinuation.Data = new byte[Unsafe.SizeOf()];
+ }
+
+ continuation.Next = finalContinuation;
+
+ while (true)
+ {
+ Continuation headContinuation = UnlinkHeadContinuation(out var awaitableProxy);
+ await awaitableProxy;
+ Continuation? finalResult = DispatchContinuations(headContinuation);
+ if (finalResult != null)
+ {
+ Debug.Assert(finalResult == finalContinuation);
+ if (IsReferenceOrContainsReferences())
+ {
+ if (typeof(T).IsValueType)
+ {
+ return Unsafe.As(ref finalResult.GCData![0]!.GetRawData());
+ }
+
+ return Unsafe.As(ref finalResult.GCData![0]!);
+ }
+ else
+ {
+ return Unsafe.As(ref finalResult.Data![0]);
+ }
+ }
+ }
+ }
+
+ private static async Task FinalizeTaskReturningThunk(Continuation continuation)
+ {
+ Continuation finalContinuation = new Continuation
+ {
+ Flags = CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION,
+ };
+ continuation.Next = finalContinuation;
+
+ while (true)
+ {
+ Continuation headContinuation = UnlinkHeadContinuation(out var awaitableProxy);
+ await awaitableProxy;
+ Continuation? finalResult = DispatchContinuations(headContinuation);
+ if (finalResult != null)
+ {
+ Debug.Assert(finalResult == finalContinuation);
+ return;
+ }
+ }
+ }
+
+ private static async ValueTask FinalizeValueTaskReturningThunk(Continuation continuation)
+ {
+ Continuation finalContinuation = new Continuation();
+
+ // Note that the exact location the return value is placed is tied
+ // into getAsyncResumptionStub in the VM, so do not change this
+ // without also changing that code (and the JIT).
+ if (IsReferenceOrContainsReferences())
+ {
+ finalContinuation.Flags = CorInfoContinuationFlags.CORINFO_CONTINUATION_RESULT_IN_GCDATA | CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION;
+ finalContinuation.GCData = new object[1];
+ }
+ else
+ {
+ finalContinuation.Flags = CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION;
+ finalContinuation.Data = new byte[Unsafe.SizeOf()];
+ }
+
+ continuation.Next = finalContinuation;
+
+ while (true)
+ {
+ Continuation headContinuation = UnlinkHeadContinuation(out var awaitableProxy);
+ await awaitableProxy;
+ Continuation? finalResult = DispatchContinuations(headContinuation);
+ if (finalResult != null)
+ {
+ Debug.Assert(finalResult == finalContinuation);
+ if (IsReferenceOrContainsReferences())
+ {
+ if (typeof(T).IsValueType)
+ {
+ return Unsafe.As(ref finalResult.GCData![0]!.GetRawData());
+ }
+
+ return Unsafe.As(ref finalResult.GCData![0]!);
+ }
+ else
+ {
+ return Unsafe.As(ref finalResult.Data![0]);
+ }
+ }
+ }
+ }
+
+ private static async ValueTask FinalizeValueTaskReturningThunk(Continuation continuation)
+ {
+ Continuation finalContinuation = new Continuation
+ {
+ Flags = CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION,
+ };
+ continuation.Next = finalContinuation;
+
+ while (true)
+ {
+ Continuation headContinuation = UnlinkHeadContinuation(out var awaitableProxy);
+ await awaitableProxy;
+ Continuation? finalResult = DispatchContinuations(headContinuation);
+ if (finalResult != null)
+ {
+ Debug.Assert(finalResult == finalContinuation);
+ return;
+ }
+ }
+ }
+
+ // Return a continuation object if that is the one which has the final
+ // result of the Task, if the real output of the series of continuations was
+ // an exception, it is allowed to propagate out.
+ // OR
+ // return NULL to indicate that this isn't yet done.
+ private static unsafe Continuation? DispatchContinuations(Continuation? continuation)
+ {
+ Debug.Assert(continuation != null);
+
+ while (true)
+ {
+ Continuation? newContinuation;
+ try
+ {
+ newContinuation = continuation.Resume(continuation);
+ }
+ catch (Exception ex)
+ {
+ continuation = UnwindToPossibleHandler(continuation);
+ if (continuation.Resume == null)
+ {
+ throw;
+ }
+
+ continuation.GCData![(continuation.Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_RESULT_IN_GCDATA) != 0 ? 1 : 0] = ex;
+ continue;
+ }
+
+ if (newContinuation != null)
+ {
+ newContinuation.Next = continuation.Next;
+ return null;
+ }
+
+ continuation = continuation.Next;
+ Debug.Assert(continuation != null);
+
+ if (continuation.Resume == null)
+ {
+ return continuation; // Return the result containing Continuation
+ }
+ }
+ }
+
+ private static Continuation UnwindToPossibleHandler(Continuation continuation)
+ {
+ while (true)
+ {
+ Debug.Assert(continuation.Next != null);
+ continuation = continuation.Next;
+ if ((continuation.Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION) != 0)
+ return continuation;
+ }
+ }
+
///
/// Create a boxed object of the specified type from the data located at the target reference.
///
@@ -587,6 +919,7 @@ public static int SizeOf(RuntimeTypeHandle type)
return result;
}
}
+
// Helper class to assist with unsafe pinning of arbitrary objects.
// It's used by VM code.
[NonVersionable] // This only applies to field layout
@@ -930,6 +1263,9 @@ public uint GetNumInstanceFieldBytesIfContainsGCPointers()
Debug.Assert((BaseSize - (nuint)(2 * sizeof(IntPtr)) == GetNumInstanceFieldBytes()));
return BaseSize - (uint)(2 * sizeof(IntPtr));
}
+
+ [MethodImpl(MethodImplOptions.InternalCall)]
+ public extern IntPtr GetLoaderAllocatorHandle();
}
// Subset of src\vm\typedesc.h
@@ -1204,4 +1540,52 @@ internal unsafe struct TailCallTls
public PortableTailCallFrame* Frame;
public IntPtr ArgBuffer;
}
+
+ [Flags]
+ internal enum CorInfoContinuationFlags
+ {
+ // Whether or not the continuation expects the result to be boxed and
+ // placed in the GCData array at index 0. Not set if the callee is void.
+ CORINFO_CONTINUATION_RESULT_IN_GCDATA = 1,
+ // If this bit is set the continuation resumes inside a try block and thus
+ // if an exception is being propagated, needs to be resumed. The exception
+ // should be placed at index 0 or 1 depending on whether the continuation
+ // also expects a result.
+ CORINFO_CONTINUATION_NEEDS_EXCEPTION = 2,
+ // If this bit is set the continuation has an OSR IL offset saved in the
+ // beginning of 'Data'.
+ CORINFO_CONTINUATION_OSR_IL_OFFSET_IN_DATA = 4,
+ }
+
+ internal sealed unsafe class Continuation
+ {
+ public Continuation? Next;
+ public delegate* Resume;
+ public uint State;
+ public CorInfoContinuationFlags Flags;
+
+ // Data and GCData contain the state of the continuation.
+ // Note: The JIT is ultimately responsible for laying out these arrays.
+ // However, other parts of the system depend on the layout to
+ // know where to locate or place various pieces of data:
+ //
+ // 1. Resumption stubs need to know where to place the return value
+ // inside the next continuation. If the return value has GC references
+ // then it is boxed and placed at GCData[0]; otherwise, it is placed
+ // inside Data at offset 0 if
+ // CORINFO_CONTINUATION_OSR_IL_OFFSET_IN_DATA is NOT set and otherwise
+ // at offset 4.
+ //
+ // 2. Likewise, Finalize[Value]TaskReturningThunk needs to know from
+ // where to extract the return value.
+ //
+ // 3. The dispatcher needs to know where to place the exception inside
+ // the next continuation with a handler. Continuations with handlers
+ // have CORINFO_CONTINUATION_NEEDS_EXCEPTION set. The exception is
+ // placed at GCData[0] if CORINFO_CONTINUATION_RESULT_IN_GCDATA is NOT
+ // set, and otherwise at GCData[1].
+ //
+ public byte[]? Data;
+ public object?[]? GCData;
+ }
}
diff --git a/src/coreclr/System.Private.CoreLib/src/System/RuntimeHandles.cs b/src/coreclr/System.Private.CoreLib/src/System/RuntimeHandles.cs
index 1d70bed0c21445..8e03b99165d557 100644
--- a/src/coreclr/System.Private.CoreLib/src/System/RuntimeHandles.cs
+++ b/src/coreclr/System.Private.CoreLib/src/System/RuntimeHandles.cs
@@ -299,6 +299,13 @@ internal static object InternalAlloc(RuntimeType type)
[LibraryImport(RuntimeHelpers.QCall, EntryPoint = "RuntimeTypeHandle_InternalAlloc")]
private static unsafe partial void InternalAlloc(MethodTable* pMT, ObjectHandleOnStack result);
+ internal static object InternalAllocNoChecks(MethodTable* pMT)
+ {
+ object? result = null;
+ InternalAllocNoChecks(pMT, ObjectHandleOnStack.Create(ref result));
+ return result!;
+ }
+
internal static object InternalAllocNoChecks(RuntimeType type)
{
Debug.Assert(!type.GetNativeTypeHandle().IsTypeDesc);
diff --git a/src/coreclr/System.Private.CoreLib/src/System/RuntimeType.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/RuntimeType.CoreCLR.cs
index 5e872581a02f60..cbee1aff762651 100644
--- a/src/coreclr/System.Private.CoreLib/src/System/RuntimeType.CoreCLR.cs
+++ b/src/coreclr/System.Private.CoreLib/src/System/RuntimeType.CoreCLR.cs
@@ -407,34 +407,34 @@ internal void Insert(ref T[] list, string? name, MemberListType listType)
switch (listType)
{
case MemberListType.CaseSensitive:
+ {
+ // Ensure we always return a list that has
+ // been merged with the global list.
+ T[]? cachedList = m_csMemberInfos[name!];
+ if (cachedList == null)
{
- // Ensure we always return a list that has
- // been merged with the global list.
- T[]? cachedList = m_csMemberInfos[name!];
- if (cachedList == null)
- {
- MergeWithGlobalList(list);
- m_csMemberInfos[name!] = list;
- }
- else
- list = cachedList;
+ MergeWithGlobalList(list);
+ m_csMemberInfos[name!] = list;
}
- break;
+ else
+ list = cachedList;
+ }
+ break;
case MemberListType.CaseInsensitive:
+ {
+ // Ensure we always return a list that has
+ // been merged with the global list.
+ T[]? cachedList = m_cisMemberInfos[name!];
+ if (cachedList == null)
{
- // Ensure we always return a list that has
- // been merged with the global list.
- T[]? cachedList = m_cisMemberInfos[name!];
- if (cachedList == null)
- {
- MergeWithGlobalList(list);
- m_cisMemberInfos[name!] = list;
- }
- else
- list = cachedList;
+ MergeWithGlobalList(list);
+ m_cisMemberInfos[name!] = list;
}
- break;
+ else
+ list = cachedList;
+ }
+ break;
case MemberListType.All:
if (!m_cacheComplete)
@@ -2371,7 +2371,7 @@ private static bool FilterApplyMethodBase(
#endregion
-#endregion
+ #endregion
#region Private Data Members
diff --git a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
index 5b33c503678275..b5dd3fe1efd726 100644
--- a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
+++ b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
@@ -1598,6 +1598,9 @@ internal static void MulticastDebuggerTraceHelper(object o, int count)
[Intrinsic]
[MethodImpl(MethodImplOptions.InternalCall)]
internal static extern IntPtr NextCallReturnAddress();
+
+ [Intrinsic]
+ internal static Continuation? Async2CallContinuation() => null;
} // class StubHelpers
#if FEATURE_COMINTEROP
diff --git a/src/coreclr/inc/corcompile.h b/src/coreclr/inc/corcompile.h
index 57ca94832e6be7..d9810b559be02b 100644
--- a/src/coreclr/inc/corcompile.h
+++ b/src/coreclr/inc/corcompile.h
@@ -185,6 +185,7 @@ enum EncodeMethodSigFlags
ENCODE_METHOD_SIG_Constrained = 0x20,
ENCODE_METHOD_SIG_OwnerType = 0x40,
ENCODE_METHOD_SIG_UpdateContext = 0x80,
+ ENCODE_METHOD_SIG_Async2Variant = 0x100,
};
enum EncodeFieldSigFlags
diff --git a/src/coreclr/inc/corhdr.h b/src/coreclr/inc/corhdr.h
index 84f7ebcf428b75..20173170693564 100644
--- a/src/coreclr/inc/corhdr.h
+++ b/src/coreclr/inc/corhdr.h
@@ -640,13 +640,15 @@ typedef enum CorMethodImpl
miNoOptimization = 0x0040, // Method may not be optimized.
miAggressiveOptimization = 0x0200, // Method may contain hot code and should be aggressively optimized.
+ miAsync = 0x0400, // Method requires async state machine rewrite.
+
// These are the flags that are allowed in MethodImplAttribute's Value
// property. This should include everything above except the code impl
// flags (which are used for MethodImplAttribute's MethodCodeType field).
miUserMask = miManagedMask | miForwardRef | miPreserveSig |
miInternalCall | miSynchronized |
miNoInlining | miAggressiveInlining |
- miNoOptimization | miAggressiveOptimization,
+ miNoOptimization | miAggressiveOptimization | miAsync,
miMaxMethodImplVal = 0xffff, // Range check value
} CorMethodImpl;
@@ -670,6 +672,7 @@ typedef enum CorMethodImpl
#define IsMiAggressiveInlining(x) ((x) & miAggressiveInlining)
#define IsMiNoOptimization(x) ((x) & miNoOptimization)
#define IsMiAggressiveOptimization(x) (((x) & (miAggressiveOptimization | miNoOptimization)) == miAggressiveOptimization)
+#define IsMiAsync(x) ((x) & miAsync)
// PinvokeMap attr bits, used by DefinePinvokeMap.
typedef enum CorPinvokeMap
diff --git a/src/coreclr/inc/corhlprpriv.h b/src/coreclr/inc/corhlprpriv.h
index 7df77ea0ca6281..e960eb416fc984 100644
--- a/src/coreclr/inc/corhlprpriv.h
+++ b/src/coreclr/inc/corhlprpriv.h
@@ -573,6 +573,16 @@ class CQuickArrayList : protected CQuickArray
return m_curSize;
}
+ T* Ptr()
+ {
+ return (T*) CQuickBytesBase::Ptr();
+ }
+
+ const T* Ptr() const
+ {
+ return (T*) CQuickBytesBase::Ptr();
+ }
+
void Shrink()
{
CQuickArray::Shrink(m_curSize);
diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h
index 08970ec4102e9b..fa8fea85f360bd 100644
--- a/src/coreclr/inc/corinfo.h
+++ b/src/coreclr/inc/corinfo.h
@@ -405,6 +405,7 @@ enum CorInfoHelpFunc
CORINFO_HELP_THROW, // Throw an exception object
CORINFO_HELP_RETHROW, // Rethrow the currently active exception
+ CORINFO_HELP_THROWEXACT, // Throw an exception object, preserving stack trace
CORINFO_HELP_USER_BREAKPOINT, // For a user program to break to the debugger
CORINFO_HELP_RNGCHKFAIL, // array bounds check failed
CORINFO_HELP_OVERFLOW, // throw an overflow exception
@@ -579,6 +580,7 @@ enum CorInfoHelpFunc
CORINFO_HELP_PATCHPOINT, // Notify runtime that code has reached a patchpoint
CORINFO_HELP_PARTIAL_COMPILATION_PATCHPOINT, // Notify runtime that code has reached a part of the method that wasn't originally jitted.
+ CORINFO_HELP_RESUME_OSR, // Resume in an OSR version of the code for the specified IL offset
CORINFO_HELP_CLASSPROFILE32, // Update 32-bit class profile for a call site
CORINFO_HELP_CLASSPROFILE64, // Update 64-bit class profile for a call site
@@ -594,6 +596,10 @@ enum CorInfoHelpFunc
CORINFO_HELP_VALIDATE_INDIRECT_CALL, // CFG: Validate function pointer
CORINFO_HELP_DISPATCH_INDIRECT_CALL, // CFG: Validate and dispatch to pointer
+ CORINFO_HELP_ALLOC_CONTINUATION,
+ CORINFO_HELP_ALLOC_CONTINUATION_METHOD,
+ CORINFO_HELP_ALLOC_CONTINUATION_CLASS,
+
CORINFO_HELP_COUNT,
};
@@ -668,6 +674,7 @@ enum CorInfoCallConv
CORINFO_CALLCONV_HASTHIS = 0x20,
CORINFO_CALLCONV_EXPLICITTHIS=0x40,
CORINFO_CALLCONV_PARAMTYPE = 0x80, // Passed last. Same as CORINFO_GENERICS_CTXT_FROM_PARAMTYPEARG
+ CORINFO_CALLCONV_ASYNCCALL = 0x100, // Is this a call to an async function?
};
// Represents the calling conventions supported with the extensible calling convention syntax
@@ -715,6 +722,8 @@ enum CorInfoOptions
CORINFO_GENERICS_CTXT_FROM_METHODDESC |
CORINFO_GENERICS_CTXT_FROM_METHODTABLE),
CORINFO_GENERICS_CTXT_KEEP_ALIVE = 0x00000100, // Keep the generics context alive throughout the method even if there is no explicit use, and report its location to the CLR
+ CORINFO_OPT_COPY_STRUCT_INSTANCE = 0x00000200, // Function is a struct instance method that operates on a copy of "this"
+
};
@@ -991,6 +1000,7 @@ struct CORINFO_SIG_INFO
unsigned totalILArgs() { return (numArgs + (hasImplicitThis() ? 1 : 0)); }
bool isVarArg() { return ((getCallConv() == CORINFO_CALLCONV_VARARG) || (getCallConv() == CORINFO_CALLCONV_NATIVEVARARG)); }
bool hasTypeArg() { return ((callConv & CORINFO_CALLCONV_PARAMTYPE) != 0); }
+ bool isAsyncCall() { return ((callConv & CORINFO_CALLCONV_ASYNCCALL) != 0); }
};
struct CORINFO_METHOD_INFO
@@ -1396,6 +1406,9 @@ enum CorInfoTokenKind
// token comes from devirtualizing a method
CORINFO_TOKENKIND_DevirtualizedMethod = 0x800 | CORINFO_TOKENKIND_Method,
+
+ // token comes from runtime async awaiting pattern
+ CORINFO_TOKENKIND_Await = 0x2000 | CORINFO_TOKENKIND_Method,
};
struct CORINFO_RESOLVED_TOKEN
@@ -1688,6 +1701,42 @@ struct CORINFO_EE_INFO
CORINFO_OS osType;
};
+enum CorInfoContinuationFlags
+{
+ // Whether or not the continuation expects the result to be boxed and
+ // placed in the GCData array at index 0. Not set if the callee is void.
+ CORINFO_CONTINUATION_RESULT_IN_GCDATA = 1,
+ // If this bit is set the continuation resumes inside a try block and thus
+ // if an exception is being propagated, needs to be resumed. The exception
+ // should be placed at index 0 or 1 depending on whether the continuation
+ // also expects a result.
+ CORINFO_CONTINUATION_NEEDS_EXCEPTION = 2,
+ // If this bit is set the continuation has an OSR IL offset saved in the
+ // beginning of 'Data'.
+ CORINFO_CONTINUATION_OSR_IL_OFFSET_IN_DATA = 4,
+};
+
+struct CORINFO_ASYNC2_INFO
+{
+ // Class handle for System.Runtime.CompilerServices.Continuation
+ CORINFO_CLASS_HANDLE continuationClsHnd;
+ // 'Next' field
+ CORINFO_FIELD_HANDLE continuationNextFldHnd;
+ // 'Resume' field
+ CORINFO_FIELD_HANDLE continuationResumeFldHnd;
+ // 'State' field
+ CORINFO_FIELD_HANDLE continuationStateFldHnd;
+ // 'Flags' field
+ CORINFO_FIELD_HANDLE continuationFlagsFldHnd;
+ // 'Data' field
+ CORINFO_FIELD_HANDLE continuationDataFldHnd;
+ // 'GCData' field
+ CORINFO_FIELD_HANDLE continuationGCDataFldHnd;
+ // Whether or not the continuation needs to be alloated through the
+ // helper that also takes a method handle
+ bool continuationsNeedMethodHandle;
+};
+
// Flags passed from JIT to runtime.
enum CORINFO_GET_TAILCALL_HELPERS_FLAGS
{
@@ -2959,6 +3008,10 @@ class ICorStaticInfo
CORINFO_EE_INFO *pEEInfoOut
) = 0;
+ virtual void getAsync2Info(
+ CORINFO_ASYNC2_INFO* pAsync2InfoOut
+ ) = 0;
+
/*********************************************************************************/
//
// Diagnostic methods
@@ -3294,6 +3347,8 @@ class ICorDynamicInfo : public ICorStaticInfo
CORINFO_TAILCALL_HELPERS* pResult
) = 0;
+ virtual CORINFO_METHOD_HANDLE getAsyncResumptionStub() = 0;
+
// Optionally, convert calli to regular method call. This is for PInvoke argument marshalling.
virtual bool convertPInvokeCalliToCall(
CORINFO_RESOLVED_TOKEN * pResolvedToken,
diff --git a/src/coreclr/inc/corjitflags.h b/src/coreclr/inc/corjitflags.h
index 920739ff0496c0..9e57c86e4544f8 100644
--- a/src/coreclr/inc/corjitflags.h
+++ b/src/coreclr/inc/corjitflags.h
@@ -64,6 +64,7 @@ class CORJIT_FLAGS
CORJIT_FLAG_SOFTFP_ABI = 30, // Enable armel calling convention
#endif
+ CORJIT_FLAG_RUNTIMEASYNCFUNCTION = 31, // Generate Code for use as an async2 function
};
CORJIT_FLAGS()
diff --git a/src/coreclr/inc/icorjitinfoimpl_generated.h b/src/coreclr/inc/icorjitinfoimpl_generated.h
index 08b1004d4642d3..aa07ffe68a0e4f 100644
--- a/src/coreclr/inc/icorjitinfoimpl_generated.h
+++ b/src/coreclr/inc/icorjitinfoimpl_generated.h
@@ -498,6 +498,9 @@ bool runWithSPMIErrorTrap(
void getEEInfo(
CORINFO_EE_INFO* pEEInfoOut) override;
+void getAsync2Info(
+ CORINFO_ASYNC2_INFO* pAsync2InfoOut) override;
+
mdMethodDef getMethodDefFromMethod(
CORINFO_METHOD_HANDLE hMethod) override;
@@ -660,6 +663,8 @@ bool getTailCallHelpers(
CORINFO_GET_TAILCALL_HELPERS_FLAGS flags,
CORINFO_TAILCALL_HELPERS* pResult) override;
+CORINFO_METHOD_HANDLE getAsyncResumptionStub() override;
+
bool convertPInvokeCalliToCall(
CORINFO_RESOLVED_TOKEN* pResolvedToken,
bool mustConvert) override;
diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h
index 7d81d32a8deb33..017728558bbdbb 100644
--- a/src/coreclr/inc/jiteeversionguid.h
+++ b/src/coreclr/inc/jiteeversionguid.h
@@ -37,11 +37,11 @@
#include
-constexpr GUID JITEEVersionIdentifier = { /* 7139df75-63b2-4610-9a53-f9a86f474db8 */
- 0x7139df75,
- 0x63b2,
- 0x4610,
- {0x9a, 0x53, 0xf9, 0xa8, 0x6f, 0x47, 0x4d, 0xb8}
+constexpr GUID JITEEVersionIdentifier = { /* 814e1025-d560-45d1-8aeb-8dcb009ba6a7 */
+ 0x814e1025,
+ 0xd560,
+ 0x45d1,
+ {0x8a, 0xeb, 0x8d, 0xcb, 0x00, 0x9b, 0xa6, 0xa7}
};
#endif // JIT_EE_VERSIONING_GUID_H
diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h
index 8543f647c93b52..e4bee7d7a95523 100644
--- a/src/coreclr/inc/jithelpers.h
+++ b/src/coreclr/inc/jithelpers.h
@@ -145,6 +145,7 @@
// Exceptions
DYNAMICJITHELPER(CORINFO_HELP_THROW, IL_Throw, METHOD__NIL)
DYNAMICJITHELPER(CORINFO_HELP_RETHROW, IL_Rethrow, METHOD__NIL)
+ DYNAMICJITHELPER(CORINFO_HELP_THROWEXACT, IL_ThrowExact, METHOD__NIL)
DYNAMICJITHELPER(CORINFO_HELP_USER_BREAKPOINT, NULL, METHOD__DEBUGGER__USERBREAKPOINT)
DYNAMICJITHELPER_NOINDIRECT(CORINFO_HELP_RNGCHKFAIL, NULL, METHOD__THROWHELPERS__THROWINDEXOUTOFRANGEEXCEPTION)
DYNAMICJITHELPER_NOINDIRECT(CORINFO_HELP_OVERFLOW, NULL, METHOD__THROWHELPERS__THROWOVERFLOWEXCEPTION)
@@ -343,6 +344,7 @@
JITHELPER(CORINFO_HELP_PATCHPOINT, JIT_Patchpoint, METHOD__NIL)
JITHELPER(CORINFO_HELP_PARTIAL_COMPILATION_PATCHPOINT, JIT_PartialCompilationPatchpoint, METHOD__NIL)
+ JITHELPER(CORINFO_HELP_RESUME_OSR, JIT_ResumeOSR, METHOD__NIL)
JITHELPER(CORINFO_HELP_CLASSPROFILE32, JIT_ClassProfile32, METHOD__NIL)
JITHELPER(CORINFO_HELP_CLASSPROFILE64, JIT_ClassProfile64, METHOD__NIL)
@@ -367,6 +369,10 @@
JITHELPER(CORINFO_HELP_DISPATCH_INDIRECT_CALL, NULL, METHOD__NIL)
#endif
+ DYNAMICJITHELPER(CORINFO_HELP_ALLOC_CONTINUATION, NULL, METHOD__RUNTIME_HELPERS__ALLOC_CONTINUATION)
+ DYNAMICJITHELPER(CORINFO_HELP_ALLOC_CONTINUATION_METHOD, NULL, METHOD__RUNTIME_HELPERS__ALLOC_CONTINUATION_METHOD)
+ DYNAMICJITHELPER(CORINFO_HELP_ALLOC_CONTINUATION_CLASS, NULL, METHOD__RUNTIME_HELPERS__ALLOC_CONTINUATION_CLASS)
+
#undef JITHELPER
#undef DYNAMICJITHELPER
#undef JITHELPER
diff --git a/src/coreclr/inc/patchpointinfo.h b/src/coreclr/inc/patchpointinfo.h
index 02b9fd89f338ae..bdff46def7ef99 100644
--- a/src/coreclr/inc/patchpointinfo.h
+++ b/src/coreclr/inc/patchpointinfo.h
@@ -38,6 +38,7 @@ struct PatchpointInfo
void Initialize(unsigned localCount, int totalFrameSize)
{
m_calleeSaveRegisters = 0;
+ m_tier0Version = 0;
m_totalFrameSize = totalFrameSize;
m_numberOfLocals = localCount;
m_genericContextArgOffset = -1;
@@ -50,6 +51,7 @@ struct PatchpointInfo
void Copy(const PatchpointInfo* original)
{
m_calleeSaveRegisters = original->m_calleeSaveRegisters;
+ m_tier0Version = original->m_tier0Version;
m_genericContextArgOffset = original->m_genericContextArgOffset;
m_keptAliveThisOffset = original->m_keptAliveThisOffset;
m_securityCookieOffset = original->m_securityCookieOffset;
@@ -173,6 +175,16 @@ struct PatchpointInfo
m_calleeSaveRegisters = registerMask;
}
+ PCODE GetTier0EntryPoint() const
+ {
+ return m_tier0Version;
+ }
+
+ void SetTier0EntryPoint(PCODE ip)
+ {
+ m_tier0Version = ip;
+ }
+
private:
enum
{
@@ -181,6 +193,7 @@ struct PatchpointInfo
};
uint64_t m_calleeSaveRegisters;
+ PCODE m_tier0Version;
unsigned m_numberOfLocals;
int m_totalFrameSize;
int m_genericContextArgOffset;
diff --git a/src/coreclr/jit/CMakeLists.txt b/src/coreclr/jit/CMakeLists.txt
index 288edf637a6dd4..c0939484ff5701 100644
--- a/src/coreclr/jit/CMakeLists.txt
+++ b/src/coreclr/jit/CMakeLists.txt
@@ -88,6 +88,7 @@ endif(CLR_CMAKE_TARGET_WIN32)
set( JIT_SOURCES
abi.cpp
alloc.cpp
+ async.cpp
assertionprop.cpp
bitset.cpp
block.cpp
@@ -285,6 +286,7 @@ set( JIT_HEADERS
_typeinfo.h
abi.h
alloc.h
+ async.h
arraystack.h
bitset.h
layout.h
diff --git a/src/coreclr/jit/ICorJitInfo_names_generated.h b/src/coreclr/jit/ICorJitInfo_names_generated.h
index 94e244c0749bfa..6832da390d29dd 100644
--- a/src/coreclr/jit/ICorJitInfo_names_generated.h
+++ b/src/coreclr/jit/ICorJitInfo_names_generated.h
@@ -124,6 +124,7 @@ DEF_CLR_API(getHFAType)
DEF_CLR_API(runWithErrorTrap)
DEF_CLR_API(runWithSPMIErrorTrap)
DEF_CLR_API(getEEInfo)
+DEF_CLR_API(getAsync2Info)
DEF_CLR_API(getMethodDefFromMethod)
DEF_CLR_API(printMethodName)
DEF_CLR_API(getMethodNameFromMetadata)
@@ -161,6 +162,7 @@ DEF_CLR_API(getFieldThreadLocalStoreID)
DEF_CLR_API(GetDelegateCtor)
DEF_CLR_API(MethodCompileComplete)
DEF_CLR_API(getTailCallHelpers)
+DEF_CLR_API(getAsyncResumptionStub)
DEF_CLR_API(convertPInvokeCalliToCall)
DEF_CLR_API(notifyInstructionSetUsage)
DEF_CLR_API(updateEntryPointForTailCall)
diff --git a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp
index 9c7e6c1099826d..55447c5016d707 100644
--- a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp
+++ b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp
@@ -1181,6 +1181,14 @@ void WrapICorJitInfo::getEEInfo(
API_LEAVE(getEEInfo);
}
+void WrapICorJitInfo::getAsync2Info(
+ CORINFO_ASYNC2_INFO* pAsync2InfoOut)
+{
+ API_ENTER(getAsync2Info);
+ wrapHnd->getAsync2Info(pAsync2InfoOut);
+ API_LEAVE(getAsync2Info);
+}
+
mdMethodDef WrapICorJitInfo::getMethodDefFromMethod(
CORINFO_METHOD_HANDLE hMethod)
{
@@ -1555,6 +1563,14 @@ bool WrapICorJitInfo::getTailCallHelpers(
return temp;
}
+CORINFO_METHOD_HANDLE WrapICorJitInfo::getAsyncResumptionStub()
+{
+ API_ENTER(getAsyncResumptionStub);
+ CORINFO_METHOD_HANDLE temp = wrapHnd->getAsyncResumptionStub();
+ API_LEAVE(getAsyncResumptionStub);
+ return temp;
+}
+
bool WrapICorJitInfo::convertPInvokeCalliToCall(
CORINFO_RESOLVED_TOKEN* pResolvedToken,
bool mustConvert)
diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp
new file mode 100644
index 00000000000000..fcbe6d4adf2403
--- /dev/null
+++ b/src/coreclr/jit/async.cpp
@@ -0,0 +1,1942 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+#include "jitpch.h"
+#include "jitstd/algorithm.h"
+#include "async.h"
+
+class AsyncLiveness
+{
+ Compiler* m_comp;
+ bool m_hasLiveness;
+ TreeLifeUpdater m_updater;
+ unsigned m_numVars;
+
+public:
+ AsyncLiveness(Compiler* comp, bool hasLiveness)
+ : m_comp(comp)
+ , m_hasLiveness(hasLiveness)
+ , m_updater(comp)
+ , m_numVars(comp->lvaCount)
+ {
+ }
+
+ void StartBlock(BasicBlock* block);
+ void Update(GenTree* node);
+ bool IsLive(unsigned lclNum);
+ void GetLiveLocals(jitstd::vector& liveLocals, unsigned fullyDefinedRetBufLcl);
+
+private:
+ bool IsLocalCaptureUnnecessary(unsigned lclNum);
+};
+
+//------------------------------------------------------------------------
+// AsyncLiveness::StartBlock:
+// Indicate that we are now starting a new block, and do relevant liveness
+// updates for it.
+//
+// Parameters:
+// block - The block that we are starting.
+//
+void AsyncLiveness::StartBlock(BasicBlock* block)
+{
+ if (!m_hasLiveness)
+ return;
+
+ VarSetOps::Assign(m_comp, m_comp->compCurLife, block->bbLiveIn);
+}
+
+//------------------------------------------------------------------------
+// AsyncLiveness::Update:
+// Update liveness to be consistent with the specified node having been
+// executed.
+//
+// Parameters:
+// node - The node.
+//
+void AsyncLiveness::Update(GenTree* node)
+{
+ if (!m_hasLiveness)
+ return;
+
+ m_updater.UpdateLife(node);
+}
+
+//------------------------------------------------------------------------
+// AsyncLiveness::IsLocalCaptureUnnecessary:
+// Check if capturing a specified local can be skipped.
+//
+// Parameters:
+// lclNum - The local
+//
+// Returns:
+// True if the local should not be captured. Even without liveness
+//
+bool AsyncLiveness::IsLocalCaptureUnnecessary(unsigned lclNum)
+{
+#if FEATURE_FIXED_OUT_ARGS
+ if (lclNum == m_comp->lvaOutgoingArgSpaceVar)
+ {
+ return true;
+ }
+#endif
+
+ if (lclNum == m_comp->info.compRetBuffArg)
+ {
+ return true;
+ }
+
+ if (lclNum == m_comp->lvaGSSecurityCookie)
+ {
+ // Initialized in prolog
+ return true;
+ }
+
+ if (lclNum == m_comp->lvaPSPSym)
+ {
+ // Initialized in prolog
+ return true;
+ }
+
+ if (lclNum == m_comp->info.compLvFrameListRoot)
+ {
+ return true;
+ }
+
+ if (lclNum == m_comp->lvaInlinedPInvokeFrameVar)
+ {
+ return true;
+ }
+
+#ifdef FEATURE_EH_WINDOWS_X86
+ if (lclNum == m_comp->lvaShadowSPslotsVar)
+ {
+ // Only expected to be live in handlers
+ return true;
+ }
+#endif
+
+ if (lclNum == m_comp->lvaRetAddrVar)
+ {
+ return true;
+ }
+
+ if (lclNum == m_comp->lvaAsyncContinuationArg)
+ {
+ return true;
+ }
+
+ return false;
+}
+
+//------------------------------------------------------------------------
+// AsyncLiveness::IsLive:
+// Check if the specified local is live at this point and should be captured.
+//
+// Parameters:
+// lclNum - The local
+//
+// Returns:
+// True if the local is live and capturing it is necessary.
+//
+bool AsyncLiveness::IsLive(unsigned lclNum)
+{
+ if (IsLocalCaptureUnnecessary(lclNum))
+ {
+ return false;
+ }
+
+ LclVarDsc* dsc = m_comp->lvaGetDesc(lclNum);
+
+ if ((dsc->TypeGet() == TYP_BYREF) || ((dsc->TypeGet() == TYP_STRUCT) && dsc->GetLayout()->HasGCByRef()))
+ {
+ // Even if these are address exposed we expect them to be dead at
+ // suspension points. TODO: It would be good to somehow verify these
+ // aren't obviously live, if the JIT creates live ranges that span a
+ // suspension point then this makes it quite hard to diagnose that.
+ return false;
+ }
+
+ if (!m_hasLiveness)
+ {
+ return true;
+ }
+
+ if (dsc->lvRefCnt(RCS_NORMAL) == 0)
+ {
+ return false;
+ }
+
+ Compiler::lvaPromotionType promoType = m_comp->lvaGetPromotionType(dsc);
+ if (promoType == Compiler::PROMOTION_TYPE_INDEPENDENT)
+ {
+ // Independently promoted structs are handled only through their
+ // fields.
+ return false;
+ }
+
+ if (promoType == Compiler::PROMOTION_TYPE_DEPENDENT)
+ {
+ // Dependently promoted structs are handled only through the base
+ // struct local.
+ //
+ // A dependently promoted struct is live if any of its fields are live.
+
+ for (unsigned i = 0; i < dsc->lvFieldCnt; i++)
+ {
+ LclVarDsc* fieldDsc = m_comp->lvaGetDesc(dsc->lvFieldLclStart + i);
+ if (!fieldDsc->lvTracked || VarSetOps::IsMember(m_comp, m_comp->compCurLife, fieldDsc->lvVarIndex))
+ {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ if (dsc->lvIsStructField && (m_comp->lvaGetParentPromotionType(dsc) == Compiler::PROMOTION_TYPE_DEPENDENT))
+ {
+ return false;
+ }
+
+ return !dsc->lvTracked || VarSetOps::IsMember(m_comp, m_comp->compCurLife, dsc->lvVarIndex);
+}
+
+//------------------------------------------------------------------------
+// AsyncLiveness::GetLiveLocals:
+// Get live locals that should be captured at this point.
+//
+// Parameters:
+// liveLocals - Vector to add live local information into
+// fullyDefinedRetBufLcl - Local to skip even if live
+//
+void AsyncLiveness::GetLiveLocals(jitstd::vector& liveLocals, unsigned fullyDefinedRetBufLcl)
+{
+ for (unsigned lclNum = 0; lclNum < m_numVars; lclNum++)
+ {
+ if ((lclNum != fullyDefinedRetBufLcl) && IsLive(lclNum))
+ {
+ liveLocals.push_back(LiveLocalInfo(lclNum));
+ }
+ }
+}
+
+//------------------------------------------------------------------------
+// TransformAsync: Run async transformation.
+//
+// Returns:
+// Suitable phase status.
+//
+PhaseStatus Compiler::TransformAsync()
+{
+ assert(compIsAsync());
+
+ AsyncTransformation transformation(this);
+ return transformation.Run();
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::Run:
+// Run the transformation over all the IR.
+//
+// Returns:
+// Suitable phase status.
+//
+PhaseStatus AsyncTransformation::Run()
+{
+ ArrayStack worklist(m_comp->getAllocator(CMK_Async));
+
+ // First find all basic blocks with awaits in them. We'll have to track
+ // liveness in these basic blocks, so it does not help to record the calls
+ // ahead of time.
+ for (BasicBlock* block : m_comp->Blocks())
+ {
+ for (GenTree* tree : LIR::AsRange(block))
+ {
+ if (tree->IsCall() && tree->AsCall()->IsAsync() && !tree->AsCall()->IsTailCall())
+ {
+ JITDUMP(FMT_BB " contains await(s)\n", block->bbNum);
+ worklist.Push(block);
+ break;
+ }
+ }
+ }
+
+ JITDUMP("Found %d blocks with awaits\n", worklist.Height());
+
+ if (worklist.Height() <= 0)
+ {
+ return PhaseStatus::MODIFIED_NOTHING;
+ }
+
+ // Ask the VM to create a resumption stub for this specific version of the
+ // code. It is stored in the continuation as a function pointer, so we need
+ // the fixed entry point here.
+ m_resumeStub = m_comp->info.compCompHnd->getAsyncResumptionStub();
+ m_comp->info.compCompHnd->getFunctionFixedEntryPoint(m_resumeStub, false, &m_resumeStubLookup);
+
+ m_returnedContinuationVar = m_comp->lvaGrabTemp(false DEBUGARG("returned continuation"));
+ m_comp->lvaGetDesc(m_returnedContinuationVar)->lvType = TYP_REF;
+ m_newContinuationVar = m_comp->lvaGrabTemp(false DEBUGARG("new continuation"));
+ m_comp->lvaGetDesc(m_newContinuationVar)->lvType = TYP_REF;
+
+ m_comp->info.compCompHnd->getAsync2Info(&m_async2Info);
+
+#ifdef JIT32_GCENCODER
+ // Due to a hard cap on epilogs we need a shared return here.
+ m_sharedReturnBB = m_comp->fgNewBBafter(BBJ_RETURN, m_comp->fgLastBBInMainFunction(), false);
+ m_sharedReturnBB->bbSetRunRarely();
+ m_sharedReturnBB->clearTryIndex();
+ m_sharedReturnBB->clearHndIndex();
+
+ GenTree* continuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF);
+ GenTree* ret = m_comp->gtNewOperNode(GT_RETURN_SUSPEND, TYP_VOID, continuation);
+ LIR::AsRange(m_sharedReturnBB).InsertAtEnd(continuation, ret);
+
+ JITDUMP("Created shared return BB " FMT_BB "\n", m_sharedReturnBB->bbNum);
+
+ DISPRANGE(LIR::AsRange(m_sharedReturnBB));
+#endif
+
+ // Compute liveness to be used for determining what must be captured on
+ // suspension. In unoptimized codegen we capture everything.
+ if (m_comp->opts.OptimizationEnabled())
+ {
+ if (m_comp->m_dfsTree == nullptr)
+ {
+ m_comp->m_dfsTree = m_comp->fgComputeDfs();
+ }
+
+ m_comp->lvaComputeRefCounts(true, false);
+ m_comp->fgLocalVarLiveness();
+ VarSetOps::AssignNoCopy(m_comp, m_comp->compCurLife, VarSetOps::MakeEmpty(m_comp));
+ }
+
+ AsyncLiveness liveness(m_comp, m_comp->opts.OptimizationEnabled());
+
+ // Now walk the IR for all the blocks that contain async2 calls. Keep track
+ // of liveness and outstanding LIR edges as we go; the LIR edges that cross
+ // async2 calls are additional live variables that must be spilled.
+ jitstd::vector defs(m_comp->getAllocator(CMK_Async));
+
+ for (int i = 0; i < worklist.Height(); i++)
+ {
+ assert(defs.size() == 0);
+
+ BasicBlock* block = worklist.Bottom(i);
+ liveness.StartBlock(block);
+
+ bool any;
+ do
+ {
+ any = false;
+ for (GenTree* tree : LIR::AsRange(block))
+ {
+ // Remove all consumed defs; those are no longer 'live' LIR
+ // edges.
+ tree->VisitOperands([&defs](GenTree* op) {
+ if (op->IsValue())
+ {
+ for (size_t i = defs.size(); i > 0; i--)
+ {
+ if (op == defs[i - 1])
+ {
+ defs[i - 1] = defs[defs.size() - 1];
+ defs.erase(defs.begin() + (defs.size() - 1), defs.end());
+ break;
+ }
+ }
+ }
+
+ return GenTree::VisitResult::Continue;
+ });
+
+ // Update liveness to reflect state after this node.
+ liveness.Update(tree);
+
+ if (tree->IsCall() && tree->AsCall()->IsAsync() && !tree->AsCall()->IsTailCall())
+ {
+ // Transform call; continue with the remainder block
+ Transform(block, tree->AsCall(), defs, liveness, &block);
+ defs.clear();
+ any = true;
+ break;
+ }
+
+ // Push a new definition if necessary; this defined value is
+ // now a live LIR edge.
+ if (tree->IsValue() && !tree->IsUnusedValue())
+ {
+ defs.push_back(tree);
+ }
+ }
+ } while (any);
+ }
+
+ // After transforming all async calls we have created resumption blocks;
+ // create the resumption switch.
+ CreateResumptionSwitch();
+
+ m_comp->fgInvalidateDfsTree();
+
+ return PhaseStatus::MODIFIED_EVERYTHING;
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::Transform:
+// Transform a single async2 call in the specified block.
+//
+// Parameters:
+// block - The block containing the async2 call
+// call - The async2 call
+// defs - Current live LIR edges
+// life - Liveness information about live locals
+// remainder - [out] Remainder block after the transformation
+//
+void AsyncTransformation::Transform(
+ BasicBlock* block, GenTreeCall* call, jitstd::vector& defs, AsyncLiveness& life, BasicBlock** remainder)
+{
+#ifdef DEBUG
+ if (m_comp->verbose)
+ {
+ printf("Processing call [%06u] in " FMT_BB "\n", Compiler::dspTreeID(call), block->bbNum);
+ printf(" %zu live LIR edges\n", defs.size());
+
+ if (defs.size() > 0)
+ {
+ const char* sep = " ";
+ for (GenTree* tree : defs)
+ {
+ printf("%s[%06u] (%s)", sep, Compiler::dspTreeID(tree), varTypeName(tree->TypeGet()));
+ sep = ", ";
+ }
+
+ printf("\n");
+ }
+ }
+#endif
+
+ m_liveLocalsScratch.clear();
+ jitstd::vector& liveLocals = m_liveLocalsScratch;
+
+ CreateLiveSetForSuspension(block, call, defs, life, liveLocals);
+
+ ContinuationLayout layout = LayOutContinuation(block, call, liveLocals);
+
+ CallDefinitionInfo callDefInfo = CanonicalizeCallDefinition(block, call, life);
+
+ unsigned stateNum = (unsigned)m_resumptionBBs.size();
+ JITDUMP(" Assigned state %u\n", stateNum);
+
+ BasicBlock* suspendBB = CreateSuspension(block, stateNum, life, liveLocals);
+
+ CreateCheckAndSuspendAfterCall(block, callDefInfo, life, suspendBB, remainder);
+
+ BasicBlock* resumeBB = CreateResumption(block, *remainder, call, callDefInfo, stateNum, layout);
+
+ m_resumptionBBs.push_back(resumeBB);
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::CreateLiveSetForSuspension:
+// Create the set of live state to be captured for suspension, for the
+// specified call.
+//
+// Parameters:
+// block - The block containing the async2 call
+// call - The async2 call
+// defs - Current live LIR edges
+// life - Liveness information about live locals
+// liveLocals - Information about each live local.
+//
+void AsyncTransformation::CreateLiveSetForSuspension(BasicBlock* block,
+ GenTreeCall* call,
+ const jitstd::vector& defs,
+ AsyncLiveness& life,
+ jitstd::vector& liveLocals)
+{
+ unsigned fullyDefinedRetBufLcl = BAD_VAR_NUM;
+ CallArg* retbufArg = call->gtArgs.GetRetBufferArg();
+ if (retbufArg != nullptr)
+ {
+ GenTree* retbuf = retbufArg->GetNode();
+ if (retbuf->IsLclVarAddr())
+ {
+ LclVarDsc* dsc = m_comp->lvaGetDesc(retbuf->AsLclVarCommon());
+ ClassLayout* defLayout = m_comp->typGetObjLayout(call->gtRetClsHnd);
+ if (defLayout->GetSize() == dsc->lvExactSize())
+ {
+ // This call fully defines this retbuf. There is no need to
+ // consider it live across the call since it is going to be
+ // overridden anyway.
+ fullyDefinedRetBufLcl = retbuf->AsLclVarCommon()->GetLclNum();
+ JITDUMP(" V%02u is a fully defined retbuf and will not be considered live\n", fullyDefinedRetBufLcl);
+ }
+ }
+ }
+
+ life.GetLiveLocals(liveLocals, fullyDefinedRetBufLcl);
+ LiftLIREdges(block, defs, liveLocals);
+
+#ifdef DEBUG
+ if (m_comp->verbose)
+ {
+ printf(" %zu live locals\n", liveLocals.size());
+
+ if (liveLocals.size() > 0)
+ {
+ const char* sep = " ";
+ for (LiveLocalInfo& inf : liveLocals)
+ {
+ printf("%sV%02u (%s)", sep, inf.LclNum, varTypeName(m_comp->lvaGetDesc(inf.LclNum)->TypeGet()));
+ sep = ", ";
+ }
+
+ printf("\n");
+ }
+ }
+#endif
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::LiftLIREdges:
+// Create locals capturing outstanding LIR edges and add information
+// indicating that these locals are live.
+//
+// Parameters:
+// block - The block containing the definitions of the LIR edges
+// defs - Current outstanding LIR edges
+// liveLocals - [out] Vector to add new live local information into
+//
+void AsyncTransformation::LiftLIREdges(BasicBlock* block,
+ const jitstd::vector& defs,
+ jitstd::vector& liveLocals)
+{
+ if (defs.size() <= 0)
+ {
+ return;
+ }
+
+ for (GenTree* tree : defs)
+ {
+ // TODO-CQ: Enable this. It currently breaks our recognition of how the
+ // call is stored.
+ // if (tree->OperIs(GT_LCL_VAR))
+ //{
+ // LclVarDsc* dsc = m_comp->lvaGetDesc(tree->AsLclVarCommon());
+ // if (!dsc->IsAddressExposed())
+ // {
+ // // No interference by IR invariants.
+ // LIR::AsRange(block).Remove(tree);
+ // LIR::AsRange(block).InsertAfter(beyond, tree);
+ // continue;
+ // }
+ //}
+
+ LIR::Use use;
+ bool gotUse = LIR::AsRange(block).TryGetUse(tree, &use);
+ assert(gotUse); // Defs list should not contain unused values.
+
+ unsigned newLclNum = use.ReplaceWithLclVar(m_comp);
+ liveLocals.push_back(LiveLocalInfo(newLclNum));
+ GenTree* newUse = use.Def();
+ LIR::AsRange(block).Remove(newUse);
+ LIR::AsRange(block).InsertBefore(use.User(), newUse);
+ }
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::LayOutContinuation:
+// Create the layout of the GC pointer and data arrays in the continuation
+// object.
+//
+// Parameters:
+// block - The block containing the async2 call
+// call - The async2 call
+// liveLocals - [in, out] Information about each live local. Size/alignment
+// information is read and offset/index information is written.
+//
+// Returns:
+// Layout information.
+//
+ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* block,
+ GenTreeCall* call,
+ jitstd::vector& liveLocals)
+{
+ ContinuationLayout layout(liveLocals);
+
+ for (LiveLocalInfo& inf : liveLocals)
+ {
+ LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum);
+
+ if ((dsc->TypeGet() == TYP_STRUCT) || dsc->IsImplicitByRef())
+ {
+ ClassLayout* layout = dsc->GetLayout();
+ assert(!layout->HasGCByRef());
+
+ if (layout->IsCustomLayout())
+ {
+ inf.Alignment = 1;
+ inf.DataSize = layout->GetSize();
+ inf.GCDataCount = layout->GetGCPtrCount();
+ }
+ else
+ {
+ inf.Alignment = m_comp->info.compCompHnd->getClassAlignmentRequirement(layout->GetClassHandle());
+ if ((layout->GetGCPtrCount() * TARGET_POINTER_SIZE) == layout->GetSize())
+ {
+ inf.DataSize = 0;
+ }
+ else
+ {
+ inf.DataSize = layout->GetSize();
+ }
+
+ inf.GCDataCount = layout->GetGCPtrCount();
+ }
+ }
+ else if (dsc->TypeGet() == TYP_REF)
+ {
+ inf.Alignment = TARGET_POINTER_SIZE;
+ inf.DataSize = 0;
+ inf.GCDataCount = 1;
+ }
+ else
+ {
+ assert(dsc->TypeGet() != TYP_BYREF);
+
+ inf.Alignment = genTypeAlignments[dsc->TypeGet()];
+ inf.DataSize = genTypeSize(dsc);
+ inf.GCDataCount = 0;
+ }
+ }
+
+ jitstd::sort(liveLocals.begin(), liveLocals.end(), [](const LiveLocalInfo& lhs, const LiveLocalInfo& rhs) {
+ if (lhs.Alignment == rhs.Alignment)
+ {
+ // Prefer lowest local num first for same alignment.
+ return lhs.LclNum < rhs.LclNum;
+ }
+
+ // Otherwise prefer highest alignment first.
+ return lhs.Alignment > rhs.Alignment;
+ });
+
+ // For OSR, we store the transition IL offset at the beginning of the data
+ // (-1 in the tier0 version):
+ if (m_comp->doesMethodHavePatchpoints() || m_comp->opts.IsOSR())
+ {
+ JITDUMP(" Method %s; keeping an IL offset at the beginning of non-GC data\n",
+ m_comp->doesMethodHavePatchpoints() ? "has patchpoints" : "is an OSR method");
+ layout.DataSize += sizeof(int);
+ }
+
+ if (call->gtReturnType == TYP_STRUCT)
+ {
+ layout.ReturnStructLayout = m_comp->typGetObjLayout(call->gtRetClsHnd);
+ layout.ReturnSize = layout.ReturnStructLayout->GetSize();
+ layout.ReturnInGCData = layout.ReturnStructLayout->HasGCPtr();
+ }
+ else
+ {
+ layout.ReturnSize = genTypeSize(call->gtReturnType);
+ layout.ReturnInGCData = varTypeIsGC(call->gtReturnType);
+ }
+
+ assert((layout.ReturnSize > 0) == (call->gtReturnType != TYP_VOID));
+
+ // The return value is always stored:
+ // 1. At index 0 in GCData if it is a TYP_REF or a struct with GC references
+ // 2. At index 0 in Data, for non OSR methods without GC ref returns
+ // 3. At index 4 in Data for OSR methods without GC ref returns. The
+ // continuation flags indicates this scenario with a flag.
+ if (layout.ReturnInGCData)
+ {
+ layout.GCRefsCount++;
+ }
+ else if (layout.ReturnSize > 0)
+ {
+ layout.ReturnValDataOffset = layout.DataSize;
+ layout.DataSize += layout.ReturnSize;
+ }
+
+#ifdef DEBUG
+ if (layout.ReturnSize > 0)
+ {
+ JITDUMP(" Will store return of type %s, size %u in",
+ call->gtReturnType == TYP_STRUCT ? layout.ReturnStructLayout->GetClassName()
+ : varTypeName(call->gtReturnType),
+ layout.ReturnSize);
+
+ if (layout.ReturnInGCData)
+ {
+ JITDUMP(" GC data\n");
+ }
+ else
+ {
+ JITDUMP(" non-GC data at offset %u\n", layout.ReturnValDataOffset);
+ }
+ }
+#endif
+
+ if (block->hasTryIndex())
+ {
+ layout.ExceptionGCDataIndex = layout.GCRefsCount++;
+ JITDUMP(" " FMT_BB " is in try region %u; exception will be at GC@+%02u in GC data\n", block->bbNum,
+ block->getTryIndex(), layout.ExceptionGCDataIndex);
+ }
+
+ for (LiveLocalInfo& inf : liveLocals)
+ {
+ layout.DataSize = roundUp(layout.DataSize, inf.Alignment);
+
+ inf.DataOffset = layout.DataSize;
+ inf.GCDataIndex = layout.GCRefsCount;
+
+ layout.DataSize += inf.DataSize;
+ layout.GCRefsCount += inf.GCDataCount;
+ }
+
+#ifdef DEBUG
+ if (m_comp->verbose)
+ {
+ printf(" Continuation layout (%u bytes, %u GC pointers):\n", layout.DataSize, layout.GCRefsCount);
+ for (LiveLocalInfo& inf : liveLocals)
+ {
+ printf(" +%03u (GC@+%02u) V%02u: %u bytes, %u GC pointers\n", inf.DataOffset, inf.GCDataIndex,
+ inf.LclNum, inf.DataSize, inf.GCDataCount);
+ }
+ }
+#endif
+
+ return layout;
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::CanonicalizeCallDefinition:
+// Put the call definition in a canonical form. This ensures that either the
+// value is defined by a LCL_ADDR retbuffer or by a
+// STORE_LCL_VAR/STORE_LCL_FLD that follows the call node.
+//
+// Parameters:
+// block - The block containing the async2 call
+// call - The async2 call
+// life - Liveness information about live locals
+//
+// Returns:
+// Information about the definition after canonicalization.
+//
+CallDefinitionInfo AsyncTransformation::CanonicalizeCallDefinition(BasicBlock* block,
+ GenTreeCall* call,
+ AsyncLiveness& life)
+{
+ CallDefinitionInfo callDefInfo;
+
+ callDefInfo.InsertAfter = call;
+
+ CallArg* retbufArg = call->gtArgs.GetRetBufferArg();
+
+ if (!call->TypeIs(TYP_VOID) && !call->IsUnusedValue())
+ {
+ assert(retbufArg == nullptr);
+ assert(call->gtNext != nullptr);
+ if (!call->gtNext->OperIsLocalStore() || (call->gtNext->Data() != call))
+ {
+ LIR::Use use;
+ bool gotUse = LIR::AsRange(block).TryGetUse(call, &use);
+ assert(gotUse);
+
+ use.ReplaceWithLclVar(m_comp);
+ }
+ else
+ {
+ // We will split after the store, but we still have to update liveness for it.
+ life.Update(call->gtNext);
+ }
+
+ assert(call->gtNext->OperIsLocalStore() && (call->gtNext->Data() == call));
+ callDefInfo.DefinitionNode = call->gtNext->AsLclVarCommon();
+ callDefInfo.InsertAfter = call->gtNext;
+ }
+
+ if (retbufArg != nullptr)
+ {
+ assert(call->TypeIs(TYP_VOID));
+
+ // For async2 methods we always expect retbufs to point to locals. We
+ // ensure this in impStoreStruct. TODO-CQ: We can handle common "direct
+ // assignment" cases, e.g. obj.StructVal = Call(), by seeing if there
+ // is a base TYP_REF and keeping that live. This would avoid
+ // introducing copies in the importer on the synchronous path.
+ noway_assert(retbufArg->GetNode()->OperIs(GT_LCL_ADDR));
+
+ callDefInfo.DefinitionNode = retbufArg->GetNode()->AsLclVarCommon();
+ }
+
+ return callDefInfo;
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::CreateSuspension:
+// Create the basic block that when branched to suspends execution after the
+// specified async2 call.
+//
+// Parameters:
+// block - The block containing the async2 call
+// stateNum - State number assigned to this suspension point
+// life - Liveness information about live locals
+// layout - Layout information for the continuation object
+//
+// Returns:
+// The new basic block that was created.
+//
+BasicBlock* AsyncTransformation::CreateSuspension(BasicBlock* block,
+ unsigned stateNum,
+ AsyncLiveness& life,
+ const ContinuationLayout& layout)
+{
+ if (m_lastSuspensionBB == nullptr)
+ {
+ m_lastSuspensionBB = m_comp->fgLastBBInMainFunction();
+ }
+
+ BasicBlock* suspendBB = m_comp->fgNewBBafter(BBJ_RETURN, m_lastSuspensionBB, false);
+ suspendBB->clearTryIndex();
+ suspendBB->clearHndIndex();
+ suspendBB->inheritWeightPercentage(block, 0);
+ m_lastSuspensionBB = suspendBB;
+
+ if (m_sharedReturnBB != nullptr)
+ {
+ suspendBB->SetKindAndTargetEdge(BBJ_ALWAYS, m_comp->fgAddRefPred(m_sharedReturnBB, suspendBB));
+ }
+
+ JITDUMP(" Creating suspension " FMT_BB " for state %u\n", suspendBB->bbNum, stateNum);
+
+ // Allocate continuation
+ GenTree* returnedContinuation = m_comp->gtNewLclvNode(m_returnedContinuationVar, TYP_REF);
+
+ GenTreeCall* allocContinuation =
+ CreateAllocContinuationCall(life, returnedContinuation, layout.GCRefsCount, layout.DataSize);
+
+ m_comp->compCurBB = suspendBB;
+ m_comp->fgMorphTree(allocContinuation);
+
+ LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, allocContinuation));
+
+ GenTree* storeNewContinuation = m_comp->gtNewStoreLclVarNode(m_newContinuationVar, allocContinuation);
+ LIR::AsRange(suspendBB).InsertAtEnd(storeNewContinuation);
+
+ // Fill in 'Resume'
+ GenTree* newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF);
+ unsigned resumeOffset = m_comp->info.compCompHnd->getFieldOffset(m_async2Info.continuationResumeFldHnd);
+ GenTree* resumeStubAddr = CreateResumptionStubAddrTree();
+ GenTree* storeResume = StoreAtOffset(newContinuation, resumeOffset, resumeStubAddr);
+ LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, storeResume));
+
+ // Fill in 'state'
+ newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF);
+ unsigned stateOffset = m_comp->info.compCompHnd->getFieldOffset(m_async2Info.continuationStateFldHnd);
+ GenTree* stateNumNode = m_comp->gtNewIconNode((ssize_t)stateNum, TYP_INT);
+ GenTree* storeState = StoreAtOffset(newContinuation, stateOffset, stateNumNode);
+ LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, storeState));
+
+ // Fill in 'flags'
+ unsigned continuationFlags = 0;
+ if (layout.ReturnInGCData)
+ continuationFlags |= CORINFO_CONTINUATION_RESULT_IN_GCDATA;
+ if (block->hasTryIndex())
+ continuationFlags |= CORINFO_CONTINUATION_NEEDS_EXCEPTION;
+ if (m_comp->doesMethodHavePatchpoints() || m_comp->opts.IsOSR())
+ continuationFlags |= CORINFO_CONTINUATION_OSR_IL_OFFSET_IN_DATA;
+
+ newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF);
+ unsigned flagsOffset = m_comp->info.compCompHnd->getFieldOffset(m_async2Info.continuationFlagsFldHnd);
+ GenTree* flagsNode = m_comp->gtNewIconNode((ssize_t)continuationFlags, TYP_INT);
+ GenTree* storeFlags = StoreAtOffset(newContinuation, flagsOffset, flagsNode);
+ LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, storeFlags));
+
+ if (layout.GCRefsCount > 0)
+ {
+ FillInGCPointersOnSuspension(layout.Locals, suspendBB);
+ }
+
+ if (layout.DataSize > 0)
+ {
+ FillInDataOnSuspension(layout.Locals, suspendBB);
+ }
+
+ if (suspendBB->KindIs(BBJ_RETURN))
+ {
+ newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF);
+ GenTree* ret = m_comp->gtNewOperNode(GT_RETURN_SUSPEND, TYP_VOID, newContinuation);
+ LIR::AsRange(suspendBB).InsertAtEnd(newContinuation, ret);
+ }
+
+ return suspendBB;
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::CreateAllocContinuationCall:
+// Create a call to the JIT helper that allocates a continuation.
+//
+// Parameters:
+// life - Liveness information about live locals
+// prevContinuation - IR node that has the value of the previous continuation object
+// gcRefsCount - Number of GC refs to allocate in the continuation object
+// dataSize - Number of bytes to allocate in the continuation object
+//
+// Returns:
+// IR node representing the allocation.
+//
+GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(AsyncLiveness& life,
+ GenTree* prevContinuation,
+ unsigned gcRefsCount,
+ unsigned dataSize)
+{
+ GenTree* gcRefsCountNode = m_comp->gtNewIconNode((ssize_t)gcRefsCount, TYP_I_IMPL);
+ GenTree* dataSizeNode = m_comp->gtNewIconNode((ssize_t)dataSize, TYP_I_IMPL);
+ // If VM requests that we report the method handle, or if we have a shared generic context method handle
+ // that is live here, then we need to call a different helper to keep the loader alive.
+ GenTree* methodHandleArg = nullptr;
+ GenTree* classHandleArg = nullptr;
+ if (((m_comp->info.compMethodInfo->options & CORINFO_GENERICS_CTXT_FROM_METHODDESC) != 0) &&
+ life.IsLive(m_comp->info.compTypeCtxtArg))
+ {
+ methodHandleArg = m_comp->gtNewLclvNode(m_comp->info.compTypeCtxtArg, TYP_I_IMPL);
+ }
+ else if (((m_comp->info.compMethodInfo->options & CORINFO_GENERICS_CTXT_FROM_METHODTABLE) != 0) &&
+ life.IsLive(m_comp->info.compTypeCtxtArg))
+ {
+ classHandleArg = m_comp->gtNewLclvNode(m_comp->info.compTypeCtxtArg, TYP_I_IMPL);
+ }
+ else if (m_async2Info.continuationsNeedMethodHandle)
+ {
+ methodHandleArg = m_comp->gtNewIconEmbMethHndNode(m_comp->info.compMethodHnd);
+ }
+
+ if (methodHandleArg != nullptr)
+ {
+ return m_comp->gtNewHelperCallNode(CORINFO_HELP_ALLOC_CONTINUATION_METHOD, TYP_REF, prevContinuation,
+ gcRefsCountNode, dataSizeNode, methodHandleArg);
+ }
+
+ if (classHandleArg != nullptr)
+ {
+ return m_comp->gtNewHelperCallNode(CORINFO_HELP_ALLOC_CONTINUATION_CLASS, TYP_REF, prevContinuation,
+ gcRefsCountNode, dataSizeNode, classHandleArg);
+ }
+
+ return m_comp->gtNewHelperCallNode(CORINFO_HELP_ALLOC_CONTINUATION, TYP_REF, prevContinuation, gcRefsCountNode,
+ dataSizeNode);
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::FillInGCPointersOnSuspension:
+// Create IR that fills the GC pointers of the continuation object.
+//
+// Parameters:
+// liveLocals - Information about each live local.
+// suspendBB - Basic block to add IR to.
+//
+void AsyncTransformation::FillInGCPointersOnSuspension(const jitstd::vector& liveLocals,
+ BasicBlock* suspendBB)
+{
+ unsigned objectArrLclNum = GetGCDataArrayVar();
+
+ GenTree* newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF);
+ unsigned gcDataOffset = m_comp->info.compCompHnd->getFieldOffset(m_async2Info.continuationGCDataFldHnd);
+ GenTree* gcDataInd = LoadFromOffset(newContinuation, gcDataOffset, TYP_REF);
+ GenTree* storeAllocedObjectArr = m_comp->gtNewStoreLclVarNode(objectArrLclNum, gcDataInd);
+ LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, storeAllocedObjectArr));
+
+ for (const LiveLocalInfo& inf : liveLocals)
+ {
+ if (inf.GCDataCount <= 0)
+ {
+ continue;
+ }
+
+ LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum);
+ if (dsc->TypeGet() == TYP_REF)
+ {
+ GenTree* value = m_comp->gtNewLclvNode(inf.LclNum, TYP_REF);
+ GenTree* objectArr = m_comp->gtNewLclvNode(objectArrLclNum, TYP_REF);
+ GenTree* store =
+ StoreAtOffset(objectArr, OFFSETOF__CORINFO_Array__data + (inf.GCDataIndex * TARGET_POINTER_SIZE),
+ value);
+ LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, store));
+ }
+ else
+ {
+ assert((dsc->TypeGet() == TYP_STRUCT) || dsc->IsImplicitByRef());
+ ClassLayout* layout = dsc->GetLayout();
+ unsigned numSlots = layout->GetSlotCount();
+ unsigned gcRefIndex = 0;
+ for (unsigned i = 0; i < numSlots; i++)
+ {
+ var_types gcPtrType = layout->GetGCPtrType(i);
+ assert((gcPtrType == TYP_I_IMPL) || (gcPtrType == TYP_REF));
+ if (gcPtrType != TYP_REF)
+ {
+ continue;
+ }
+
+ GenTree* value;
+ if (dsc->IsImplicitByRef())
+ {
+ GenTree* baseAddr = m_comp->gtNewLclvNode(inf.LclNum, dsc->TypeGet());
+ value = LoadFromOffset(baseAddr, i * TARGET_POINTER_SIZE, TYP_REF);
+ }
+ else
+ {
+ value = m_comp->gtNewLclFldNode(inf.LclNum, TYP_REF, i * TARGET_POINTER_SIZE);
+ }
+
+ GenTree* objectArr = m_comp->gtNewLclvNode(objectArrLclNum, TYP_REF);
+ unsigned offset =
+ OFFSETOF__CORINFO_Array__data + ((inf.GCDataIndex + gcRefIndex) * TARGET_POINTER_SIZE);
+ GenTree* store = StoreAtOffset(objectArr, offset, value);
+ LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, store));
+
+ gcRefIndex++;
+
+ if (inf.DataSize > 0)
+ {
+ // Null out the GC field in preparation of storing the rest.
+ GenTree* null = m_comp->gtNewNull();
+
+ if (dsc->IsImplicitByRef())
+ {
+ GenTree* baseAddr = m_comp->gtNewLclvNode(inf.LclNum, dsc->TypeGet());
+ store = StoreAtOffset(baseAddr, i * TARGET_POINTER_SIZE, null);
+ }
+ else
+ {
+ store = m_comp->gtNewStoreLclFldNode(inf.LclNum, TYP_REF, i * TARGET_POINTER_SIZE, null);
+ }
+
+ LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, store));
+ }
+ }
+
+ m_comp->lvaSetVarDoNotEnregister(inf.LclNum DEBUGARG(DoNotEnregisterReason::LocalField));
+ }
+ }
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::FillInDataOnSuspension:
+// Create IR that fills the data array of the continuation object.
+//
+// Parameters:
+// liveLocals - Information about each live local.
+// suspendBB - Basic block to add IR to.
+//
+void AsyncTransformation::FillInDataOnSuspension(const jitstd::vector& liveLocals, BasicBlock* suspendBB)
+{
+ unsigned byteArrLclNum = GetDataArrayVar();
+
+ GenTree* newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF);
+ unsigned dataOffset = m_comp->info.compCompHnd->getFieldOffset(m_async2Info.continuationDataFldHnd);
+ GenTree* dataInd = LoadFromOffset(newContinuation, dataOffset, TYP_REF);
+ GenTree* storeAllocedByteArr = m_comp->gtNewStoreLclVarNode(byteArrLclNum, dataInd);
+ LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, storeAllocedByteArr));
+
+ if (m_comp->doesMethodHavePatchpoints() || m_comp->opts.IsOSR())
+ {
+ GenTree* ilOffsetToStore;
+ if (m_comp->doesMethodHavePatchpoints())
+ ilOffsetToStore = m_comp->gtNewIconNode(-1);
+ else
+ ilOffsetToStore = m_comp->gtNewIconNode((int)m_comp->info.compILEntry);
+
+ GenTree* byteArr = m_comp->gtNewLclvNode(byteArrLclNum, TYP_REF);
+ unsigned offset = OFFSETOF__CORINFO_Array__data;
+ GenTree* storePatchpointOffset = StoreAtOffset(byteArr, offset, ilOffsetToStore);
+ LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, storePatchpointOffset));
+ }
+
+ // Fill in data
+ for (const LiveLocalInfo& inf : liveLocals)
+ {
+ if (inf.DataSize <= 0)
+ {
+ continue;
+ }
+
+ LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum);
+
+ GenTree* byteArr = m_comp->gtNewLclvNode(byteArrLclNum, TYP_REF);
+ unsigned offset = OFFSETOF__CORINFO_Array__data + inf.DataOffset;
+
+ GenTree* value;
+ if (dsc->IsImplicitByRef())
+ {
+ GenTree* baseAddr = m_comp->gtNewLclvNode(inf.LclNum, dsc->TypeGet());
+ value = m_comp->gtNewBlkIndir(dsc->GetLayout(), baseAddr, GTF_IND_NONFAULTING);
+ }
+ else
+ {
+ value = m_comp->gtNewLclvNode(inf.LclNum, genActualType(dsc->TypeGet()));
+ }
+
+ GenTree* store;
+ if ((dsc->TypeGet() == TYP_STRUCT) || dsc->IsImplicitByRef())
+ {
+ GenTree* cns = m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL);
+ GenTree* addr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, byteArr, cns);
+ // This is to heap, but all GC refs are nulled out already, so we can skip the write barrier.
+ // TODO-CQ: Backend does not care about GTF_IND_TGT_NOT_HEAP for STORE_BLK.
+ store =
+ m_comp->gtNewStoreBlkNode(dsc->GetLayout(), addr, value, GTF_IND_NONFAULTING | GTF_IND_TGT_NOT_HEAP);
+ }
+ else
+ {
+ store = StoreAtOffset(byteArr, offset, value);
+ }
+
+ LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, store));
+ }
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::CreateCheckAndSuspendAfterCall:
+// Split the block containing the specified async2 call, and create the IR
+// that checks whether suspension should be done after an async call.
+//
+// Parameters:
+// block - The block containing the async2 call
+// callDefInfo - Information about the async2 call's definition
+// life - Liveness information about live locals
+// suspendBB - Basic block to add IR to
+// remainder - [out] The remainder block containing the IR that was after the async2 call.
+//
+void AsyncTransformation::CreateCheckAndSuspendAfterCall(BasicBlock* block,
+ const CallDefinitionInfo& callDefInfo,
+ AsyncLiveness& life,
+ BasicBlock* suspendBB,
+ BasicBlock** remainder)
+{
+ GenTree* continuationArg = new (m_comp, GT_ASYNC_CONTINUATION) GenTree(GT_ASYNC_CONTINUATION, TYP_REF);
+ continuationArg->SetHasOrderingSideEffect();
+
+ GenTree* storeContinuation = m_comp->gtNewStoreLclVarNode(m_returnedContinuationVar, continuationArg);
+ LIR::AsRange(block).InsertAfter(callDefInfo.InsertAfter, continuationArg, storeContinuation);
+
+ GenTree* null = m_comp->gtNewNull();
+ GenTree* returnedContinuation = m_comp->gtNewLclvNode(m_returnedContinuationVar, TYP_REF);
+ GenTree* neNull = m_comp->gtNewOperNode(GT_NE, TYP_INT, returnedContinuation, null);
+ GenTree* jtrue = m_comp->gtNewOperNode(GT_JTRUE, TYP_VOID, neNull);
+
+ LIR::AsRange(block).InsertAfter(storeContinuation, null, returnedContinuation, neNull, jtrue);
+ *remainder = m_comp->fgSplitBlockAfterNode(block, jtrue);
+ JITDUMP(" Remainder is " FMT_BB "\n", (*remainder)->bbNum);
+
+ FlowEdge* retBBEdge = m_comp->fgAddRefPred(suspendBB, block);
+ block->SetCond(retBBEdge, block->GetTargetEdge());
+
+ block->GetTrueEdge()->setLikelihood(0);
+ block->GetFalseEdge()->setLikelihood(1);
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::CreateResumption:
+// Create the basic block that when branched to resumes execution on entry to
+// the function.
+//
+// Parameters:
+// block - The block containing the async2 call
+// remainder - The block that contains the IR after the (split) async2 call
+// call - The async2 call
+// callDefInfo - Information about the async2 call's definition
+// stateNum - State number assigned to this suspension point
+// layout - Layout information for the continuation object
+//
+// Returns:
+// The new basic block that was created.
+//
+BasicBlock* AsyncTransformation::CreateResumption(BasicBlock* block,
+ BasicBlock* remainder,
+ GenTreeCall* call,
+ const CallDefinitionInfo& callDefInfo,
+ unsigned stateNum,
+ const ContinuationLayout& layout)
+{
+ if (m_lastResumptionBB == nullptr)
+ {
+ m_lastResumptionBB = m_comp->fgLastBBInMainFunction();
+ }
+
+ BasicBlock* resumeBB = m_comp->fgNewBBafter(BBJ_ALWAYS, m_lastResumptionBB, true);
+ FlowEdge* remainderEdge = m_comp->fgAddRefPred(remainder, resumeBB);
+
+ // It does not really make sense to inherit from the target, but given this
+ // is always 0% this just propagates the profile weight flag + sets
+ // BBF_RUN_RARELY.
+ resumeBB->inheritWeightPercentage(remainder, 0);
+ resumeBB->SetTargetEdge(remainderEdge);
+ resumeBB->clearTryIndex();
+ resumeBB->clearHndIndex();
+ resumeBB->SetFlags(BBF_ASYNC_RESUMPTION);
+ m_lastResumptionBB = resumeBB;
+
+ JITDUMP(" Creating resumption " FMT_BB " for state %u\n", resumeBB->bbNum, stateNum);
+
+ unsigned resumeByteArrLclNum = BAD_VAR_NUM;
+ if (layout.DataSize > 0)
+ {
+ resumeByteArrLclNum = GetDataArrayVar();
+
+ GenTree* newContinuation = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF);
+ unsigned dataOffset = m_comp->info.compCompHnd->getFieldOffset(m_async2Info.continuationDataFldHnd);
+ GenTree* dataInd = LoadFromOffset(newContinuation, dataOffset, TYP_REF);
+ GenTree* storeAllocedByteArr = m_comp->gtNewStoreLclVarNode(resumeByteArrLclNum, dataInd);
+
+ LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, storeAllocedByteArr));
+
+ RestoreFromDataOnResumption(resumeByteArrLclNum, layout.Locals, resumeBB);
+ }
+
+ unsigned resumeObjectArrLclNum = BAD_VAR_NUM;
+ BasicBlock* storeResultBB = resumeBB;
+
+ if (layout.GCRefsCount > 0)
+ {
+ resumeObjectArrLclNum = GetGCDataArrayVar();
+
+ GenTree* newContinuation = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF);
+ unsigned gcDataOffset = m_comp->info.compCompHnd->getFieldOffset(m_async2Info.continuationGCDataFldHnd);
+ GenTree* gcDataInd = LoadFromOffset(newContinuation, gcDataOffset, TYP_REF);
+ GenTree* storeAllocedObjectArr = m_comp->gtNewStoreLclVarNode(resumeObjectArrLclNum, gcDataInd);
+ LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, storeAllocedObjectArr));
+
+ RestoreFromGCPointersOnResumption(resumeObjectArrLclNum, layout.Locals, resumeBB);
+
+ if (layout.ExceptionGCDataIndex != UINT_MAX)
+ {
+ storeResultBB = RethrowExceptionOnResumption(block, remainder, resumeObjectArrLclNum, layout, resumeBB);
+ }
+ }
+
+ // Copy call return value.
+ if (layout.ReturnSize > 0)
+ {
+ CopyReturnValueOnResumption(call, callDefInfo, resumeByteArrLclNum, resumeObjectArrLclNum, layout,
+ storeResultBB);
+ }
+
+ return resumeBB;
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::RestoreFromDataOnResumption:
+// Create IR that restores locals from the data array of the continuation
+// object.
+//
+// Parameters:
+// resumeByteArrLclNum - Local that has the continuation object's data array
+// liveLocals - Information about each live local.
+// resumeBB - Basic block to append IR to
+//
+void AsyncTransformation::RestoreFromDataOnResumption(unsigned resumeByteArrLclNum,
+ const jitstd::vector& liveLocals,
+ BasicBlock* resumeBB)
+{
+ // Copy data
+ for (const LiveLocalInfo& inf : liveLocals)
+ {
+ if (inf.DataSize <= 0)
+ {
+ continue;
+ }
+
+ LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum);
+
+ GenTree* byteArr = m_comp->gtNewLclvNode(resumeByteArrLclNum, TYP_REF);
+ unsigned offset = OFFSETOF__CORINFO_Array__data + inf.DataOffset;
+ GenTree* cns = m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL);
+ GenTree* addr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, byteArr, cns);
+
+ GenTree* value;
+ if ((dsc->TypeGet() == TYP_STRUCT) || dsc->IsImplicitByRef())
+ {
+ value = m_comp->gtNewBlkIndir(dsc->GetLayout(), addr, GTF_IND_NONFAULTING);
+ }
+ else
+ {
+ value = m_comp->gtNewIndir(dsc->TypeGet(), addr, GTF_IND_NONFAULTING);
+ }
+
+ GenTree* store;
+ if (dsc->IsImplicitByRef())
+ {
+ GenTree* baseAddr = m_comp->gtNewLclvNode(inf.LclNum, dsc->TypeGet());
+ // TODO-CQ: Incoming data has no non-null GC refs, so this does not need write barriers.
+ // Backend does not handle GTF_IND_TGT_NOT_HEAP for STORE_BLK.
+ store = m_comp->gtNewStoreBlkNode(dsc->GetLayout(), baseAddr, value,
+ GTF_IND_NONFAULTING | GTF_IND_TGT_NOT_HEAP);
+ }
+ else
+ {
+ store = m_comp->gtNewStoreLclVarNode(inf.LclNum, value);
+ }
+
+ LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, store));
+ }
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::RestoreFromGCPointersOnResumption:
+// Create IR that restores locals from the GC pointers array of the
+// continuation object.
+//
+// Parameters:
+// resumeObjectArrLclNum - Local that has the continuation object's GC pointers array
+// liveLocals - Information about each live local.
+// resumeBB - Basic block to append IR to
+//
+void AsyncTransformation::RestoreFromGCPointersOnResumption(unsigned resumeObjectArrLclNum,
+ const jitstd::vector& liveLocals,
+ BasicBlock* resumeBB)
+{
+ for (const LiveLocalInfo& inf : liveLocals)
+ {
+ if (inf.GCDataCount <= 0)
+ {
+ continue;
+ }
+
+ LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum);
+ if (dsc->TypeGet() == TYP_REF)
+ {
+ GenTree* objectArr = m_comp->gtNewLclvNode(resumeObjectArrLclNum, TYP_REF);
+ unsigned offset = OFFSETOF__CORINFO_Array__data + (inf.GCDataIndex * TARGET_POINTER_SIZE);
+ GenTree* value = LoadFromOffset(objectArr, offset, TYP_REF);
+ GenTree* store = m_comp->gtNewStoreLclVarNode(inf.LclNum, value);
+
+ LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, store));
+ }
+ else
+ {
+ assert((dsc->TypeGet() == TYP_STRUCT) || dsc->IsImplicitByRef());
+ ClassLayout* layout = dsc->GetLayout();
+ unsigned numSlots = layout->GetSlotCount();
+ unsigned gcRefIndex = 0;
+ for (unsigned i = 0; i < numSlots; i++)
+ {
+ var_types gcPtrType = layout->GetGCPtrType(i);
+ assert((gcPtrType == TYP_I_IMPL) || (gcPtrType == TYP_REF));
+ if (gcPtrType != TYP_REF)
+ {
+ continue;
+ }
+
+ GenTree* objectArr = m_comp->gtNewLclvNode(resumeObjectArrLclNum, TYP_REF);
+ unsigned offset =
+ OFFSETOF__CORINFO_Array__data + ((inf.GCDataIndex + gcRefIndex) * TARGET_POINTER_SIZE);
+ GenTree* value = LoadFromOffset(objectArr, offset, TYP_REF);
+ GenTree* store;
+ if (dsc->IsImplicitByRef())
+ {
+ GenTree* baseAddr = m_comp->gtNewLclvNode(inf.LclNum, dsc->TypeGet());
+ store = StoreAtOffset(baseAddr, i * TARGET_POINTER_SIZE, value);
+ // Implicit byref args are never on heap
+ store->gtFlags |= GTF_IND_TGT_NOT_HEAP;
+ }
+ else
+ {
+ store = m_comp->gtNewStoreLclFldNode(inf.LclNum, TYP_REF, i * TARGET_POINTER_SIZE, value);
+ }
+
+ LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, store));
+
+ gcRefIndex++;
+ }
+ }
+ }
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::RethrowExceptionOnResumption:
+// Create IR that checks for an exception and rethrows it at the original
+// suspension point if necessary.
+//
+// Parameters:
+// block - The block containing the async2 call
+// remainder - The block that contains the IR after the (split) async2 call
+// resumeObjectArrLclNum - Local that has the continuation object's GC pointers array
+// layout - Layout information for the continuation object
+// resumeBB - Basic block to append IR to
+//
+// Returns:
+// The new non-exception successor basic block for resumption. This is the
+// basic block where execution will continue if there was no exception to
+// rethrow.
+//
+BasicBlock* AsyncTransformation::RethrowExceptionOnResumption(BasicBlock* block,
+ BasicBlock* remainder,
+ unsigned resumeObjectArrLclNum,
+ const ContinuationLayout& layout,
+ BasicBlock* resumeBB)
+{
+ JITDUMP(" We need to rethrow an exception\n");
+
+ BasicBlock* rethrowExceptionBB =
+ m_comp->fgNewBBinRegion(BBJ_THROW, block, /* runRarely */ true, /* insertAtEnd */ true);
+ JITDUMP(" Created " FMT_BB " to rethrow exception on resumption\n", rethrowExceptionBB->bbNum);
+
+ BasicBlock* storeResultBB = m_comp->fgNewBBafter(BBJ_ALWAYS, resumeBB, true);
+ JITDUMP(" Created " FMT_BB " to store result when resuming with no exception\n", storeResultBB->bbNum);
+
+ FlowEdge* rethrowEdge = m_comp->fgAddRefPred(rethrowExceptionBB, resumeBB);
+ FlowEdge* storeResultEdge = m_comp->fgAddRefPred(storeResultBB, resumeBB);
+
+ assert(resumeBB->KindIs(BBJ_ALWAYS));
+ m_comp->fgRemoveRefPred(resumeBB->GetTargetEdge());
+
+ resumeBB->SetCond(rethrowEdge, storeResultEdge);
+ rethrowEdge->setLikelihood(0);
+ storeResultEdge->setLikelihood(1);
+ rethrowExceptionBB->inheritWeightPercentage(resumeBB, 0);
+ storeResultBB->inheritWeightPercentage(resumeBB, 100);
+ JITDUMP(" Resumption " FMT_BB " becomes BBJ_COND to check for non-null exception\n", resumeBB->bbNum);
+
+ FlowEdge* remainderEdge = m_comp->fgAddRefPred(remainder, storeResultBB);
+ storeResultBB->SetTargetEdge(remainderEdge);
+
+ m_lastResumptionBB = storeResultBB;
+
+ // Check if we have an exception.
+ unsigned exceptionLclNum = GetExceptionVar();
+ GenTree* objectArr = m_comp->gtNewLclvNode(resumeObjectArrLclNum, TYP_REF);
+ unsigned exceptionOffset = OFFSETOF__CORINFO_Array__data + layout.ExceptionGCDataIndex * TARGET_POINTER_SIZE;
+ GenTree* exceptionInd = LoadFromOffset(objectArr, exceptionOffset, TYP_REF);
+ GenTree* storeException = m_comp->gtNewStoreLclVarNode(exceptionLclNum, exceptionInd);
+ LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, storeException));
+
+ GenTree* exception = m_comp->gtNewLclVarNode(exceptionLclNum, TYP_REF);
+ GenTree* null = m_comp->gtNewNull();
+ GenTree* neNull = m_comp->gtNewOperNode(GT_NE, TYP_INT, exception, null);
+ GenTree* jtrue = m_comp->gtNewOperNode(GT_JTRUE, TYP_VOID, neNull);
+ LIR::AsRange(resumeBB).InsertAtEnd(exception, null, neNull, jtrue);
+
+ exception = m_comp->gtNewLclVarNode(exceptionLclNum, TYP_REF);
+ GenTreeCall* rethrowException = m_comp->gtNewHelperCallNode(CORINFO_HELP_THROWEXACT, TYP_VOID, exception);
+
+ m_comp->compCurBB = rethrowExceptionBB;
+ m_comp->fgMorphTree(rethrowException);
+
+ LIR::AsRange(rethrowExceptionBB).InsertAtEnd(LIR::SeqTree(m_comp, rethrowException));
+
+ storeResultBB->SetFlags(BBF_ASYNC_RESUMPTION);
+ JITDUMP(" Added " FMT_BB " to rethrow exception at suspension point\n", rethrowExceptionBB->bbNum);
+
+ return storeResultBB;
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::CopyReturnValueOnResumption:
+// Create IR that copies the return value from the continuation object to the
+// right local.
+//
+// Parameters:
+// call - The async2 call
+// callDefInfo - Information about the async2 call's definition
+// block - The block containing the async2 call
+// resumeByteArrLclNum - Local that has the continuation object's data array
+// resumeObjectArrLclNum - Local that has the continuation object's GC pointers array
+// layout - Layout information for the continuation object
+// storeResultBB - Basic block to append IR to
+//
+void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* call,
+ const CallDefinitionInfo& callDefInfo,
+ unsigned resumeByteArrLclNum,
+ unsigned resumeObjectArrLclNum,
+ const ContinuationLayout& layout,
+ BasicBlock* storeResultBB)
+{
+ GenTree* resultBase;
+ unsigned resultOffset;
+ GenTreeFlags resultIndirFlags = GTF_IND_NONFAULTING;
+ if (layout.ReturnInGCData)
+ {
+ assert(resumeObjectArrLclNum != BAD_VAR_NUM);
+ resultBase = m_comp->gtNewLclvNode(resumeObjectArrLclNum, TYP_REF);
+
+ if (call->gtReturnType == TYP_STRUCT)
+ {
+ // Boxed struct.
+ resultBase = LoadFromOffset(resultBase, OFFSETOF__CORINFO_Array__data, TYP_REF);
+ resultOffset = TARGET_POINTER_SIZE; // Offset of data inside box
+ }
+ else
+ {
+ assert(call->gtReturnType == TYP_REF);
+ resultOffset = OFFSETOF__CORINFO_Array__data;
+ }
+ }
+ else
+ {
+ assert(resumeByteArrLclNum != BAD_VAR_NUM);
+ resultBase = m_comp->gtNewLclvNode(resumeByteArrLclNum, TYP_REF);
+ resultOffset = OFFSETOF__CORINFO_Array__data + layout.ReturnValDataOffset;
+ if (layout.ReturnValDataOffset != 0)
+ resultIndirFlags = GTF_IND_UNALIGNED;
+ }
+
+ assert(callDefInfo.DefinitionNode != nullptr);
+ LclVarDsc* resultLcl = m_comp->lvaGetDesc(callDefInfo.DefinitionNode);
+ assert((resultLcl->TypeGet() == TYP_STRUCT) == (call->gtReturnType == TYP_STRUCT));
+
+ // TODO-TP: We can use liveness to avoid generating a lot of this IR.
+ if (call->gtReturnType == TYP_STRUCT)
+ {
+ if (m_comp->lvaGetPromotionType(resultLcl) != Compiler::PROMOTION_TYPE_INDEPENDENT)
+ {
+ GenTree* resultOffsetNode = m_comp->gtNewIconNode((ssize_t)resultOffset, TYP_I_IMPL);
+ GenTree* resultAddr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, resultBase, resultOffsetNode);
+ GenTree* resultData = m_comp->gtNewBlkIndir(layout.ReturnStructLayout, resultAddr, resultIndirFlags);
+ GenTree* storeResult;
+ if ((callDefInfo.DefinitionNode->GetLclOffs() == 0) &&
+ ClassLayout::AreCompatible(resultLcl->GetLayout(), layout.ReturnStructLayout))
+ {
+ storeResult = m_comp->gtNewStoreLclVarNode(callDefInfo.DefinitionNode->GetLclNum(), resultData);
+ }
+ else
+ {
+ storeResult = m_comp->gtNewStoreLclFldNode(callDefInfo.DefinitionNode->GetLclNum(), TYP_STRUCT,
+ layout.ReturnStructLayout,
+ callDefInfo.DefinitionNode->GetLclOffs(), resultData);
+ }
+
+ LIR::AsRange(storeResultBB).InsertAtEnd(LIR::SeqTree(m_comp, storeResult));
+ }
+ else
+ {
+ assert(!call->gtArgs.HasRetBuffer()); // Locals defined through retbufs are never independently promoted.
+
+ if ((resultLcl->lvFieldCnt > 1) && !resultBase->OperIsLocal())
+ {
+ unsigned resultBaseVar = GetResultBaseVar();
+ GenTree* storeResultBase = m_comp->gtNewStoreLclVarNode(resultBaseVar, resultBase);
+ LIR::AsRange(storeResultBB).InsertAtEnd(LIR::SeqTree(m_comp, storeResultBase));
+
+ resultBase = m_comp->gtNewLclVarNode(resultBaseVar, TYP_REF);
+ }
+
+ assert(callDefInfo.DefinitionNode->OperIs(GT_STORE_LCL_VAR));
+ for (unsigned i = 0; i < resultLcl->lvFieldCnt; i++)
+ {
+ unsigned fieldLclNum = resultLcl->lvFieldLclStart + i;
+ LclVarDsc* fieldDsc = m_comp->lvaGetDesc(fieldLclNum);
+
+ unsigned fldOffset = resultOffset + fieldDsc->lvFldOffset;
+ GenTree* value = LoadFromOffset(resultBase, fldOffset, fieldDsc->TypeGet(), resultIndirFlags);
+ GenTree* store = m_comp->gtNewStoreLclVarNode(fieldLclNum, value);
+ LIR::AsRange(storeResultBB).InsertAtEnd(LIR::SeqTree(m_comp, store));
+
+ if (i + 1 != resultLcl->lvFieldCnt)
+ {
+ resultBase = m_comp->gtCloneExpr(resultBase);
+ }
+ }
+ }
+ }
+ else
+ {
+ GenTree* value = LoadFromOffset(resultBase, resultOffset, call->gtReturnType, resultIndirFlags);
+
+ GenTree* storeResult;
+ if (callDefInfo.DefinitionNode->OperIs(GT_STORE_LCL_VAR))
+ {
+ storeResult = m_comp->gtNewStoreLclVarNode(callDefInfo.DefinitionNode->GetLclNum(), value);
+ }
+ else
+ {
+ storeResult = m_comp->gtNewStoreLclFldNode(callDefInfo.DefinitionNode->GetLclNum(),
+ callDefInfo.DefinitionNode->TypeGet(),
+ callDefInfo.DefinitionNode->GetLclOffs(), value);
+ }
+
+ LIR::AsRange(storeResultBB).InsertAtEnd(LIR::SeqTree(m_comp, storeResult));
+ }
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::LoadFromOffset:
+// Create a load.
+//
+// Parameters:
+// base - Base address of the load
+// offset - Offset to add on top of the base address
+// type - Type of the load to create
+// indirFlags - Flags to add to the load
+//
+// Returns:
+// IR node of the load.
+//
+GenTreeIndir* AsyncTransformation::LoadFromOffset(GenTree* base,
+ unsigned offset,
+ var_types type,
+ GenTreeFlags indirFlags)
+{
+ assert(base->TypeIs(TYP_REF, TYP_BYREF, TYP_I_IMPL));
+ GenTree* cns = m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL);
+ var_types addrType = base->TypeIs(TYP_I_IMPL) ? TYP_I_IMPL : TYP_BYREF;
+ GenTree* addr = m_comp->gtNewOperNode(GT_ADD, addrType, base, cns);
+ GenTreeIndir* load = m_comp->gtNewIndir(type, addr, indirFlags);
+ return load;
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::LoadFromOffset:
+// Create a store.
+//
+// Parameters:
+// base - Base address of the store
+// offset - Offset to add on top of the base address
+// value - Value to store
+//
+// Returns:
+// IR node of the store.
+//
+GenTreeStoreInd* AsyncTransformation::StoreAtOffset(GenTree* base, unsigned offset, GenTree* value)
+{
+ assert(base->TypeIs(TYP_REF, TYP_BYREF, TYP_I_IMPL));
+ GenTree* cns = m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL);
+ var_types addrType = base->TypeIs(TYP_I_IMPL) ? TYP_I_IMPL : TYP_BYREF;
+ GenTree* addr = m_comp->gtNewOperNode(GT_ADD, addrType, base, cns);
+ GenTreeStoreInd* store = m_comp->gtNewStoreIndNode(value->TypeGet(), addr, value, GTF_IND_NONFAULTING);
+ return store;
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::GetDataArrayVar:
+// Create a new local to hold the data array of the continuation object. This
+// local can be validly used for the entire suspension point; the returned
+// local may be used by multiple suspension points.
+//
+// Returns:
+// Local number.
+//
+unsigned AsyncTransformation::GetDataArrayVar()
+{
+ // Create separate locals unless we have many locals in the method for live
+ // range splitting purposes. This helps LSRA to avoid create additional
+ // callee saves that harm the prolog/epilog.
+ if ((m_dataArrayVar == BAD_VAR_NUM) || !m_comp->lvaHaveManyLocals())
+ {
+ m_dataArrayVar = m_comp->lvaGrabTemp(false DEBUGARG("byte[] for continuation"));
+ m_comp->lvaGetDesc(m_dataArrayVar)->lvType = TYP_REF;
+ }
+
+ return m_dataArrayVar;
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::GetGCDataArrayVar:
+// Create a new local to hold the GC pointers array of the continuation
+// object. This local can be validly used for the entire suspension point;
+// the returned local may be used by multiple suspension points.
+//
+// Returns:
+// Local number.
+//
+unsigned AsyncTransformation::GetGCDataArrayVar()
+{
+ if ((m_gcDataArrayVar == BAD_VAR_NUM) || !m_comp->lvaHaveManyLocals())
+ {
+ m_gcDataArrayVar = m_comp->lvaGrabTemp(false DEBUGARG("object[] for continuation"));
+ m_comp->lvaGetDesc(m_gcDataArrayVar)->lvType = TYP_REF;
+ }
+
+ return m_gcDataArrayVar;
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::GetResultBaseVar:
+// Create a new local to hold the base address of the incoming result from
+// the continuation. This local can be validly used for the entire suspension
+// point; the returned local may be used by multiple suspension points.
+//
+// Returns:
+// Local number.
+//
+unsigned AsyncTransformation::GetResultBaseVar()
+{
+ if ((m_resultBaseVar == BAD_VAR_NUM) || !m_comp->lvaHaveManyLocals())
+ {
+ m_resultBaseVar = m_comp->lvaGrabTemp(false DEBUGARG("object for resuming result base"));
+ m_comp->lvaGetDesc(m_resultBaseVar)->lvType = TYP_REF;
+ }
+
+ return m_resultBaseVar;
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::GetExceptionVar:
+// Create a new local to hold the exception in the continuation. This
+// local can be validly used for the entire suspension point; the returned
+// local may be used by multiple suspension points.
+//
+// Returns:
+// Local number.
+//
+unsigned AsyncTransformation::GetExceptionVar()
+{
+ if ((m_exceptionVar == BAD_VAR_NUM) || !m_comp->lvaHaveManyLocals())
+ {
+ m_exceptionVar = m_comp->lvaGrabTemp(false DEBUGARG("object for resuming exception"));
+ m_comp->lvaGetDesc(m_exceptionVar)->lvType = TYP_REF;
+ }
+
+ return m_exceptionVar;
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::CreateResumptionStubAddrTree:
+// Create a tree that represents the address of the resumption stub entry
+// point.
+//
+// Returns:
+// IR node.
+//
+GenTree* AsyncTransformation::CreateResumptionStubAddrTree()
+{
+ switch (m_resumeStubLookup.accessType)
+ {
+ case IAT_VALUE:
+ {
+ return CreateFunctionTargetAddr(m_resumeStub, m_resumeStubLookup);
+ }
+ case IAT_PVALUE:
+ {
+ GenTree* tree = CreateFunctionTargetAddr(m_resumeStub, m_resumeStubLookup);
+ tree = m_comp->gtNewIndir(TYP_I_IMPL, tree, GTF_IND_NONFAULTING | GTF_IND_INVARIANT);
+ return tree;
+ }
+ case IAT_PPVALUE:
+ {
+ noway_assert(!"Unexpected IAT_PPVALUE");
+ return nullptr;
+ }
+ case IAT_RELPVALUE:
+ {
+ GenTree* addr = CreateFunctionTargetAddr(m_resumeStub, m_resumeStubLookup);
+ GenTree* tree = CreateFunctionTargetAddr(m_resumeStub, m_resumeStubLookup);
+ tree = m_comp->gtNewIndir(TYP_I_IMPL, tree, GTF_IND_NONFAULTING | GTF_IND_INVARIANT);
+ tree = m_comp->gtNewOperNode(GT_ADD, TYP_I_IMPL, tree, addr);
+ return tree;
+ }
+ default:
+ {
+ noway_assert(!"Bad accessType");
+ return nullptr;
+ }
+ }
+}
+
+//------------------------------------------------------------------------
+// AsyncTransformation::CreateFunctionTargetAddr:
+// Create a tree that represents the address of the resumption stub entry
+// point.
+//
+// Returns:
+// IR node.
+//
+GenTree* AsyncTransformation::CreateFunctionTargetAddr(CORINFO_METHOD_HANDLE methHnd,
+ const CORINFO_CONST_LOOKUP& lookup)
+{
+ GenTree* con = m_comp->gtNewIconHandleNode((size_t)lookup.addr, GTF_ICON_FTN_ADDR);
+ INDEBUG(con->AsIntCon()->gtTargetHandle = (size_t)methHnd);
+ return con;
+}
+
+void AsyncTransformation::CreateResumptionSwitch()
+{
+ m_comp->fgCreateNewInitBB();
+ BasicBlock* newEntryBB = m_comp->fgFirstBB;
+
+ GenTree* continuationArg = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF);
+ GenTree* null = m_comp->gtNewNull();
+ GenTree* neNull = m_comp->gtNewOperNode(GT_NE, TYP_INT, continuationArg, null);
+ GenTree* jtrue = m_comp->gtNewOperNode(GT_JTRUE, TYP_VOID, neNull);
+ LIR::AsRange(newEntryBB).InsertAtEnd(continuationArg, null, neNull, jtrue);
+
+ FlowEdge* resumingEdge;
+
+ if (m_resumptionBBs.size() == 1)
+ {
+ JITDUMP(" Redirecting entry " FMT_BB " directly to " FMT_BB " as it is the only resumption block\n",
+ newEntryBB->bbNum, m_resumptionBBs[0]->bbNum);
+ resumingEdge = m_comp->fgAddRefPred(m_resumptionBBs[0], newEntryBB);
+ }
+ else if (m_resumptionBBs.size() == 2)
+ {
+ BasicBlock* condBB = m_comp->fgNewBBbefore(BBJ_COND, m_resumptionBBs[0], true);
+ condBB->inheritWeightPercentage(newEntryBB, 0);
+
+ FlowEdge* to0 = m_comp->fgAddRefPred(m_resumptionBBs[0], condBB);
+ FlowEdge* to1 = m_comp->fgAddRefPred(m_resumptionBBs[1], condBB);
+ condBB->SetCond(to1, to0);
+ to1->setLikelihood(0.5);
+ to0->setLikelihood(0.5);
+
+ resumingEdge = m_comp->fgAddRefPred(condBB, newEntryBB);
+
+ JITDUMP(" Redirecting entry " FMT_BB " to BBJ_COND " FMT_BB " for resumption with 2 states\n",
+ newEntryBB->bbNum, condBB->bbNum);
+
+ continuationArg = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF);
+ unsigned stateOffset = m_comp->info.compCompHnd->getFieldOffset(m_async2Info.continuationStateFldHnd);
+ GenTree* stateOffsetNode = m_comp->gtNewIconNode((ssize_t)stateOffset, TYP_I_IMPL);
+ GenTree* stateAddr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, continuationArg, stateOffsetNode);
+ GenTree* stateInd = m_comp->gtNewIndir(TYP_INT, stateAddr, GTF_IND_NONFAULTING);
+ GenTree* zero = m_comp->gtNewZeroConNode(TYP_INT);
+ GenTree* stateNeZero = m_comp->gtNewOperNode(GT_NE, TYP_INT, stateInd, zero);
+ GenTree* jtrue = m_comp->gtNewOperNode(GT_JTRUE, TYP_VOID, stateNeZero);
+
+ LIR::AsRange(condBB).InsertAtEnd(continuationArg, stateOffsetNode, stateAddr, stateInd, zero, stateNeZero,
+ jtrue);
+ }
+ else
+ {
+ BasicBlock* switchBB = m_comp->fgNewBBbefore(BBJ_SWITCH, m_resumptionBBs[0], true);
+ switchBB->inheritWeightPercentage(newEntryBB, 0);
+
+ resumingEdge = m_comp->fgAddRefPred(switchBB, newEntryBB);
+
+ JITDUMP(" Redirecting entry " FMT_BB " to BBJ_SWITCH " FMT_BB " for resumption with %zu states\n",
+ newEntryBB->bbNum, switchBB->bbNum, m_resumptionBBs.size());
+
+ continuationArg = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF);
+ unsigned stateOffset = m_comp->info.compCompHnd->getFieldOffset(m_async2Info.continuationStateFldHnd);
+ GenTree* stateOffsetNode = m_comp->gtNewIconNode((ssize_t)stateOffset, TYP_I_IMPL);
+ GenTree* stateAddr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, continuationArg, stateOffsetNode);
+ GenTree* stateInd = m_comp->gtNewIndir(TYP_INT, stateAddr, GTF_IND_NONFAULTING);
+ GenTree* switchNode = m_comp->gtNewOperNode(GT_SWITCH, TYP_VOID, stateInd);
+
+ LIR::AsRange(switchBB).InsertAtEnd(continuationArg, stateOffsetNode, stateAddr, stateInd, switchNode);
+
+ m_comp->fgHasSwitch = true;
+
+ // Default case. TODO-CQ: Support bbsHasDefault = false before lowering.
+ m_resumptionBBs.push_back(m_resumptionBBs[0]);
+ BBswtDesc* swtDesc = new (m_comp, CMK_BasicBlock) BBswtDesc;
+ swtDesc->bbsCount = (unsigned)m_resumptionBBs.size();
+ swtDesc->bbsHasDefault = true;
+ swtDesc->bbsDstTab = new (m_comp, CMK_Async) FlowEdge*[m_resumptionBBs.size()];
+
+ weight_t stateLikelihood = 1.0 / m_resumptionBBs.size();
+ for (size_t i = 0; i < m_resumptionBBs.size(); i++)
+ {
+ swtDesc->bbsDstTab[i] = m_comp->fgAddRefPred(m_resumptionBBs[i], switchBB);
+ swtDesc->bbsDstTab[i]->setLikelihood(stateLikelihood);
+ }
+
+ switchBB->SetSwitch(swtDesc);
+ }
+
+ newEntryBB->SetCond(resumingEdge, newEntryBB->GetTargetEdge());
+ resumingEdge->setLikelihood(0);
+ newEntryBB->GetFalseEdge()->setLikelihood(1);
+
+ if (m_comp->doesMethodHavePatchpoints())
+ {
+ JITDUMP(" Method has patch points...\n");
+ // If we have patchpoints then first check if we need to resume in the OSR version.
+ BasicBlock* callHelperBB = m_comp->fgNewBBafter(BBJ_THROW, m_comp->fgLastBBInMainFunction(), false);
+ callHelperBB->bbSetRunRarely();
+ callHelperBB->clearTryIndex();
+ callHelperBB->clearHndIndex();
+
+ JITDUMP(" Created " FMT_BB " for transitions back into OSR method\n", callHelperBB->bbNum);
+
+ BasicBlock* onContinuationBB = newEntryBB->GetTrueTarget();
+ BasicBlock* checkILOffsetBB = m_comp->fgNewBBbefore(BBJ_COND, onContinuationBB, true);
+
+ JITDUMP(" Created " FMT_BB " to check whether we should transition immediately to OSR\n",
+ checkILOffsetBB->bbNum);
+
+ // Redirect newEntryBB -> onContinuationBB into newEntryBB -> checkILOffsetBB -> onContinuationBB
+ m_comp->fgRemoveRefPred(newEntryBB->GetTrueEdge());
+
+ FlowEdge* toCheckILOffsetBB = m_comp->fgAddRefPred(checkILOffsetBB, newEntryBB);
+ newEntryBB->SetTrueEdge(toCheckILOffsetBB);
+ toCheckILOffsetBB->setLikelihood(0);
+ checkILOffsetBB->inheritWeightPercentage(newEntryBB, 0);
+
+ FlowEdge* toOnContinuationBB = m_comp->fgAddRefPred(onContinuationBB, checkILOffsetBB);
+ FlowEdge* toCallHelperBB = m_comp->fgAddRefPred(callHelperBB, checkILOffsetBB);
+ checkILOffsetBB->SetCond(toCallHelperBB, toOnContinuationBB);
+ toCallHelperBB->setLikelihood(0);
+ toOnContinuationBB->setLikelihood(1);
+ callHelperBB->inheritWeightPercentage(checkILOffsetBB, 0);
+
+ // We need to dispatch to the OSR version if the IL offset is non-negative.
+ continuationArg = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF);
+ unsigned offsetOfData = m_comp->info.compCompHnd->getFieldOffset(m_async2Info.continuationDataFldHnd);
+ GenTree* dataArr = LoadFromOffset(continuationArg, offsetOfData, TYP_REF);
+ unsigned offsetOfIlOffset = OFFSETOF__CORINFO_Array__data;
+ GenTree* ilOffset = LoadFromOffset(dataArr, offsetOfIlOffset, TYP_INT);
+ unsigned ilOffsetLclNum = m_comp->lvaGrabTemp(false DEBUGARG("IL offset for tier0 OSR method"));
+ m_comp->lvaGetDesc(ilOffsetLclNum)->lvType = TYP_INT;
+ GenTree* storeIlOffset = m_comp->gtNewStoreLclVarNode(ilOffsetLclNum, ilOffset);
+ LIR::AsRange(checkILOffsetBB).InsertAtEnd(LIR::SeqTree(m_comp, storeIlOffset));
+
+ ilOffset = m_comp->gtNewLclvNode(ilOffsetLclNum, TYP_INT);
+ GenTree* zero = m_comp->gtNewIconNode(0);
+ GenTree* geZero = m_comp->gtNewOperNode(GT_GE, TYP_INT, ilOffset, zero);
+ GenTree* jtrue = m_comp->gtNewOperNode(GT_JTRUE, TYP_VOID, geZero);
+ LIR::AsRange(checkILOffsetBB).InsertAtEnd(ilOffset, zero, geZero, jtrue);
+
+ ilOffset = m_comp->gtNewLclvNode(ilOffsetLclNum, TYP_INT);
+ GenTreeCall* callHelper = m_comp->gtNewHelperCallNode(CORINFO_HELP_RESUME_OSR, TYP_VOID, ilOffset);
+ callHelper->gtCallMoreFlags |= GTF_CALL_M_DOES_NOT_RETURN;
+
+ m_comp->compCurBB = callHelperBB;
+ m_comp->fgMorphTree(callHelper);
+
+ LIR::AsRange(callHelperBB).InsertAtEnd(LIR::SeqTree(m_comp, callHelper));
+ }
+ else if (m_comp->opts.IsOSR())
+ {
+ JITDUMP(" Method is an OSR function\n");
+ // If the tier-0 version resumed and then transitioned to the OSR
+ // version by normal means then we will see a non-zero continuation
+ // here that belongs to the tier0 method. In that case we should just
+ // ignore it, so create a BB that jumps back.
+ BasicBlock* onContinuationBB = newEntryBB->GetTrueTarget();
+ BasicBlock* onNoContinuationBB = newEntryBB->GetFalseTarget();
+ BasicBlock* checkILOffsetBB = m_comp->fgNewBBbefore(BBJ_COND, onContinuationBB, true);
+
+ // Switch newEntryBB -> onContinuationBB into newEntryBB -> checkILOffsetBB
+ m_comp->fgRemoveRefPred(newEntryBB->GetTrueEdge());
+ FlowEdge* toCheckILOffset = m_comp->fgAddRefPred(checkILOffsetBB, newEntryBB);
+ newEntryBB->SetTrueEdge(toCheckILOffset);
+ toCheckILOffset->setLikelihood(0);
+ checkILOffsetBB->inheritWeightPercentage(newEntryBB, 0);
+
+ // Make checkILOffsetBB ->(true) onNoContinuationBB
+ // ->(false) onContinuationBB
+
+ FlowEdge* toOnContinuationBB = m_comp->fgAddRefPred(onContinuationBB, checkILOffsetBB);
+ FlowEdge* toOnNoContinuationBB = m_comp->fgAddRefPred(onNoContinuationBB, checkILOffsetBB);
+ checkILOffsetBB->SetCond(toOnNoContinuationBB, toOnContinuationBB);
+ toOnContinuationBB->setLikelihood(0);
+ toOnNoContinuationBB->setLikelihood(1);
+
+ JITDUMP(" Created " FMT_BB " to check for Tier-0 continuations\n", checkILOffsetBB->bbNum);
+
+ continuationArg = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF);
+ unsigned offsetOfData = m_comp->info.compCompHnd->getFieldOffset(m_async2Info.continuationDataFldHnd);
+ GenTree* dataArr = LoadFromOffset(continuationArg, offsetOfData, TYP_REF);
+ unsigned offsetOfIlOffset = OFFSETOF__CORINFO_Array__data;
+ GenTree* ilOffset = LoadFromOffset(dataArr, offsetOfIlOffset, TYP_INT);
+ GenTree* zero = m_comp->gtNewIconNode(0);
+ GenTree* ltZero = m_comp->gtNewOperNode(GT_LT, TYP_INT, ilOffset, zero);
+ GenTree* jtrue = m_comp->gtNewOperNode(GT_JTRUE, TYP_VOID, ltZero);
+ LIR::AsRange(checkILOffsetBB).InsertAtEnd(LIR::SeqTree(m_comp, jtrue));
+ }
+}
diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h
new file mode 100644
index 00000000000000..e7d3933be65a9d
--- /dev/null
+++ b/src/coreclr/jit/async.h
@@ -0,0 +1,152 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+struct LiveLocalInfo
+{
+ unsigned LclNum;
+ unsigned Alignment;
+ unsigned DataOffset;
+ unsigned DataSize;
+ unsigned GCDataIndex;
+ unsigned GCDataCount;
+
+ explicit LiveLocalInfo(unsigned lclNum)
+ : LclNum(lclNum)
+ {
+ }
+};
+
+struct ContinuationLayout
+{
+ unsigned DataSize = 0;
+ unsigned GCRefsCount = 0;
+ ClassLayout* ReturnStructLayout = nullptr;
+ unsigned ReturnSize = 0;
+ bool ReturnInGCData = false;
+ unsigned ReturnValDataOffset = UINT_MAX;
+ unsigned ExceptionGCDataIndex = UINT_MAX;
+ const jitstd::vector& Locals;
+
+ ContinuationLayout(jitstd::vector& locals)
+ : Locals(locals)
+ {
+ }
+};
+
+struct CallDefinitionInfo
+{
+ GenTreeLclVarCommon* DefinitionNode = nullptr;
+
+ // Where to insert new IR for suspension checks.
+ GenTree* InsertAfter = nullptr;
+};
+
+class AsyncTransformation
+{
+ friend class AsyncLiveness;
+
+ Compiler* m_comp;
+ jitstd::vector m_liveLocalsScratch;
+ CORINFO_ASYNC2_INFO m_async2Info;
+ jitstd::vector m_resumptionBBs;
+ CORINFO_METHOD_HANDLE m_resumeStub = NO_METHOD_HANDLE;
+ CORINFO_CONST_LOOKUP m_resumeStubLookup;
+ unsigned m_returnedContinuationVar = BAD_VAR_NUM;
+ unsigned m_newContinuationVar = BAD_VAR_NUM;
+ unsigned m_dataArrayVar = BAD_VAR_NUM;
+ unsigned m_gcDataArrayVar = BAD_VAR_NUM;
+ unsigned m_resultBaseVar = BAD_VAR_NUM;
+ unsigned m_exceptionVar = BAD_VAR_NUM;
+ BasicBlock* m_lastSuspensionBB = nullptr;
+ BasicBlock* m_lastResumptionBB = nullptr;
+ BasicBlock* m_sharedReturnBB = nullptr;
+
+ bool IsLive(unsigned lclNum);
+ void Transform(BasicBlock* block,
+ GenTreeCall* call,
+ jitstd::vector& defs,
+ class AsyncLiveness& life,
+ BasicBlock** remainder);
+
+ void CreateLiveSetForSuspension(BasicBlock* block,
+ GenTreeCall* call,
+ const jitstd::vector& defs,
+ AsyncLiveness& life,
+ jitstd::vector& liveLocals);
+
+ void LiftLIREdges(BasicBlock* block,
+ const jitstd::vector& defs,
+ jitstd::vector& liveLocals);
+
+ ContinuationLayout LayOutContinuation(BasicBlock* block,
+ GenTreeCall* call,
+ jitstd::vector& liveLocals);
+
+ CallDefinitionInfo CanonicalizeCallDefinition(BasicBlock* block, GenTreeCall* call, AsyncLiveness& life);
+
+ BasicBlock* CreateSuspension(BasicBlock* block,
+ unsigned stateNum,
+ AsyncLiveness& life,
+ const ContinuationLayout& layout);
+ GenTreeCall* CreateAllocContinuationCall(AsyncLiveness& life,
+ GenTree* prevContinuation,
+ unsigned gcRefsCount,
+ unsigned int dataSize);
+ void FillInGCPointersOnSuspension(const jitstd::vector& liveLocals, BasicBlock* suspendBB);
+ void FillInDataOnSuspension(const jitstd::vector& liveLocals, BasicBlock* suspendBB);
+ void CreateCheckAndSuspendAfterCall(BasicBlock* block,
+ const CallDefinitionInfo& callDefInfo,
+ AsyncLiveness& life,
+ BasicBlock* suspendBB,
+ BasicBlock** remainder);
+
+ BasicBlock* CreateResumption(BasicBlock* block,
+ BasicBlock* remainder,
+ GenTreeCall* call,
+ const CallDefinitionInfo& callDefInfo,
+ unsigned stateNum,
+ const ContinuationLayout& layout);
+ void RestoreFromDataOnResumption(unsigned resumeByteArrLclNum,
+ const jitstd::vector& liveLocals,
+ BasicBlock* resumeBB);
+ void RestoreFromGCPointersOnResumption(unsigned resumeObjectArrLclNum,
+ const jitstd::vector& liveLocals,
+ BasicBlock* resumeBB);
+ BasicBlock* RethrowExceptionOnResumption(BasicBlock* block,
+ BasicBlock* remainder,
+ unsigned resumeObjectArrLclNum,
+ const ContinuationLayout& layout,
+ BasicBlock* resumeBB);
+ void CopyReturnValueOnResumption(GenTreeCall* call,
+ const CallDefinitionInfo& callDefInfo,
+ unsigned resumeByteArrLclNum,
+ unsigned resumeObjectArrLclNum,
+ const ContinuationLayout& layout,
+ BasicBlock* storeResultBB);
+
+ GenTreeIndir* LoadFromOffset(GenTree* base,
+ unsigned offset,
+ var_types type,
+ GenTreeFlags indirFlags = GTF_IND_NONFAULTING);
+ GenTreeStoreInd* StoreAtOffset(GenTree* base, unsigned offset, GenTree* value);
+
+ unsigned GetDataArrayVar();
+ unsigned GetGCDataArrayVar();
+ unsigned GetResultBaseVar();
+ unsigned GetExceptionVar();
+
+ GenTree* CreateResumptionStubAddrTree();
+ GenTree* CreateFunctionTargetAddr(CORINFO_METHOD_HANDLE methHnd, const CORINFO_CONST_LOOKUP& lookup);
+
+ void CreateResumptionSwitch();
+
+public:
+ AsyncTransformation(Compiler* comp)
+ : m_comp(comp)
+ , m_liveLocalsScratch(comp->getAllocator(CMK_Async))
+ , m_resumptionBBs(comp->getAllocator(CMK_Async))
+ {
+ }
+
+ PhaseStatus Run();
+};
diff --git a/src/coreclr/jit/block.cpp b/src/coreclr/jit/block.cpp
index d0a6833400faa1..cbbf8249c669dc 100644
--- a/src/coreclr/jit/block.cpp
+++ b/src/coreclr/jit/block.cpp
@@ -527,6 +527,7 @@ void BasicBlock::dspFlags() const
{BBF_HAS_ALIGN, "has-align"},
{BBF_HAS_MDARRAYREF, "mdarr"},
{BBF_NEEDS_GCPOLL, "gcpoll"},
+ {BBF_ASYNC_RESUMPTION, "resume"},
};
bool first = true;
diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h
index 35595928fc7a5c..0ecf8bdf42557f 100644
--- a/src/coreclr/jit/block.h
+++ b/src/coreclr/jit/block.h
@@ -463,6 +463,7 @@ enum BasicBlockFlags : uint64_t
BBF_HAS_VALUE_PROFILE = MAKE_BBFLAG(38), // Block has a node that needs a value probing
BBF_HAS_NEWARR = MAKE_BBFLAG(39), // BB contains 'new' of an array type.
BBF_MAY_HAVE_BOUNDS_CHECKS = MAKE_BBFLAG(40), // BB *likely* has a bounds check (after rangecheck phase).
+ BBF_ASYNC_RESUMPTION = MAKE_BBFLAG(41), // Block is a resumption block in an async method
// The following are sets of flags.
@@ -480,7 +481,7 @@ enum BasicBlockFlags : uint64_t
// For example, the top block might or might not have BBF_GC_SAFE_POINT,
// but we assume it does not have BBF_GC_SAFE_POINT any more.
- BBF_SPLIT_LOST = BBF_GC_SAFE_POINT | BBF_NEEDS_GCPOLL | BBF_HAS_JMP | BBF_KEEP_BBJ_ALWAYS | BBF_CLONED_FINALLY_END | BBF_RECURSIVE_TAILCALL,
+ BBF_SPLIT_LOST = BBF_GC_SAFE_POINT | BBF_NEEDS_GCPOLL | BBF_HAS_JMP | BBF_KEEP_BBJ_ALWAYS | BBF_CLONED_FINALLY_END | BBF_RECURSIVE_TAILCALL | BBF_ASYNC_RESUMPTION,
// Flags gained by the bottom block when a block is split.
// Note, this is a conservative guess.
@@ -488,7 +489,7 @@ enum BasicBlockFlags : uint64_t
// TODO: Should BBF_RUN_RARELY be added to BBF_SPLIT_GAINED ?
BBF_SPLIT_GAINED = BBF_DONT_REMOVE | BBF_HAS_JMP | BBF_BACKWARD_JUMP | BBF_HAS_IDX_LEN | BBF_HAS_MD_IDX_LEN | BBF_PROF_WEIGHT | BBF_HAS_NEWARR | \
- BBF_HAS_NEWOBJ | BBF_KEEP_BBJ_ALWAYS | BBF_CLONED_FINALLY_END | BBF_HAS_NULLCHECK | BBF_HAS_HISTOGRAM_PROFILE | BBF_HAS_VALUE_PROFILE | BBF_HAS_MDARRAYREF | BBF_NEEDS_GCPOLL | BBF_MAY_HAVE_BOUNDS_CHECKS,
+ BBF_HAS_NEWOBJ | BBF_KEEP_BBJ_ALWAYS | BBF_CLONED_FINALLY_END | BBF_HAS_NULLCHECK | BBF_HAS_HISTOGRAM_PROFILE | BBF_HAS_VALUE_PROFILE | BBF_HAS_MDARRAYREF | BBF_NEEDS_GCPOLL | BBF_MAY_HAVE_BOUNDS_CHECKS | BBF_ASYNC_RESUMPTION,
// Flags that must be propagated to a new block if code is copied from a block to a new block. These are flags that
// limit processing of a block if the code in question doesn't exist. This is conservative; we might not
diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h
index 99cb565b68ea62..98ace787cc6648 100644
--- a/src/coreclr/jit/codegen.h
+++ b/src/coreclr/jit/codegen.h
@@ -557,6 +557,7 @@ class CodeGen final : public CodeGenInterface
X86_ARG(int argSize),
emitAttr retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet,
const DebugInfo& di,
regNumber base,
bool isJump,
@@ -571,6 +572,7 @@ class CodeGen final : public CodeGenInterface
X86_ARG(int argSize),
emitAttr retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet,
const DebugInfo& di,
bool isJump);
// clang-format on
@@ -1238,6 +1240,7 @@ class CodeGen final : public CodeGenInterface
#ifdef SWIFT_SUPPORT
void genCodeForSwiftErrorReg(GenTree* tree);
#endif // SWIFT_SUPPORT
+ void genCodeForAsyncContinuation(GenTree* tree);
void genCodeForNullCheck(GenTreeIndir* tree);
void genCodeForCmpXchg(GenTreeCmpXchg* tree);
void genCodeForReuseVal(GenTree* treeNode);
@@ -1370,6 +1373,8 @@ class CodeGen final : public CodeGenInterface
#endif // TARGET_ARM64 || TARGET_LOONGARCH64 || TARGET_RISCV64
void genReturn(GenTree* treeNode);
+ void genReturnSuspend(GenTreeUnOp* treeNode);
+ void genMarkReturnGCInfo();
#ifdef SWIFT_SUPPORT
void genSwiftErrorReturn(GenTree* treeNode);
diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp
index c91f401cc4c697..2eec1d69d04613 100644
--- a/src/coreclr/jit/codegenarm.cpp
+++ b/src/coreclr/jit/codegenarm.cpp
@@ -1641,8 +1641,9 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize,
GetEmitter()->emitIns_Call(emitter::EC_INDIR_R, compiler->eeFindHelper(helper),
INDEBUG_LDISASM_COMMA(nullptr) NULL, // addr
- argSize, retSize, gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur,
- gcInfo.gcRegByrefSetCur, DebugInfo(),
+ argSize, retSize,
+ false, // hasAsyncRet
+ gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur, DebugInfo(),
callTargetReg, // ireg
REG_NA, 0, 0, // xreg, xmul, disp
false // isJump
@@ -1651,10 +1652,11 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize,
else
{
GetEmitter()->emitIns_Call(emitter::EC_FUNC_TOKEN, compiler->eeFindHelper(helper),
- INDEBUG_LDISASM_COMMA(nullptr) addr, argSize, retSize, gcInfo.gcVarPtrSetCur,
- gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur, DebugInfo(), REG_NA, REG_NA, 0,
- 0, /* ilOffset, ireg, xreg, xmul, disp */
- false /* isJump */
+ INDEBUG_LDISASM_COMMA(nullptr) addr, argSize, retSize,
+ false, // hasAsyncRet
+ gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur, DebugInfo(),
+ REG_NA, REG_NA, 0, 0, /* ilOffset, ireg, xreg, xmul, disp */
+ false /* isJump */
);
}
@@ -2084,7 +2086,16 @@ regMaskTP CodeGen::genStackAllocRegisterMask(unsigned frameSize, regMaskTP maskC
// We can't do this optimization with callee saved floating point registers because
// the stack would be allocated in a wrong spot.
if (maskCalleeSavedFloat != RBM_NONE)
+ {
+ return RBM_NONE;
+ }
+
+ // We similarly skip it for async2 due to the extra async continuation
+ // return that may be overridden by the pop.
+ if (compiler->compIsAsync())
+ {
return RBM_NONE;
+ }
// Allocate space for small frames by pushing extra registers. It generates smaller and faster code
// that extra sub sp,XXX/add sp,XXX.
diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp
index ea110494cb1aca..a38dd5da99d6f1 100644
--- a/src/coreclr/jit/codegenarm64.cpp
+++ b/src/coreclr/jit/codegenarm64.cpp
@@ -5353,10 +5353,12 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize,
}
GetEmitter()->emitIns_Call(callType, compiler->eeFindHelper(helper), INDEBUG_LDISASM_COMMA(nullptr) addr, argSize,
- retSize, EA_UNKNOWN, gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur,
- gcInfo.gcRegByrefSetCur, DebugInfo(), callTarget, /* ireg */
- REG_NA, 0, 0, /* xreg, xmul, disp */
- false /* isJump */
+ retSize, EA_UNKNOWN,
+ false, // hasAsyncRet
+ gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur, DebugInfo(),
+ callTarget, /* ireg */
+ REG_NA, 0, 0, /* xreg, xmul, disp */
+ false /* isJump */
);
regMaskTP killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper);
diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp
index c4cbf359d87229..7bb5c13f742e3d 100644
--- a/src/coreclr/jit/codegenarmarch.cpp
+++ b/src/coreclr/jit/codegenarmarch.cpp
@@ -295,6 +295,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
break;
#endif // SWIFT_SUPPORT
+ case GT_RETURN_SUSPEND:
+ genReturnSuspend(treeNode->AsUnOp());
+ break;
+
case GT_LEA:
// If we are here, it is the case where there is an LEA that cannot be folded into a parent instruction.
genLeaInstruction(treeNode->AsAddrMode());
@@ -510,6 +514,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
genConsumeReg(treeNode);
break;
+ case GT_ASYNC_CONTINUATION:
+ genCodeForAsyncContinuation(treeNode);
+ break;
+
case GT_PINVOKE_PROLOG:
noway_assert(((gcInfo.gcRegGCrefSetCur | gcInfo.gcRegByrefSetCur) &
~fullIntArgRegMask(compiler->info.compCallConv)) == 0);
@@ -3536,6 +3544,8 @@ void CodeGen::genCallInstruction(GenTreeCall* call)
}
}
#endif // DEBUG
+
+ bool hasAsyncRet = call->IsAsync();
CORINFO_METHOD_HANDLE methHnd;
GenTree* target = getCallTarget(call, &methHnd);
@@ -3604,6 +3614,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call)
nullptr, // addr
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
di,
target->GetRegNum(),
call->IsFastTailCall(),
@@ -3671,6 +3682,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call)
nullptr, // addr
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
di,
targetAddrReg,
call->IsFastTailCall());
@@ -3719,6 +3731,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call)
INDEBUG_LDISASM_COMMA(sigInfo)
NULL,
retSize,
+ hasAsyncRet,
di,
tmpReg,
call->IsFastTailCall());
@@ -3734,6 +3747,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call)
addr,
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
di,
REG_NA,
call->IsFastTailCall());
@@ -5395,6 +5409,7 @@ void CodeGen::genFnEpilog(BasicBlock* block)
#if defined(TARGET_ARM64)
EA_UNKNOWN, // secondRetSize
#endif
+ false, // hasAsyncRet
gcInfo.gcVarPtrSetCur,
gcInfo.gcRegGCrefSetCur,
gcInfo.gcRegByrefSetCur,
diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp
index 60a44c3356c628..164db290b34bb0 100644
--- a/src/coreclr/jit/codegencommon.cpp
+++ b/src/coreclr/jit/codegencommon.cpp
@@ -1938,6 +1938,11 @@ void CodeGen::genGenerateMachineCode()
printf("; OSR variant for entry point 0x%x\n", compiler->info.compILEntry);
}
+ if (compiler->compIsAsync())
+ {
+ printf("; async2\n");
+ }
+
if ((compiler->opts.compFlags & CLFLG_MAXOPT) == CLFLG_MAXOPT)
{
printf("; optimized code\n");
@@ -7238,23 +7243,14 @@ void CodeGen::genReturn(GenTree* treeNode)
}
}
- if (treeNode->OperIs(GT_RETURN, GT_SWIFT_ERROR_RET))
+ if (treeNode->OperIs(GT_RETURN) && compiler->compIsAsync())
{
- const ReturnTypeDesc& retTypeDesc = compiler->compRetTypeDesc;
+ instGen_Set_Reg_To_Zero(EA_PTRSIZE, REG_ASYNC_CONTINUATION_RET);
+ }
- if (compiler->compMethodReturnsRetBufAddr())
- {
- gcInfo.gcMarkRegPtrVal(REG_INTRET, TYP_BYREF);
- }
- else
- {
- unsigned retRegCount = retTypeDesc.GetReturnRegCount();
- for (unsigned i = 0; i < retRegCount; ++i)
- {
- gcInfo.gcMarkRegPtrVal(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv),
- retTypeDesc.GetReturnRegType(i));
- }
- }
+ if (treeNode->OperIs(GT_RETURN, GT_SWIFT_ERROR_RET))
+ {
+ genMarkReturnGCInfo();
}
#ifdef PROFILING_SUPPORTED
@@ -7321,6 +7317,83 @@ void CodeGen::genSwiftErrorReturn(GenTree* treeNode)
}
#endif // SWIFT_SUPPORT
+//------------------------------------------------------------------------
+// genReturnSuspend:
+// Generate code for a GT_RETURN_SUSPEND node
+//
+// Arguments:
+// treeNode - The node
+//
+void CodeGen::genReturnSuspend(GenTreeUnOp* treeNode)
+{
+ GenTree* op = treeNode->gtGetOp1();
+ assert(op->TypeIs(TYP_REF));
+
+ regNumber reg = genConsumeReg(op);
+ inst_Mov(TYP_REF, REG_ASYNC_CONTINUATION_RET, reg, /* canSkip */ true);
+
+ ReturnTypeDesc retTypeDesc = compiler->compRetTypeDesc;
+ unsigned numRetRegs = retTypeDesc.GetReturnRegCount();
+ for (unsigned i = 0; i < numRetRegs; i++)
+ {
+ if (varTypeIsGC(retTypeDesc.GetReturnRegType(i)))
+ {
+ regNumber returnReg = retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv);
+ instGen_Set_Reg_To_Zero(EA_PTRSIZE, returnReg);
+ }
+ }
+
+ genMarkReturnGCInfo();
+}
+
+//------------------------------------------------------------------------
+// genMarkReturnGCInfo:
+// Mark GC and non-GC pointers of return registers going into the epilog..
+//
+void CodeGen::genMarkReturnGCInfo()
+{
+ const ReturnTypeDesc& retTypeDesc = compiler->compRetTypeDesc;
+
+ if (compiler->compMethodReturnsRetBufAddr())
+ {
+ gcInfo.gcMarkRegPtrVal(REG_INTRET, TYP_BYREF);
+ }
+ else
+ {
+ unsigned retRegCount = retTypeDesc.GetReturnRegCount();
+ for (unsigned i = 0; i < retRegCount; ++i)
+ {
+ gcInfo.gcMarkRegPtrVal(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv),
+ retTypeDesc.GetReturnRegType(i));
+ }
+ }
+
+ if (compiler->compIsAsync())
+ {
+ gcInfo.gcMarkRegPtrVal(REG_ASYNC_CONTINUATION_RET, TYP_REF);
+ }
+}
+
+//------------------------------------------------------------------------
+// genCodeForAsyncContinuation:
+// Generate code for a GC_ASYNC_CONTINUATION node.
+//
+// Arguments:
+// tree - The node
+//
+void CodeGen::genCodeForAsyncContinuation(GenTree* tree)
+{
+ assert(tree->OperIs(GT_ASYNC_CONTINUATION));
+
+ var_types targetType = tree->TypeGet();
+ regNumber targetReg = tree->GetRegNum();
+
+ inst_Mov(targetType, targetReg, REG_ASYNC_CONTINUATION_RET, /* canSkip */ true);
+ genTransferRegGCState(targetReg, REG_ASYNC_CONTINUATION_RET);
+
+ genProduceReg(tree);
+}
+
//------------------------------------------------------------------------
// isStructReturn: Returns whether the 'treeNode' is returning a struct.
//
diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp
index 6fdcd0ee23d36e..f5ae5a5e9ba593 100644
--- a/src/coreclr/jit/codegenlinear.cpp
+++ b/src/coreclr/jit/codegenlinear.cpp
@@ -503,8 +503,13 @@ void CodeGen::genCodeForBBlist()
}
}
+ if (compiler->compIsAsync())
+ {
+ nonVarPtrRegs &= ~RBM_ASYNC_CONTINUATION_RET;
+ }
+
// For a tailcall arbitrary argument registers may be live into the
- // prolog. Skip validating those.
+ // epilog. Skip validating those.
if (block->HasFlag(BBF_HAS_JMP))
{
nonVarPtrRegs &= ~fullIntArgRegMask(CorInfoCallConvExtension::Managed);
@@ -2268,6 +2273,7 @@ void CodeGen::genEmitCall(int callType,
X86_ARG(int argSize),
emitAttr retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet,
const DebugInfo& di,
regNumber base,
bool isJump,
@@ -2288,6 +2294,7 @@ void CodeGen::genEmitCall(int callType,
argSize,
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
gcInfo.gcVarPtrSetCur,
gcInfo.gcRegGCrefSetCur,
gcInfo.gcRegByrefSetCur,
@@ -2307,6 +2314,7 @@ void CodeGen::genEmitCallIndir(int callType,
X86_ARG(int argSize),
emitAttr retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet,
const DebugInfo& di,
bool isJump)
{
@@ -2329,6 +2337,7 @@ void CodeGen::genEmitCallIndir(int callType,
argSize,
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
gcInfo.gcVarPtrSetCur,
gcInfo.gcRegGCrefSetCur,
gcInfo.gcRegByrefSetCur,
diff --git a/src/coreclr/jit/codegenloongarch64.cpp b/src/coreclr/jit/codegenloongarch64.cpp
index 32826a7f56e50f..b74b5ccd0deed5 100644
--- a/src/coreclr/jit/codegenloongarch64.cpp
+++ b/src/coreclr/jit/codegenloongarch64.cpp
@@ -827,6 +827,7 @@ void CodeGen::genFnEpilog(BasicBlock* block)
0, // argSize
EA_UNKNOWN // retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(EA_UNKNOWN), // secondRetSize
+ false, // hasAsyncRet
gcInfo.gcVarPtrSetCur,
gcInfo.gcRegGCrefSetCur,
gcInfo.gcRegByrefSetCur,
@@ -2891,11 +2892,13 @@ void CodeGen::genCodeForReturnTrap(GenTreeOp* tree)
// TODO-LOONGARCH64: can optimize further !!!
GetEmitter()->emitIns_Call(callType, compiler->eeFindHelper(CORINFO_HELP_STOP_FOR_GC),
- INDEBUG_LDISASM_COMMA(nullptr) addr, 0, EA_UNKNOWN, EA_UNKNOWN, gcInfo.gcVarPtrSetCur,
- gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur, DebugInfo(), /* IL offset */
- callTarget, /* ireg */
- REG_NA, 0, 0, /* xreg, xmul, disp */
- false /* isJump */
+ INDEBUG_LDISASM_COMMA(nullptr) addr, 0, EA_UNKNOWN, EA_UNKNOWN,
+ false, // hasAsyncRet
+ gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur,
+ DebugInfo(), /* IL offset */
+ callTarget, /* ireg */
+ REG_NA, 0, 0, /* xreg, xmul, disp */
+ false /* isJump */
);
regMaskTP killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC);
@@ -3982,11 +3985,13 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize,
}
GetEmitter()->emitIns_Call(callType, compiler->eeFindHelper(helper), INDEBUG_LDISASM_COMMA(nullptr) addr, argSize,
- retSize, EA_UNKNOWN, gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur,
- gcInfo.gcRegByrefSetCur, DebugInfo(), /* IL offset */
- callTarget, /* ireg */
- REG_NA, 0, 0, /* xreg, xmul, disp */
- false /* isJump */
+ retSize, EA_UNKNOWN,
+ false, // hasAsyncRet
+ gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur,
+ DebugInfo(), /* IL offset */
+ callTarget, /* ireg */
+ REG_NA, 0, 0, /* xreg, xmul, disp */
+ false /* isJump */
);
regMaskTP killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper);
@@ -6117,6 +6122,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call)
nullptr, // addr
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ false, // hasAsyncRet
di,
target->GetRegNum(),
call->IsFastTailCall());
@@ -6165,6 +6171,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call)
nullptr, // addr
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ false, // hasAsyncRet
di,
targetAddrReg,
call->IsFastTailCall());
@@ -6208,6 +6215,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call)
addr,
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ false, // hasAsyncRet
di,
REG_NA,
call->IsFastTailCall());
@@ -6972,11 +6980,13 @@ inline void CodeGen::genJumpToThrowHlpBlk_la(
BasicBlock* skipLabel = genCreateTempLabel();
emit->emitIns_Call(callType, compiler->eeFindHelper(compiler->acdHelper(codeKind)),
- INDEBUG_LDISASM_COMMA(nullptr) addr, 0, EA_UNKNOWN, EA_UNKNOWN, gcInfo.gcVarPtrSetCur,
- gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur, DebugInfo(), /* IL offset */
- callTarget, /* ireg */
- REG_NA, 0, 0, /* xreg, xmul, disp */
- false /* isJump */
+ INDEBUG_LDISASM_COMMA(nullptr) addr, 0, EA_UNKNOWN, EA_UNKNOWN,
+ false, // hasAsyncRet
+ gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur,
+ DebugInfo(), /* IL offset */
+ callTarget, /* ireg */
+ REG_NA, 0, 0, /* xreg, xmul, disp */
+ false /* isJump */
);
regMaskTP killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)(compiler->acdHelper(codeKind)));
diff --git a/src/coreclr/jit/codegenriscv64.cpp b/src/coreclr/jit/codegenriscv64.cpp
index 45974ea7e1d950..ce63d04851a582 100644
--- a/src/coreclr/jit/codegenriscv64.cpp
+++ b/src/coreclr/jit/codegenriscv64.cpp
@@ -779,6 +779,7 @@ void CodeGen::genFnEpilog(BasicBlock* block)
0, // argSize
EA_UNKNOWN // retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(EA_UNKNOWN), // secondRetSize
+ false, // hasAsyncRet
gcInfo.gcVarPtrSetCur,
gcInfo.gcRegGCrefSetCur,
gcInfo.gcRegByrefSetCur,
@@ -2823,11 +2824,13 @@ void CodeGen::genCodeForReturnTrap(GenTreeOp* tree)
// TODO-RISCV64: can optimize further !!!
GetEmitter()->emitIns_Call(callType, compiler->eeFindHelper(CORINFO_HELP_STOP_FOR_GC),
- INDEBUG_LDISASM_COMMA(nullptr) addr, 0, EA_UNKNOWN, EA_UNKNOWN, gcInfo.gcVarPtrSetCur,
- gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur, DebugInfo(), /* IL offset */
- callTarget, /* ireg */
- REG_NA, 0, 0, /* xreg, xmul, disp */
- false /* isJump */
+ INDEBUG_LDISASM_COMMA(nullptr) addr, 0, EA_UNKNOWN, EA_UNKNOWN,
+ false, // hasAsyncRet
+ gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur,
+ DebugInfo(), /* IL offset */
+ callTarget, /* ireg */
+ REG_NA, 0, 0, /* xreg, xmul, disp */
+ false /* isJump */
);
regMaskTP killMask = compiler->compHelperCallKillSet(CORINFO_HELP_STOP_FOR_GC);
@@ -3844,11 +3847,13 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize,
}
GetEmitter()->emitIns_Call(callType, compiler->eeFindHelper(helper), INDEBUG_LDISASM_COMMA(nullptr) addr, argSize,
- retSize, EA_UNKNOWN, gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur,
- gcInfo.gcRegByrefSetCur, DebugInfo(), /* IL offset */
- callTarget, /* ireg */
- REG_NA, 0, 0, /* xreg, xmul, disp */
- false /* isJump */
+ retSize, EA_UNKNOWN,
+ false, // hasAsyncRet
+ gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur,
+ DebugInfo(), /* IL offset */
+ callTarget, /* ireg */
+ REG_NA, 0, 0, /* xreg, xmul, disp */
+ false /* isJump */
);
regMaskTP killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)helper);
@@ -6132,6 +6137,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call)
nullptr, // addr
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ false, // hasAsyncRet
di,
target->GetRegNum(),
call->IsFastTailCall());
@@ -6180,6 +6186,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call)
nullptr, // addr
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ false, // hasAsyncRet
di,
targetAddrReg,
call->IsFastTailCall());
@@ -6223,6 +6230,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call)
addr,
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ false, // hasAsyncRet
di,
REG_NA,
call->IsFastTailCall());
@@ -6809,11 +6817,13 @@ void CodeGen::genJumpToThrowHlpBlk_la(
BasicBlock* skipLabel = genCreateTempLabel();
emit->emitIns_Call(callType, compiler->eeFindHelper(compiler->acdHelper(codeKind)),
- INDEBUG_LDISASM_COMMA(nullptr) addr, 0, EA_UNKNOWN, EA_UNKNOWN, gcInfo.gcVarPtrSetCur,
- gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur, DebugInfo(), /* IL offset */
- callTarget, /* ireg */
- REG_NA, 0, 0, /* xreg, xmul, disp */
- false /* isJump */
+ INDEBUG_LDISASM_COMMA(nullptr) addr, 0, EA_UNKNOWN, EA_UNKNOWN,
+ false, // hasAsyncRet
+ gcInfo.gcVarPtrSetCur, gcInfo.gcRegGCrefSetCur, gcInfo.gcRegByrefSetCur,
+ DebugInfo(), /* IL offset */
+ callTarget, /* ireg */
+ REG_NA, 0, 0, /* xreg, xmul, disp */
+ false /* isJump */
);
regMaskTP killMask = compiler->compHelperCallKillSet((CorInfoHelpFunc)(compiler->acdHelper(codeKind)));
diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp
index 2bac5cd88093f7..b670cb3a50b397 100644
--- a/src/coreclr/jit/codegenxarch.cpp
+++ b/src/coreclr/jit/codegenxarch.cpp
@@ -2056,6 +2056,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
break;
#endif // SWIFT_SUPPORT
+ case GT_RETURN_SUSPEND:
+ genReturnSuspend(treeNode->AsUnOp());
+ break;
+
case GT_LEA:
// If we are here, it is the case where there is an LEA that cannot be folded into a parent instruction.
genLeaInstruction(treeNode->AsAddrMode());
@@ -2240,6 +2244,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
genConsumeReg(treeNode);
break;
+ case GT_ASYNC_CONTINUATION:
+ genCodeForAsyncContinuation(treeNode);
+ break;
+
#if defined(FEATURE_EH_WINDOWS_X86)
case GT_END_LFIN:
{
@@ -6361,6 +6369,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
}
#endif // DEBUG
+ bool hasAsyncRet = call->IsAsync();
CORINFO_METHOD_HANDLE methHnd;
GenTree* target = getCallTarget(call, &methHnd);
if (target != nullptr)
@@ -6397,6 +6406,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
argSizeForEmitter,
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
gcInfo.gcVarPtrSetCur,
gcInfo.gcRegGCrefSetCur,
gcInfo.gcRegByrefSetCur,
@@ -6426,6 +6436,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
X86_ARG(argSizeForEmitter),
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
di,
REG_NA,
call->IsFastTailCall());
@@ -6448,6 +6459,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
X86_ARG(argSizeForEmitter),
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
di,
call->IsFastTailCall());
// clang-format on
@@ -6476,6 +6488,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
X86_ARG(argSizeForEmitter),
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
di,
target->GetRegNum(),
call->IsFastTailCall());
@@ -6498,6 +6511,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
X86_ARG(argSizeForEmitter),
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
di,
target->GetRegNum(),
call->IsFastTailCall(),
@@ -6527,6 +6541,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
0,
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
gcInfo.gcVarPtrSetCur,
gcInfo.gcRegGCrefSetCur,
gcInfo.gcRegByrefSetCur,
@@ -6547,6 +6562,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
X86_ARG(argSizeForEmitter),
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
di,
REG_NA,
call->IsFastTailCall());
@@ -6587,6 +6603,7 @@ void CodeGen::genCallInstruction(GenTreeCall* call X86_ARG(target_ssize_t stackA
X86_ARG(argSizeForEmitter),
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize),
+ hasAsyncRet,
di,
REG_NA,
call->IsFastTailCall());
@@ -8961,6 +8978,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize,
argSize,
retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(EA_UNKNOWN),
+ /* hasAsyncRet */ false,
gcInfo.gcVarPtrSetCur,
gcInfo.gcRegGCrefSetCur,
gcInfo.gcRegByrefSetCur,
@@ -10645,10 +10663,10 @@ void CodeGen::genFnEpilog(BasicBlock* block)
if (frameSize > 0)
{
#ifdef TARGET_X86
- /* Add 'compiler->compLclFrameSize' to ESP */
- /* Use pop ECX to increment ESP by 4, unless compiler->compJmpOpUsed is true */
+ // Add 'compiler->compLclFrameSize' to ESP. Use "pop ECX" for that, except in cases
+ // where ECX may contain some state.
- if ((frameSize == TARGET_POINTER_SIZE) && !compiler->compJmpOpUsed)
+ if ((frameSize == TARGET_POINTER_SIZE) && !compiler->compJmpOpUsed && !compiler->compIsAsync())
{
inst_RV(INS_pop, REG_ECX, TYP_I_IMPL);
regSet.verifyRegUsed(REG_ECX);
@@ -10656,8 +10674,8 @@ void CodeGen::genFnEpilog(BasicBlock* block)
else
#endif // TARGET_X86
{
- /* Add 'compiler->compLclFrameSize' to ESP */
- /* Generate "add esp, " */
+ // Add 'compiler->compLclFrameSize' to ESP
+ // Generate "add esp, "
inst_RV_IV(INS_add, REG_SPBASE, frameSize, EA_PTRSIZE);
}
}
@@ -10735,7 +10753,8 @@ void CodeGen::genFnEpilog(BasicBlock* block)
// do nothing before popping the callee-saved registers
}
#ifdef TARGET_X86
- else if (compiler->compLclFrameSize == REGSIZE_BYTES)
+ else if ((compiler->compLclFrameSize == REGSIZE_BYTES) && !compiler->compJmpOpUsed &&
+ !compiler->compIsAsync())
{
// "pop ecx" will make ESP point to the callee-saved registers
inst_RV(INS_pop, REG_ECX, TYP_I_IMPL);
@@ -10896,6 +10915,7 @@ void CodeGen::genFnEpilog(BasicBlock* block)
0, // argSize
EA_UNKNOWN // retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(EA_UNKNOWN), // secondRetSize
+ /* hasAsyncRet */ false,
gcInfo.gcVarPtrSetCur,
gcInfo.gcRegGCrefSetCur,
gcInfo.gcRegByrefSetCur,
diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp
index 11bc8cd3868824..3d14e7cce64632 100644
--- a/src/coreclr/jit/compiler.cpp
+++ b/src/coreclr/jit/compiler.cpp
@@ -488,8 +488,9 @@ Compiler::Compiler(ArenaAllocator* arena,
info.compILCodeSize = methodInfo->ILCodeSize;
info.compILImportSize = 0;
- info.compHasNextCallRetAddr = false;
- info.compIsVarArgs = false;
+ info.compHasNextCallRetAddr = false;
+ info.compIsVarArgs = false;
+ info.compUsesAsyncContinuation = false;
}
//------------------------------------------------------------------------
@@ -3159,6 +3160,11 @@ void Compiler::compInitOptions(JitFlags* jitFlags)
{
printf("OPTIONS: Jit invoked for AOT\n");
}
+
+ if (compIsAsync())
+ {
+ printf("OPTIONS: compilation is an async2 state machine\n");
+ }
}
#endif
@@ -5006,6 +5012,11 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
}
#endif // TARGET_ARM
+ if (compIsAsync())
+ {
+ DoPhase(this, PHASE_ASYNC, &Compiler::TransformAsync);
+ }
+
// Assign registers to variables, etc.
// Create LinearScan before Lowering, so that Lowering can call LinearScan methods
diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h
index 891b33a169e46d..baaf90d3c1d4b2 100644
--- a/src/coreclr/jit/compiler.h
+++ b/src/coreclr/jit/compiler.h
@@ -683,6 +683,15 @@ class LclVarDsc
unsigned char lvAllDefsAreNoGc : 1; // For pinned locals: true if all defs of this local are no-gc
unsigned char lvStackAllocatedObject : 1; // Local is a stack allocated object (class, box, array, ...)
+ bool IsImplicitByRef()
+ {
+#if FEATURE_IMPLICIT_BYREFS
+ return lvIsImplicitByRef;
+#else
+ return false;
+#endif
+ }
+
// lvIsMultiRegArgOrRet()
// returns true if this is a multireg LclVar struct used in an argument context
// or if this is a multireg LclVar struct assigned from a multireg call
@@ -3060,7 +3069,7 @@ class Compiler
GenTreeCall* gtNewIndCallNode(GenTree* addr, var_types type, const DebugInfo& di = DebugInfo());
GenTreeCall* gtNewHelperCallNode(
- unsigned helper, var_types type, GenTree* arg1 = nullptr, GenTree* arg2 = nullptr, GenTree* arg3 = nullptr);
+ unsigned helper, var_types type, GenTree* arg1 = nullptr, GenTree* arg2 = nullptr, GenTree* arg3 = nullptr, GenTree* arg4 = nullptr);
GenTreeCall* gtNewRuntimeLookupHelperCallNode(CORINFO_RUNTIME_LOOKUP* pRuntimeLookup,
GenTree* ctxTree,
@@ -3955,6 +3964,11 @@ class Compiler
// However, if there is a "ldarga 0" or "starg 0" in the IL,
// we will redirect all "ldarg(a) 0" and "starg 0" to this temp.
+ // For struct instance functions with CORINFO_OPT_COPY_STRUCT_INSTANCE
+ // this is the local that has the copy of "this" to which accesses on
+ // "this" are redirected to
+ unsigned lvaThisCopyVar = BAD_VAR_NUM;
+
unsigned lvaInlineeReturnSpillTemp = BAD_VAR_NUM; // The temp to spill the non-VOID return expression
// in case there are multiple BBJ_RETURN blocks in the inlinee
// or if the inlinee has GC ref locals.
@@ -3983,6 +3997,9 @@ class Compiler
unsigned lvaSwiftErrorLocal;
#endif
+ // Variable representing async continuation argument passed.
+ unsigned lvaAsyncContinuationArg = BAD_VAR_NUM;
+
#if defined(DEBUG) && defined(TARGET_XARCH)
unsigned lvaReturnSpCheck = BAD_VAR_NUM; // Stores SP to confirm it is not corrupted on return.
@@ -4094,6 +4111,7 @@ class Compiler
void lvaInitUserArgs(unsigned* curVarNum, unsigned skipArgs, unsigned takeArgs);
void lvaInitGenericsCtxt(unsigned* curVarNum);
void lvaInitVarArgsHandle(unsigned* curVarNum);
+ void lvaInitAsyncContinuation(unsigned* curVarNum);
void lvaInitVarDsc(LclVarDsc* varDsc,
unsigned varNum,
@@ -4836,6 +4854,8 @@ class Compiler
bool impMatchIsInstBooleanConversion(const BYTE* codeAddr, const BYTE* codeEndp, int* consumed);
+ bool impMatchAwaitPattern(const BYTE * codeAddr, const BYTE * codeEndp, int* configVal);
+
GenTree* impCastClassOrIsInstToTree(
GenTree* op1, GenTree* op2, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass, bool* booleanCheck, IL_OFFSET ilOffset);
@@ -5513,6 +5533,8 @@ class Compiler
PhaseStatus placeLoopAlignInstructions();
#endif
+ PhaseStatus TransformAsync();
+
// This field keep the R2R helper call that would be inserted to trigger the constructor
// of the static class. It is set as nongc or gc static base if they are imported, so
// CSE can eliminate the repeated call, or the chepeast helper function that triggers it.
@@ -6413,6 +6435,7 @@ class Compiler
void fgObserveInlineConstants(OPCODE opcode, const FgStack& stack, bool isInlining);
void fgAdjustForAddressExposedOrWrittenThis();
+ void fgInitializeThisCopyVar();
unsigned fgStressBBProf()
{
@@ -6836,7 +6859,7 @@ class Compiler
void fgInvokeInlineeCompiler(GenTreeCall* call, InlineResult* result, InlineContext** createdContext);
void fgInsertInlineeBlocks(InlineInfo* pInlineInfo);
- void fgInsertInlineeArgument(const InlArgInfo& argInfo, BasicBlock* block, Statement** afterStmt, Statement** newStmt, const DebugInfo& callDI);
+ void fgInsertInlineeArgument(InlineInfo* pInlineInfo, const InlArgInfo& argInfo, BasicBlock* block, Statement** afterStmt, Statement** newStmt, const DebugInfo& callDI);
Statement* fgInlinePrependStatements(InlineInfo* inlineInfo);
void fgInlineAppendStatements(InlineInfo* inlineInfo, BasicBlock* block, Statement* stmt);
@@ -7171,6 +7194,7 @@ class Compiler
void optPrintCSEDataFlowSet(EXPSET_VALARG_TP cseDataFlowSet, bool includeBits = true);
EXPSET_TP cseCallKillsMask; // Computed once - A mask that is used to kill available CSEs at callsites
+ EXPSET_TP cseAsyncKillsMask; // Computed once - A mask that is used to kill available BYREF CSEs at async suspension points
static const size_t s_optCSEhashSizeInitial;
static const size_t s_optCSEhashGrowthFactor;
@@ -7259,6 +7283,7 @@ class Compiler
unsigned optValnumCSE_Index(GenTree* tree, Statement* stmt);
bool optValnumCSE_Locate(CSE_HeuristicCommon* heuristic);
void optValnumCSE_InitDataFlow();
+ void optValnumCSE_SetUpAsyncByrefKills();
void optValnumCSE_DataFlow();
void optValnumCSE_Availability();
void optValnumCSE_Heuristic(CSE_HeuristicCommon* heuristic);
@@ -10724,12 +10749,13 @@ class Compiler
// (2) the code is hot/cold split, and we issued less code than we expected
// in the cold section (the hot section will always be padded out to compTotalHotCodeSize).
- bool compIsStatic : 1; // Is the method static (no 'this' pointer)?
- bool compIsVarArgs : 1; // Does the method have varargs parameters?
- bool compInitMem : 1; // Is the CORINFO_OPT_INIT_LOCALS bit set in the method info options?
- bool compProfilerCallback : 1; // JIT inserted a profiler Enter callback
- bool compPublishStubParam : 1; // EAX captured in prolog will be available through an intrinsic
- bool compHasNextCallRetAddr : 1; // The NextCallReturnAddress intrinsic is used.
+ bool compIsStatic : 1; // Is the method static (no 'this' pointer)?
+ bool compIsVarArgs : 1; // Does the method have varargs parameters?
+ bool compInitMem : 1; // Is the CORINFO_OPT_INIT_LOCALS bit set in the method info options?
+ bool compProfilerCallback : 1; // JIT inserted a profiler Enter callback
+ bool compPublishStubParam : 1; // EAX captured in prolog will be available through an intrinsic
+ bool compHasNextCallRetAddr : 1; // The NextCallReturnAddress intrinsic is used.
+ bool compUsesAsyncContinuation : 1; // The Async2CallContinuation intrinsic is used.
var_types compRetType; // Return type of the method as declared in IL (including SIMD normalization)
var_types compRetNativeType; // Normalized return type as per target arch ABI
@@ -10862,6 +10888,16 @@ class Compiler
#endif // TARGET_AMD64
}
+ bool compIsAsync() const
+ {
+ return opts.jitFlags->IsSet(JitFlags::JIT_FLAG_RUNTIMEASYNCFUNCTION);
+ }
+
+ bool compIsStructMethodThatOperatesOnCopy() const
+ {
+ return (info.compMethodInfo->options & CORINFO_OPT_COPY_STRUCT_INSTANCE) != 0;
+ }
+
//------------------------------------------------------------------------
// compMethodReturnsMultiRegRetType: Does this method return a multi-reg value?
//
@@ -10886,6 +10922,13 @@ class Compiler
bool compObjectStackAllocation()
{
+ if (compIsAsync())
+ {
+ // Object stack allocation takes the address of locals around
+ // suspension points. Disable entirely for now.
+ return false;
+ }
+
return (JitConfig.JitObjectStackAllocation() != 0);
}
@@ -11845,6 +11888,7 @@ class GenTreeVisitor
// Leaf nodes
case GT_CATCH_ARG:
+ case GT_ASYNC_CONTINUATION:
case GT_LABEL:
case GT_FTN_ADDR:
case GT_RET_EXPR:
@@ -11921,6 +11965,7 @@ class GenTreeVisitor
case GT_RETURNTRAP:
case GT_FIELD_ADDR:
case GT_RETURN:
+ case GT_RETURN_SUSPEND:
case GT_RETFILT:
case GT_RUNTIMELOOKUP:
case GT_ARR_ADDR:
diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp
index a1c563f0ce0ea6..a8d19927838a4d 100644
--- a/src/coreclr/jit/compiler.hpp
+++ b/src/coreclr/jit/compiler.hpp
@@ -1552,7 +1552,7 @@ inline GenTree* Compiler::gtNewIconEmbFldHndNode(CORINFO_FIELD_HANDLE fldHnd)
// New CT_HELPER node
//
inline GenTreeCall* Compiler::gtNewHelperCallNode(
- unsigned helper, var_types type, GenTree* arg1, GenTree* arg2, GenTree* arg3)
+ unsigned helper, var_types type, GenTree* arg1, GenTree* arg2, GenTree* arg3, GenTree* arg4)
{
GenTreeCall* const result = gtNewCallNode(CT_HELPER, eeFindHelper(helper), type);
@@ -1571,6 +1571,12 @@ inline GenTreeCall* Compiler::gtNewHelperCallNode(
result->gtInlineObservation = InlineObservation::CALLSITE_IS_CALL_TO_HELPER;
#endif
+ if (arg4 != nullptr)
+ {
+ result->gtArgs.PushFront(this, NewCallArg::Primitive(arg4));
+ result->gtFlags |= arg4->gtFlags & GTF_ALL_EFFECT;
+ }
+
if (arg3 != nullptr)
{
result->gtArgs.PushFront(this, NewCallArg::Primitive(arg3));
@@ -2921,6 +2927,12 @@ inline unsigned Compiler::compMapILargNum(unsigned ILargNum)
assert(ILargNum < info.compLocalsCount); // compLocals count already adjusted.
}
+ if (ILargNum >= lvaAsyncContinuationArg)
+ {
+ ILargNum++;
+ assert(ILargNum < info.compLocalsCount); // compLocals count already adjusted.
+ }
+
if (ILargNum >= lvaVarargsHandleArg)
{
ILargNum++;
@@ -4374,6 +4386,7 @@ void GenTree::VisitOperands(TVisitor visitor)
case GT_LCL_FLD:
case GT_LCL_ADDR:
case GT_CATCH_ARG:
+ case GT_ASYNC_CONTINUATION:
case GT_LABEL:
case GT_FTN_ADDR:
case GT_RET_EXPR:
@@ -4454,6 +4467,7 @@ void GenTree::VisitOperands(TVisitor visitor)
case GT_RETURNTRAP:
case GT_KEEPALIVE:
case GT_INC_SATURATE:
+ case GT_RETURN_SUSPEND:
visitor(this->AsUnOp()->gtOp1);
return;
diff --git a/src/coreclr/jit/compmemkind.h b/src/coreclr/jit/compmemkind.h
index 8b3f84a0cf3a45..eb2c0dffc0ee6e 100644
--- a/src/coreclr/jit/compmemkind.h
+++ b/src/coreclr/jit/compmemkind.h
@@ -66,6 +66,7 @@ CompMemKindMacro(ZeroInit)
CompMemKindMacro(Pgo)
CompMemKindMacro(MaskConversionOpt)
CompMemKindMacro(TryRegionClone)
+CompMemKindMacro(Async)
CompMemKindMacro(RangeCheckCloning)
//clang-format on
diff --git a/src/coreclr/jit/compphases.h b/src/coreclr/jit/compphases.h
index 21915bf4a13a2f..2c816dc0ef9a43 100644
--- a/src/coreclr/jit/compphases.h
+++ b/src/coreclr/jit/compphases.h
@@ -119,6 +119,7 @@ CompPhaseNameMacro(PHASE_RATIONALIZE, "Rationalize IR",
CompPhaseNameMacro(PHASE_REPAIR_PROFILE_POST_MORPH, "Repair profile post-morph", false, -1, false)
CompPhaseNameMacro(PHASE_REPAIR_PROFILE_PRE_LAYOUT, "Repair profile pre-layout", false, -1, false)
+CompPhaseNameMacro(PHASE_ASYNC, "Transform async", false, -1, true)
CompPhaseNameMacro(PHASE_LCLVARLIVENESS, "Local var liveness", true, -1, false)
CompPhaseNameMacro(PHASE_LCLVARLIVENESS_INIT, "Local var liveness init", false, PHASE_LCLVARLIVENESS, false)
CompPhaseNameMacro(PHASE_LCLVARLIVENESS_PERBLOCK, "Per block local var liveness", false, PHASE_LCLVARLIVENESS, false)
diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp
index e03ec89ac87777..f2c248c90e7d6b 100644
--- a/src/coreclr/jit/emit.cpp
+++ b/src/coreclr/jit/emit.cpp
@@ -3564,7 +3564,8 @@ emitter::instrDesc* emitter::emitNewInstrCallInd(int argCnt,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
emitAttr retSizeIn
- MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize))
+ MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet)
{
emitAttr retSize = (retSizeIn != EA_UNKNOWN) ? retSizeIn : EA_PTRSIZE;
@@ -3588,7 +3589,8 @@ emitter::instrDesc* emitter::emitNewInstrCallInd(int argCnt,
(argCnt > ID_MAX_SMALL_CNS) || // too many args
(argCnt < 0) // caller pops arguments
// There is a second ref/byref return register.
- MULTIREG_HAS_SECOND_GC_RET_ONLY(|| EA_IS_GCREF_OR_BYREF(secondRetSize)))
+ MULTIREG_HAS_SECOND_GC_RET_ONLY(|| EA_IS_GCREF_OR_BYREF(secondRetSize)) ||
+ hasAsyncRet)
{
instrDescCGCA* id;
@@ -3605,6 +3607,7 @@ emitter::instrDesc* emitter::emitNewInstrCallInd(int argCnt,
#if MULTIREG_HAS_SECOND_GC_RET
emitSetSecondRetRegGCType(id, secondRetSize);
#endif // MULTIREG_HAS_SECOND_GC_RET
+ id->hasAsyncContinuationRet(hasAsyncRet);
return id;
}
@@ -3648,7 +3651,8 @@ emitter::instrDesc* emitter::emitNewInstrCallDir(int argCnt,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
emitAttr retSizeIn
- MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize))
+ MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet)
{
emitAttr retSize = (retSizeIn != EA_UNKNOWN) ? retSizeIn : EA_PTRSIZE;
@@ -3668,7 +3672,8 @@ emitter::instrDesc* emitter::emitNewInstrCallDir(int argCnt,
(argCnt > ID_MAX_SMALL_CNS) || // too many args
(argCnt < 0) // caller pops arguments
// There is a second ref/byref return register.
- MULTIREG_HAS_SECOND_GC_RET_ONLY(|| EA_IS_GCREF_OR_BYREF(secondRetSize)))
+ MULTIREG_HAS_SECOND_GC_RET_ONLY(|| EA_IS_GCREF_OR_BYREF(secondRetSize)) ||
+ hasAsyncRet)
{
instrDescCGCA* id = emitAllocInstrCGCA(retSize);
@@ -3685,6 +3690,7 @@ emitter::instrDesc* emitter::emitNewInstrCallDir(int argCnt,
#if MULTIREG_HAS_SECOND_GC_RET
emitSetSecondRetRegGCType(id, secondRetSize);
#endif // MULTIREG_HAS_SECOND_GC_RET
+ id->hasAsyncContinuationRet(hasAsyncRet);
return id;
}
diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h
index ef1fd2f701fc15..780a9f05005bcc 100644
--- a/src/coreclr/jit/emit.h
+++ b/src/coreclr/jit/emit.h
@@ -2234,8 +2234,19 @@ class emitter
{
_idcSecondRetRegGCType = gctype;
}
+#endif
+
+ bool hasAsyncContinuationRet() const
+ {
+ return _hasAsyncContinuationRet;
+ }
+ void hasAsyncContinuationRet(bool value)
+ {
+ _hasAsyncContinuationRet = value;
+ }
private:
+#if MULTIREG_HAS_SECOND_GC_RET
// This member stores the GC-ness of the second register in a 2 register returned struct on System V.
// It is added to the call struct since it is not needed by the base instrDesc struct, which keeps GC-ness
// of the first register for the instCall nodes.
@@ -2245,6 +2256,7 @@ class emitter
// The base struct's member keeping the GC-ness of the first return register is _idGCref.
GCtype _idcSecondRetRegGCType : 2; // ... GC type for the second return register.
#endif // MULTIREG_HAS_SECOND_GC_RET
+ bool _hasAsyncContinuationRet : 1;
};
// TODO-Cleanup: Uses of stack-allocated instrDescs should be refactored to be unnecessary.
diff --git a/src/coreclr/jit/emitarm.cpp b/src/coreclr/jit/emitarm.cpp
index 902399f58e4fe7..28c708e91a28a1 100644
--- a/src/coreclr/jit/emitarm.cpp
+++ b/src/coreclr/jit/emitarm.cpp
@@ -4668,6 +4668,7 @@ void emitter::emitIns_Call(EmitCallType callType,
void* addr,
int argSize,
emitAttr retSize,
+ bool hasAsyncRet,
VARSET_VALARG_TP ptrVars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
@@ -4740,7 +4741,7 @@ void emitter::emitIns_Call(EmitCallType callType,
{
/* Indirect call, virtual calls */
- id = emitNewInstrCallInd(argCnt, 0 /* disp */, ptrVars, gcrefRegs, byrefRegs, retSize);
+ id = emitNewInstrCallInd(argCnt, 0 /* disp */, ptrVars, gcrefRegs, byrefRegs, retSize, hasAsyncRet);
}
else
{
@@ -4749,7 +4750,7 @@ void emitter::emitIns_Call(EmitCallType callType,
assert(callType == EC_FUNC_TOKEN);
- id = emitNewInstrCallDir(argCnt, ptrVars, gcrefRegs, byrefRegs, retSize);
+ id = emitNewInstrCallDir(argCnt, ptrVars, gcrefRegs, byrefRegs, retSize, hasAsyncRet);
}
/* Update the emitter's live GC ref sets */
diff --git a/src/coreclr/jit/emitarm.h b/src/coreclr/jit/emitarm.h
index 20c7b851499cf9..088c6ba13e2473 100644
--- a/src/coreclr/jit/emitarm.h
+++ b/src/coreclr/jit/emitarm.h
@@ -65,10 +65,15 @@ void emitDispInsHelp(instrDesc* id,
private:
instrDesc* emitNewInstrCallDir(
- int argCnt, VARSET_VALARG_TP GCvars, regMaskTP gcrefRegs, regMaskTP byrefRegs, emitAttr retSize);
+ int argCnt, VARSET_VALARG_TP GCvars, regMaskTP gcrefRegs, regMaskTP byrefRegs, emitAttr retSize, bool hasAsyncRet);
-instrDesc* emitNewInstrCallInd(
- int argCnt, ssize_t disp, VARSET_VALARG_TP GCvars, regMaskTP gcrefRegs, regMaskTP byrefRegs, emitAttr retSize);
+instrDesc* emitNewInstrCallInd(int argCnt,
+ ssize_t disp,
+ VARSET_VALARG_TP GCvars,
+ regMaskTP gcrefRegs,
+ regMaskTP byrefRegs,
+ emitAttr retSize,
+ bool hasAsyncRet);
/************************************************************************/
/* Private helpers for instruction output */
@@ -327,6 +332,7 @@ void emitIns_Call(EmitCallType callType,
void* addr,
int argSize,
emitAttr retSize,
+ bool hasAsyncRet,
VARSET_VALARG_TP ptrVars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp
index ea9a9b53b2797e..13a681f80bde66 100644
--- a/src/coreclr/jit/emitarm64.cpp
+++ b/src/coreclr/jit/emitarm64.cpp
@@ -9106,6 +9106,7 @@ void emitter::emitIns_Call(EmitCallType callType,
ssize_t argSize,
emitAttr retSize,
emitAttr secondRetSize,
+ bool hasAsyncRet,
VARSET_VALARG_TP ptrVars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
@@ -9170,7 +9171,8 @@ void emitter::emitIns_Call(EmitCallType callType,
{
/* Indirect call, virtual calls */
- id = emitNewInstrCallInd(argCnt, 0 /* disp */, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize);
+ id = emitNewInstrCallInd(argCnt, 0 /* disp */, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize,
+ hasAsyncRet);
}
else
{
@@ -9179,7 +9181,7 @@ void emitter::emitIns_Call(EmitCallType callType,
assert(callType == EC_FUNC_TOKEN);
- id = emitNewInstrCallDir(argCnt, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize);
+ id = emitNewInstrCallDir(argCnt, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize, hasAsyncRet);
}
/* Update the emitter's live GC ref sets */
diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h
index 8e2ed80c6cdf28..1e5da60038f952 100644
--- a/src/coreclr/jit/emitarm64.h
+++ b/src/coreclr/jit/emitarm64.h
@@ -98,7 +98,8 @@ instrDesc* emitNewInstrCallDir(int argCnt,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
emitAttr retSize,
- emitAttr secondRetSize);
+ emitAttr secondRetSize,
+ bool hasAsyncRet);
instrDesc* emitNewInstrCallInd(int argCnt,
ssize_t disp,
@@ -106,7 +107,8 @@ instrDesc* emitNewInstrCallInd(int argCnt,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
emitAttr retSize,
- emitAttr secondRetSize);
+ emitAttr secondRetSize,
+ bool hasAsyncRet);
/************************************************************************/
/* enum to allow instruction optimisation to specify register order */
@@ -1748,6 +1750,7 @@ void emitIns_Call(EmitCallType callType,
ssize_t argSize,
emitAttr retSize,
emitAttr secondRetSize,
+ bool hasAsyncRet,
VARSET_VALARG_TP ptrVars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
diff --git a/src/coreclr/jit/emitloongarch64.cpp b/src/coreclr/jit/emitloongarch64.cpp
index 1202a8e31c0fa6..bbe810a99a66d4 100644
--- a/src/coreclr/jit/emitloongarch64.cpp
+++ b/src/coreclr/jit/emitloongarch64.cpp
@@ -2381,6 +2381,7 @@ void emitter::emitIns_Call(EmitCallType callType,
void* addr,
ssize_t argSize,
emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet,
VARSET_VALARG_TP ptrVars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
@@ -2448,7 +2449,7 @@ void emitter::emitIns_Call(EmitCallType callType,
assert(callType == EC_INDIR_R);
- id = emitNewInstrCallInd(argCnt, disp, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize);
+ id = emitNewInstrCallInd(argCnt, disp, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize, hasAsyncRet);
}
else
{
@@ -2457,7 +2458,7 @@ void emitter::emitIns_Call(EmitCallType callType,
assert(callType == EC_FUNC_TOKEN);
- id = emitNewInstrCallDir(argCnt, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize);
+ id = emitNewInstrCallDir(argCnt, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize, hasAsyncRet);
}
/* Update the emitter's live GC ref sets */
diff --git a/src/coreclr/jit/emitloongarch64.h b/src/coreclr/jit/emitloongarch64.h
index bfaf243edfabe6..c088c275172665 100644
--- a/src/coreclr/jit/emitloongarch64.h
+++ b/src/coreclr/jit/emitloongarch64.h
@@ -117,14 +117,16 @@ instrDesc* emitNewInstrCallDir(int argCnt,
VARSET_VALARG_TP GCvars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
- emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize));
+ emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet);
instrDesc* emitNewInstrCallInd(int argCnt,
ssize_t disp,
VARSET_VALARG_TP GCvars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
- emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize));
+ emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet);
/************************************************************************/
/* Private helpers for instruction output */
@@ -311,6 +313,7 @@ void emitIns_Call(EmitCallType callType,
void* addr,
ssize_t argSize,
emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet,
VARSET_VALARG_TP ptrVars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
diff --git a/src/coreclr/jit/emitriscv64.cpp b/src/coreclr/jit/emitriscv64.cpp
index c2911921f9b492..87179bb2575e62 100644
--- a/src/coreclr/jit/emitriscv64.cpp
+++ b/src/coreclr/jit/emitriscv64.cpp
@@ -1616,6 +1616,7 @@ void emitter::emitIns_Call(EmitCallType callType,
void* addr,
ssize_t argSize,
emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet,
VARSET_VALARG_TP ptrVars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
@@ -1683,7 +1684,7 @@ void emitter::emitIns_Call(EmitCallType callType,
assert(callType == EC_INDIR_R);
- id = emitNewInstrCallInd(argCnt, disp, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize);
+ id = emitNewInstrCallInd(argCnt, disp, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize, hasAsyncRet);
}
else
{
@@ -1692,7 +1693,7 @@ void emitter::emitIns_Call(EmitCallType callType,
assert(callType == EC_FUNC_TOKEN);
- id = emitNewInstrCallDir(argCnt, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize);
+ id = emitNewInstrCallDir(argCnt, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize, hasAsyncRet);
}
/* Update the emitter's live GC ref sets */
diff --git a/src/coreclr/jit/emitriscv64.h b/src/coreclr/jit/emitriscv64.h
index 8fb130d42f239f..61eaf0eb30025d 100644
--- a/src/coreclr/jit/emitriscv64.h
+++ b/src/coreclr/jit/emitriscv64.h
@@ -42,14 +42,16 @@ instrDesc* emitNewInstrCallDir(int argCnt,
VARSET_VALARG_TP GCvars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
- emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize));
+ emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet);
instrDesc* emitNewInstrCallInd(int argCnt,
ssize_t disp,
VARSET_VALARG_TP GCvars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
- emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize));
+ emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet);
/************************************************************************/
/* Private helpers for instruction output */
@@ -365,6 +367,7 @@ void emitIns_Call(EmitCallType callType,
void* addr,
ssize_t argSize,
emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet,
VARSET_VALARG_TP ptrVars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp
index 84b97b1fd73833..1e435e652cf008 100644
--- a/src/coreclr/jit/emitxarch.cpp
+++ b/src/coreclr/jit/emitxarch.cpp
@@ -11102,6 +11102,7 @@ void emitter::emitIns_Call(EmitCallType callType,
ssize_t argSize,
emitAttr retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet,
VARSET_VALARG_TP ptrVars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
@@ -11179,7 +11180,7 @@ void emitter::emitIns_Call(EmitCallType callType,
/* Indirect call, virtual calls */
id = emitNewInstrCallInd(argCnt, disp, ptrVars, gcrefRegs, byrefRegs,
- retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize));
+ retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), hasAsyncRet);
}
else
{
@@ -11189,7 +11190,7 @@ void emitter::emitIns_Call(EmitCallType callType,
assert(callType == EC_FUNC_TOKEN || callType == EC_FUNC_TOKEN_INDIR);
id = emitNewInstrCallDir(argCnt, ptrVars, gcrefRegs, byrefRegs,
- retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize));
+ retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(secondRetSize), hasAsyncRet);
}
/* Update the emitter's live GC ref sets */
@@ -18585,11 +18586,11 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp)
byrefRegs |= RBM_EAX;
}
-#ifdef UNIX_AMD64_ABI
// If is a multi-register return method is called, mark RDX appropriately (for System V AMD64).
if (id->idIsLargeCall())
{
instrDescCGCA* idCall = (instrDescCGCA*)id;
+#ifdef UNIX_AMD64_ABI
if (idCall->idSecondGCref() == GCT_GCREF)
{
gcrefRegs |= RBM_RDX;
@@ -18598,8 +18599,12 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp)
{
byrefRegs |= RBM_RDX;
}
- }
#endif // UNIX_AMD64_ABI
+ if (idCall->hasAsyncContinuationRet())
+ {
+ gcrefRegs |= RBM_ASYNC_CONTINUATION_RET;
+ }
+ }
// If the GC register set has changed, report the new set
if (gcrefRegs != emitThisGCrefRegs)
diff --git a/src/coreclr/jit/emitxarch.h b/src/coreclr/jit/emitxarch.h
index d7d9d6bfd31ac0..3e21324f61b083 100644
--- a/src/coreclr/jit/emitxarch.h
+++ b/src/coreclr/jit/emitxarch.h
@@ -756,14 +756,16 @@ instrDesc* emitNewInstrCallDir(int argCnt,
VARSET_VALARG_TP GCvars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
- emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize));
+ emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet);
instrDesc* emitNewInstrCallInd(int argCnt,
ssize_t disp,
VARSET_VALARG_TP GCvars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
- emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize));
+ emitAttr retSize MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet);
void emitGetInsCns(const instrDesc* id, CnsVal* cv) const;
ssize_t emitGetInsAmdCns(const instrDesc* id, CnsVal* cv) const;
@@ -1245,6 +1247,7 @@ void emitIns_Call(EmitCallType callType,
ssize_t argSize,
emitAttr retSize
MULTIREG_HAS_SECOND_GC_RET_ONLY_ARG(emitAttr secondRetSize),
+ bool hasAsyncRet,
VARSET_VALARG_TP ptrVars,
regMaskTP gcrefRegs,
regMaskTP byrefRegs,
diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp
index 9905526766a9cf..cf0ad613796281 100644
--- a/src/coreclr/jit/fgbasic.cpp
+++ b/src/coreclr/jit/fgbasic.cpp
@@ -2541,6 +2541,11 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
fgAdjustForAddressExposedOrWrittenThis();
}
+ if (compIsStructMethodThatOperatesOnCopy())
+ {
+ fgInitializeThisCopyVar();
+ }
+
// Now that we've seen the IL, set lvSingleDef for root method
// locals.
//
@@ -2578,7 +2583,7 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
// change. The original this (info.compThisArg) then remains
// unmodified in the method. fgAddInternal is responsible for
// adding the code to copy the initial this into the temp.
-
+//
void Compiler::fgAdjustForAddressExposedOrWrittenThis()
{
LclVarDsc* thisVarDsc = lvaGetDesc(info.compThisArg);
@@ -2611,6 +2616,21 @@ void Compiler::fgAdjustForAddressExposedOrWrittenThis()
}
}
+//------------------------------------------------------------------------
+// fgInitializeThisCopyVar:
+// Initialize the local used to copy the "this" instance to for struct
+// methods with CORINFO_OPT_COPY_STRUCT_INSTANCE set.
+//
+void Compiler::fgInitializeThisCopyVar()
+{
+ assert(lvaThisCopyVar == BAD_VAR_NUM);
+ lvaThisCopyVar = lvaGrabTemp(false DEBUGARG("Copy of 'this'"));
+ lvaSetStruct(lvaThisCopyVar, info.compClassHnd, false);
+
+ LclVarDsc* lclDsc = lvaGetDesc(lvaThisCopyVar);
+ lclDsc->lvHasLdAddrOp = 1;
+}
+
//------------------------------------------------------------------------
// fgObserveInlineConstants: look for operations that might get optimized
// if this method were to be inlined, and report these to the inliner.
diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp
index 5410d946e97840..8ad1b378a1f195 100644
--- a/src/coreclr/jit/fgdiagnostic.cpp
+++ b/src/coreclr/jit/fgdiagnostic.cpp
@@ -2706,6 +2706,14 @@ bool BBPredsChecker::CheckEhTryDsc(BasicBlock* block, BasicBlock* blockPred, EHb
return true;
}
+ // Async resumptions are allowed to jump into try blocks at any point. They
+ // are introduced late enough that the invariant of single entry is no
+ // longer necessary.
+ if (blockPred->HasFlag(BBF_ASYNC_RESUMPTION))
+ {
+ return true;
+ }
+
printf("Jump into the middle of try region: " FMT_BB " branches to " FMT_BB "\n", blockPred->bbNum, block->bbNum);
assert(!"Jump into middle of try region");
return false;
@@ -3102,7 +3110,8 @@ void Compiler::fgDebugCheckBBlist(bool checkBBNum /* = false */, bool checkBBRef
// A branch or fall-through to a BBJ_CALLFINALLY block must come from the `try` region associated
// with the finally block the BBJ_CALLFINALLY is targeting. There is one special case: if the
// BBJ_CALLFINALLY is the first block of a `try`, then its predecessor can be outside the `try`:
- // either a branch or fall-through to the first block.
+ // either a branch or fall-through to the first block. Similarly internal resumption blocks for
+ // async2 are allowed to do this as they are introduced late enough that we no longer need the invariant.
//
// Note that this IR condition is a choice. It naturally occurs when importing EH constructs.
// This condition prevents flow optimizations from skipping blocks in a `try` and branching
@@ -3140,7 +3149,7 @@ void Compiler::fgDebugCheckBBlist(bool checkBBNum /* = false */, bool checkBBRef
}
else
{
- assert(bbInTryRegions(finallyIndex, block));
+ assert(bbInTryRegions(finallyIndex, block) || block->HasFlag(BBF_ASYNC_RESUMPTION));
}
}
}
@@ -3357,6 +3366,7 @@ void Compiler::fgDebugCheckFlags(GenTree* tree, BasicBlock* block)
break;
case GT_CATCH_ARG:
+ case GT_ASYNC_CONTINUATION:
expectedFlags |= GTF_ORDER_SIDEEFF;
break;
@@ -3629,6 +3639,10 @@ void Compiler::fgDebugCheckNodeLinks(BasicBlock* block, Statement* stmt)
// The root of the tree should have GTF_ORDER_SIDEEFF set
noway_assert(stmt->GetRootNode()->gtFlags & GTF_ORDER_SIDEEFF);
}
+ else if (tree->OperIs(GT_ASYNC_CONTINUATION))
+ {
+ assert(tree->gtFlags & GTF_ORDER_SIDEEFF);
+ }
}
if (tree->OperIsUnary() && tree->AsOp()->gtOp1)
diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp
index e6be97f1d38e74..de1ad80ba38c06 100644
--- a/src/coreclr/jit/fginline.cpp
+++ b/src/coreclr/jit/fginline.cpp
@@ -1072,6 +1072,14 @@ void Compiler::fgMorphCallInlineHelper(GenTreeCall* call, InlineResult* result,
return;
}
+ if (call->IsAsync() && info.compUsesAsyncContinuation)
+ {
+ // Currently not supported. Could provide a nice perf benefit for
+ // async1 -> async2 thunks if we supported it.
+ result->NoteFatal(InlineObservation::CALLER_ASYNC2_USED_CONTINUATION);
+ return;
+ }
+
// impMarkInlineCandidate() is expected not to mark tail prefixed calls
// and recursive tail calls as inline candidates.
noway_assert(!call->IsTailPrefixedCall());
@@ -1994,8 +2002,12 @@ void Compiler::fgInsertInlineeBlocks(InlineInfo* pInlineInfo)
// newStmt - updated with the new statement
// callDI - debug info for the call
//
-void Compiler::fgInsertInlineeArgument(
- const InlArgInfo& argInfo, BasicBlock* block, Statement** afterStmt, Statement** newStmt, const DebugInfo& callDI)
+void Compiler::fgInsertInlineeArgument(InlineInfo* inlineInfo,
+ const InlArgInfo& argInfo,
+ BasicBlock* block,
+ Statement** afterStmt,
+ Statement** newStmt,
+ const DebugInfo& callDI)
{
const bool argIsSingleDef = !argInfo.argHasLdargaOp && !argInfo.argHasStargOp;
CallArg* arg = argInfo.arg;
@@ -2029,6 +2041,24 @@ void Compiler::fgInsertInlineeArgument(
argSingleUseNode->ReplaceWith(argNode, this);
return;
}
+ else if (argInfo.argIsByRefToCopy)
+ {
+ ClassLayout* layout = typGetObjLayout(inlineInfo->inlineCandidateInfo->clsHandle);
+ unsigned copyOfThisLcl = lvaGrabTemp(false DEBUGARG("Copy of inlinee struct instance"));
+ lvaSetStruct(copyOfThisLcl, layout, false);
+ GenTree* copyBlock = gtNewStoreLclVarNode(copyOfThisLcl, gtNewBlkIndir(layout, argNode));
+ *newStmt = gtNewStmt(copyBlock, callDI);
+ fgInsertStmtAfter(block, *afterStmt, *newStmt);
+ DISPSTMT(*newStmt);
+ *afterStmt = *newStmt;
+
+ GenTree* storeTmp =
+ gtNewTempStore(argInfo.argTmpNum, gtNewLclVarAddrNode(copyOfThisLcl, argNode->TypeGet()));
+ *newStmt = gtNewStmt(storeTmp, callDI);
+ fgInsertStmtAfter(block, *afterStmt, *newStmt);
+ DISPSTMT(*newStmt);
+ *afterStmt = *newStmt;
+ }
else
{
// We're going to assign the argument value to the temp we use for it in the inline body.
@@ -2051,6 +2081,7 @@ void Compiler::fgInsertInlineeArgument(
noway_assert(!argInfo.argIsUsed || argInfo.argIsInvariant || argInfo.argIsLclVar);
noway_assert((argInfo.argIsLclVar == 0) ==
(argNode->gtOper != GT_LCL_VAR || (argNode->gtFlags & GTF_GLOB_REF)));
+ noway_assert(!argInfo.argIsByRefToCopy);
// If the argument has side effects, append it
if (argInfo.argHasSideEff)
@@ -2229,6 +2260,7 @@ Statement* Compiler::fgInlinePrependStatements(InlineInfo* inlineInfo)
switch (arg.GetWellKnownArg())
{
case WellKnownArg::RetBuffer:
+ case WellKnownArg::AsyncContinuation:
continue;
case WellKnownArg::InstParam:
argInfo = inlineInfo->inlInstParamArgInfo;
@@ -2240,7 +2272,7 @@ Statement* Compiler::fgInlinePrependStatements(InlineInfo* inlineInfo)
}
assert(argInfo != nullptr);
- fgInsertInlineeArgument(*argInfo, block, &afterStmt, &newStmt, callDI);
+ fgInsertInlineeArgument(inlineInfo, *argInfo, block, &afterStmt, &newStmt, callDI);
}
// Add the CCTOR check if asked for.
diff --git a/src/coreclr/jit/fgstmt.cpp b/src/coreclr/jit/fgstmt.cpp
index 85809339965ff0..f5ab387e262416 100644
--- a/src/coreclr/jit/fgstmt.cpp
+++ b/src/coreclr/jit/fgstmt.cpp
@@ -539,6 +539,7 @@ inline bool OperIsControlFlow(genTreeOps oper)
case GT_RETURN:
case GT_RETFILT:
case GT_SWIFT_ERROR_RET:
+ case GT_RETURN_SUSPEND:
#if defined(FEATURE_EH_WINDOWS_X86)
case GT_END_LFIN:
#endif // FEATURE_EH_WINDOWS_X86
diff --git a/src/coreclr/jit/flowgraph.cpp b/src/coreclr/jit/flowgraph.cpp
index 539a8ca7c7c4b6..a8ea2f5c471ab6 100644
--- a/src/coreclr/jit/flowgraph.cpp
+++ b/src/coreclr/jit/flowgraph.cpp
@@ -2394,6 +2394,19 @@ PhaseStatus Compiler::fgAddInternal()
madeChanges = true;
}
+
+ if (lvaThisCopyVar != BAD_VAR_NUM)
+ {
+ ClassLayout* layout = lvaGetDesc(lvaThisCopyVar)->GetLayout();
+ GenTree* addr = gtNewLclVarNode(info.compThisArg);
+ GenTree* store = gtNewStoreLclVarNode(lvaThisCopyVar, gtNewBlkIndir(layout, addr));
+ Statement* stmt = fgNewStmtAtBeg(fgFirstBB, store);
+
+ JITDUMP("\nCopy \"this\" to V%02u for struct instance method operating on copy\n", lvaThisCopyVar);
+ DISPSTMT(stmt);
+
+ madeChanges = true;
+ }
}
// Merge return points if required or beneficial
@@ -2449,7 +2462,18 @@ PhaseStatus Compiler::fgAddInternal()
}
else
{
- merger.SetMaxReturns(MergedReturns::ReturnCountHardLimit);
+ unsigned limit = MergedReturns::ReturnCountHardLimit;
+#ifdef JIT32_GCENCODER
+ // For the jit32 GC encoder the limit is an actual hard limit. In
+ // async2 functions we will be introducing another return during
+ // the async2 transformation, so make sure there's a free epilog
+ // for it.
+ if (compIsAsync())
+ {
+ limit--;
+ }
+#endif
+ merger.SetMaxReturns(limit);
}
}
diff --git a/src/coreclr/jit/forwardsub.cpp b/src/coreclr/jit/forwardsub.cpp
index c54e53d27bde49..4f004ed800c935 100644
--- a/src/coreclr/jit/forwardsub.cpp
+++ b/src/coreclr/jit/forwardsub.cpp
@@ -502,7 +502,7 @@ bool Compiler::fgForwardSubStatement(Statement* stmt)
// Can't substitute GT_LCLHEAP.
//
// Don't substitute a no return call (trips up morph in some cases).
- if (fwdSubNode->OperIs(GT_CATCH_ARG, GT_LCLHEAP))
+ if (fwdSubNode->OperIs(GT_CATCH_ARG, GT_LCLHEAP, GT_ASYNC_CONTINUATION))
{
JITDUMP(" tree to sub is catch arg, or lcl heap\n");
return false;
diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp
index 6a062b02f2b12d..5658494e9510f5 100644
--- a/src/coreclr/jit/gentree.cpp
+++ b/src/coreclr/jit/gentree.cpp
@@ -1241,6 +1241,7 @@ CallArgs::CallArgs()
#endif
, m_hasThisPointer(false)
, m_hasRetBuffer(false)
+ , m_hasAsyncContinuation(false)
, m_isVarArgs(false)
, m_abiInformationDetermined(false)
, m_hasAddedFinalArgs(false)
@@ -1519,6 +1520,9 @@ void CallArgs::AddedWellKnownArg(WellKnownArg arg)
case WellKnownArg::RetBuffer:
m_hasRetBuffer = true;
break;
+ case WellKnownArg::AsyncContinuation:
+ m_hasAsyncContinuation = true;
+ break;
default:
break;
}
@@ -1542,6 +1546,10 @@ void CallArgs::RemovedWellKnownArg(WellKnownArg arg)
assert(FindWellKnownArg(arg) == nullptr);
m_hasRetBuffer = false;
break;
+ case WellKnownArg::AsyncContinuation:
+ assert(FindWellKnownArg(arg) == nullptr);
+ m_hasAsyncContinuation = false;
+ break;
default:
break;
}
@@ -2247,6 +2255,37 @@ bool GenTreeCall::HasSideEffects(Compiler* compiler, bool ignoreExceptions, bool
(!helperProperties.IsAllocator(helper) || ((gtCallMoreFlags & GTF_CALL_M_ALLOC_SIDE_EFFECTS) != 0));
}
+//-------------------------------------------------------------------------
+// IsAsync2: Whether or not this call is to an async2 function.
+//
+// Return Value:
+// True if so.
+//
+// Remarks:
+// async2 involves passing an async continuation as a separate argument and
+// returning an async continuation in REG_ASYNC_CONTINUATION_RET.
+//
+// The async continuation is usually JIT added
+// (WellKnownArg::AsyncContinuation). This is the case for an async2 method
+// calling another async2 method by normal means. However, the VM also
+// creates stubs that call async2 runtimes through calli where the async
+// continuations are passed explicitly. See
+// CEEJitInfo::getAsyncResumptionStub and
+// MethodDesc::EmitJitStateMachineBasedRuntimeAsyncThunk for examples. In
+// those cases the JIT does not know (and does not need to know) which arg is
+// the async continuation.
+//
+// The VM also uses the StubHelpers.Async2CallContinuation() intrinsic in the
+// stubs discussed above. The JIT must take care in those cases to still mark
+// the preceding call as an async call; this is required for correct LSRA
+// behavior and GC reporting around the returned async continuation. This is
+// currently done in lowering; see LowerAsyncContinuation().
+//
+bool GenTreeCall::IsAsync() const
+{
+ return (gtCallMoreFlags & GTF_CALL_M_ASYNC) != 0;
+}
+
//-------------------------------------------------------------------------
// HasNonStandardAddedArgs: Return true if the method has non-standard args added to the call
// argument list during argument morphing (fgMorphArgs), e.g., passed in R10 or R11 on AMD64.
@@ -6553,6 +6592,7 @@ bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse)
case GT_LCL_FLD:
case GT_LCL_ADDR:
case GT_CATCH_ARG:
+ case GT_ASYNC_CONTINUATION:
case GT_LABEL:
case GT_FTN_ADDR:
case GT_RET_EXPR:
@@ -6618,6 +6658,7 @@ bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse)
case GT_RETURNTRAP:
case GT_RETURN:
case GT_RETFILT:
+ case GT_RETURN_SUSPEND:
case GT_BSWAP:
case GT_BSWAP16:
case GT_KEEPALIVE:
@@ -6917,6 +6958,8 @@ bool GenTree::OperRequiresCallFlag(Compiler* comp) const
case GT_GCPOLL:
case GT_KEEPALIVE:
+ case GT_ASYNC_CONTINUATION:
+ case GT_RETURN_SUSPEND:
return true;
case GT_SWIFT_ERROR:
@@ -7246,6 +7289,8 @@ bool GenTree::OperRequiresGlobRefFlag(Compiler* comp) const
case GT_CMPXCHG:
case GT_MEMORYBARRIER:
case GT_KEEPALIVE:
+ case GT_ASYNC_CONTINUATION:
+ case GT_RETURN_SUSPEND:
case GT_SWIFT_ERROR:
case GT_GCPOLL:
return true;
@@ -7306,6 +7351,8 @@ bool GenTree::OperSupportsOrderingSideEffect() const
case GT_CMPXCHG:
case GT_MEMORYBARRIER:
case GT_CATCH_ARG:
+ case GT_ASYNC_CONTINUATION:
+ case GT_RETURN_SUSPEND:
case GT_SWIFT_ERROR:
return true;
default:
@@ -9419,6 +9466,7 @@ GenTree* Compiler::gtCloneExpr(GenTree* tree)
goto DONE;
case GT_CATCH_ARG:
+ case GT_ASYNC_CONTINUATION:
case GT_NO_OP:
case GT_NOP:
case GT_LABEL:
@@ -10160,6 +10208,7 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node)
case GT_LCL_FLD:
case GT_LCL_ADDR:
case GT_CATCH_ARG:
+ case GT_ASYNC_CONTINUATION:
case GT_LABEL:
case GT_FTN_ADDR:
case GT_RET_EXPR:
@@ -10231,6 +10280,7 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node)
case GT_PUTARG_SPLIT:
#endif // FEATURE_ARG_SPLIT
case GT_RETURNTRAP:
+ case GT_RETURN_SUSPEND:
m_edge = &m_node->AsUnOp()->gtOp1;
assert(*m_edge != nullptr);
m_advance = &GenTreeUseEdgeIterator::Terminate;
@@ -11770,6 +11820,10 @@ void Compiler::gtGetLclVarNameInfo(unsigned lclNum, const char** ilKindOut, cons
{
ilName = "this";
}
+ else if (lclNum == lvaAsyncContinuationArg)
+ {
+ ilName = "AsyncCont";
+ }
else
{
ilKind = "arg";
@@ -12330,6 +12384,7 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack)
case GT_START_PREEMPTGC:
case GT_PROF_HOOK:
case GT_CATCH_ARG:
+ case GT_ASYNC_CONTINUATION:
case GT_MEMORYBARRIER:
case GT_PINVOKE_PROLOG:
case GT_JMPTABLE:
@@ -13072,6 +13127,8 @@ const char* Compiler::gtGetWellKnownArgNameForArgMsg(WellKnownArg arg)
return "va cookie";
case WellKnownArg::InstParam:
return "gctx";
+ case WellKnownArg::AsyncContinuation:
+ return "async";
case WellKnownArg::RetBuffer:
return "retbuf";
case WellKnownArg::PInvokeFrame:
diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h
index f3fb94b09429e8..2b0a4e963598f4 100644
--- a/src/coreclr/jit/gentree.h
+++ b/src/coreclr/jit/gentree.h
@@ -4238,6 +4238,7 @@ enum GenTreeCallFlags : unsigned int
GTF_CALL_M_GUARDED_DEVIRT_CHAIN = 0x00080000, // this call is a candidate for chained guarded devirtualization
GTF_CALL_M_ALLOC_SIDE_EFFECTS = 0x00100000, // this is a call to an allocator with side effects
GTF_CALL_M_SUPPRESS_GC_TRANSITION = 0x00200000, // suppress the GC transition (i.e. during a pinvoke) but a separate GC safe point is required.
+ GTF_CALL_M_ASYNC = 0x00400000, // this call is a runtime async method call
GTF_CALL_M_EXPANDED_EARLY = 0x00800000, // the Virtual Call target address is expanded and placed in gtControlExpr in Morph rather than in Lower
GTF_CALL_M_LDVIRTFTN_INTERFACE = 0x01000000, // ldvirtftn on an interface type
GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x02000000, // this cast (helper call) can be expanded if it's profitable. To be removed.
@@ -4563,6 +4564,7 @@ enum class WellKnownArg : unsigned
ThisPointer,
VarArgsCookie,
InstParam,
+ AsyncContinuation,
RetBuffer,
PInvokeFrame,
WrapperDelegateCell,
@@ -4741,6 +4743,7 @@ class CallArgs
#endif
bool m_hasThisPointer : 1;
bool m_hasRetBuffer : 1;
+ bool m_hasAsyncContinuation : 1;
bool m_isVarArgs : 1;
bool m_abiInformationDetermined : 1;
bool m_hasAddedFinalArgs : 1;
@@ -4788,6 +4791,7 @@ class CallArgs
CallArg* InsertAfter(Compiler* comp, CallArg* after, const NewCallArg& arg);
CallArg* InsertAfterUnchecked(Compiler* comp, CallArg* after, const NewCallArg& arg);
CallArg* InsertInstParam(Compiler* comp, GenTree* node);
+ CallArg* InsertAsyncContinuationParam(Compiler* comp, GenTree* node);
CallArg* InsertAfterThisOrFirst(Compiler* comp, const NewCallArg& arg);
void PushLateBack(CallArg* arg);
void Remove(CallArg* arg);
@@ -4815,6 +4819,7 @@ class CallArgs
// clang-format off
bool HasThisPointer() const { return m_hasThisPointer; }
bool HasRetBuffer() const { return m_hasRetBuffer; }
+ bool HasAsyncContinuation() const { return m_hasAsyncContinuation; }
bool IsVarArgs() const { return m_isVarArgs; }
void SetIsVarArgs() { m_isVarArgs = true; }
void ClearIsVarArgs() { m_isVarArgs = false; }
@@ -5019,6 +5024,13 @@ struct GenTreeCall final : public GenTree
#endif
}
+ void SetIsAsync()
+ {
+ gtCallMoreFlags |= GTF_CALL_M_ASYNC;
+ }
+
+ bool IsAsync() const;
+
//---------------------------------------------------------------------------
// GetRegNumByIdx: get i'th return register allocated to this call node.
//
diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h
index 7a13212fa6eaf3..cfe7a750082aa0 100644
--- a/src/coreclr/jit/gtlist.h
+++ b/src/coreclr/jit/gtlist.h
@@ -33,6 +33,7 @@ GTNODE(LCL_ADDR , GenTreeLclFld ,0,0,GTK_LEAF) // local
//-----------------------------------------------------------------------------
GTNODE(CATCH_ARG , GenTree ,0,0,GTK_LEAF) // Exception object in a catch block
+GTNODE(ASYNC_CONTINUATION, GenTree ,0,0,GTK_LEAF) // Returned continuation by an async2 call
GTNODE(LABEL , GenTree ,0,0,GTK_LEAF) // Jump-target
GTNODE(JMP , GenTreeVal ,0,0,GTK_LEAF|GTK_NOVALUE) // Jump to another function
GTNODE(FTN_ADDR , GenTreeFptrVal ,0,0,GTK_LEAF) // Address of a function
@@ -290,6 +291,11 @@ GTNODE(RETURN , GenTreeOp ,0,1,GTK_UNOP|GTK_NOVALUE)
GTNODE(SWITCH , GenTreeOp ,0,1,GTK_UNOP|GTK_NOVALUE)
GTNODE(NO_OP , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE) // A NOP that cannot be deleted.
+// Suspend an async2 method, returning a continuation.
+// Before lowering this is a seemingly normal TYP_VOID node with a lot of side effects (GTF_CALL | GTF_GLOB_REF | GTF_ORDER_SIDEEFF).
+// Lowering then removes all successor nodes and leaves it as the terminator node.
+GTNODE(RETURN_SUSPEND , GenTreeOp ,0,1,GTK_UNOP|GTK_NOVALUE) // Return a continuation in an async2 method
+
GTNODE(START_NONGC , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // Starts a new instruction group that will be non-gc interruptible.
GTNODE(START_PREEMPTGC , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // Starts a new instruction group where preemptive GC is enabled.
GTNODE(PROF_HOOK , GenTree ,0,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // Profiler Enter/Leave/TailCall hook.
diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp
index 856065a737f7f5..92a4c7f9c88c59 100644
--- a/src/coreclr/jit/importer.cpp
+++ b/src/coreclr/jit/importer.cpp
@@ -846,7 +846,8 @@ GenTree* Compiler::impStoreStruct(GenTree* store,
// Make sure we don't pass something other than a local address to the return buffer arg.
// It is allowed to pass current's method return buffer as it is a local too.
- if (fgAddrCouldBeHeap(destAddr) && !eeIsByrefLike(srcCall->gtRetClsHnd))
+ if ((fgAddrCouldBeHeap(destAddr) && !eeIsByrefLike(srcCall->gtRetClsHnd)) ||
+ (compIsAsync() && !destAddr->OperIs(GT_LCL_ADDR)))
{
unsigned tmp = lvaGrabTemp(false DEBUGARG("stack copy for value returned via return buffer"));
lvaSetStruct(tmp, srcCall->gtRetClsHnd, false);
@@ -972,7 +973,8 @@ GenTree* Compiler::impStoreStruct(GenTree* store,
// Make sure we don't pass something other than a local address to the return buffer arg.
// It is allowed to pass current's method return buffer as it is a local too.
- if (fgAddrCouldBeHeap(destAddr) && !eeIsByrefLike(call->gtRetClsHnd))
+ if ((fgAddrCouldBeHeap(destAddr) && !eeIsByrefLike(call->gtRetClsHnd)) ||
+ (compIsAsync() && !destAddr->OperIs(GT_LCL_ADDR)))
{
unsigned tmp = lvaGrabTemp(false DEBUGARG("stack copy for value returned via return buffer"));
lvaSetStruct(tmp, call->gtRetClsHnd, false);
@@ -5977,6 +5979,85 @@ bool Compiler::impBlockIsInALoop(BasicBlock* block)
block->HasFlag(BBF_BACKWARD_JUMP);
}
+//------------------------------------------------------------------------
+// impMatchAwaitPattern: check if a method call starts an Await pattern
+// that can be optimized for runtime async
+//
+// Arguments:
+// codeAddr - IL after call[virt]
+// codeEndp - End of IL code stream
+// configVal - [out] set to 0 or 1, accordingly, if we saw ConfigureAwait(0|1)
+//
+// Returns:
+// true if this is an Await that we can optimize
+//
+bool Compiler::impMatchAwaitPattern(const BYTE* codeAddr, const BYTE* codeEndp, int* configVal)
+{
+ // If we see the following code pattern in runtime async methods:
+ //
+ // call[virt]
+ // [ OPTIONAL ]
+ // ldc.i4.0 / ldc.i4.1
+ // call[virt]
+ // call
+ //
+ // We emit an eqivalent of:
+ //
+ // call[virt]
+ //
+ // where "RtMethod" is the runtime-async counterpart of a Task-returning method.
+ //
+ // NOTE: we could potentially check if Method is not a thunk and, in cases when we can tell,
+ // bypass this optimization. Otherwise in a non-thunk case we would be
+ // replacing the pattern with a call to a thunk, which contains roughly the same code.
+
+ const BYTE* nextOpcode = codeAddr + sizeof(mdToken);
+ // There must be enough space after ldc for {call + tk + call + tk}
+ if (nextOpcode + 2 * (1 + sizeof(mdToken)) < codeEndp)
+ {
+ uint8_t nextOp = getU1LittleEndian(nextOpcode);
+ uint8_t nextNextOp = getU1LittleEndian(nextOpcode + 1);
+ if ((nextOp != CEE_LDC_I4_0 && nextOp != CEE_LDC_I4_1) ||
+ (nextNextOp != CEE_CALL && nextNextOp != CEE_CALLVIRT))
+ {
+ goto checkForAwait;
+ }
+
+ // check if the token after {ldc, call[virt]} is ConfigAwait
+ CORINFO_RESOLVED_TOKEN nextCallTok;
+ impResolveToken(nextOpcode + 2, &nextCallTok, CORINFO_TOKENKIND_Method);
+
+ if (!eeIsIntrinsic(nextCallTok.hMethod) ||
+ lookupNamedIntrinsic(nextCallTok.hMethod) != NI_System_Threading_Tasks_Task_ConfigureAwait)
+ {
+ goto checkForAwait;
+ }
+
+ *configVal = nextOp == CEE_LDC_I4_0 ? 0 : 1;
+ // skip {ldc; call; }
+ nextOpcode += 1 + 1 + sizeof(mdToken);
+ }
+
+checkForAwait:
+
+ if ((nextOpcode + sizeof(mdToken) < codeEndp) && (getU1LittleEndian(nextOpcode) == CEE_CALL))
+ {
+ // resolve the next token
+ CORINFO_RESOLVED_TOKEN nextCallTok;
+ impResolveToken(nextOpcode + 1, &nextCallTok, CORINFO_TOKENKIND_Method);
+
+ // check if it is an Await intrinsic
+ if (eeIsIntrinsic(nextCallTok.hMethod) &&
+ lookupNamedIntrinsic(nextCallTok.hMethod) == NI_System_Runtime_CompilerServices_RuntimeHelpers_Await)
+ {
+ // yes, this is an Await
+ return true;
+ }
+ }
+
+ return false;
+}
+
#ifdef _PREFAST_
#pragma warning(push)
#pragma warning(disable : 21000) // Suppress PREFast warning about overly large function
@@ -6696,6 +6777,11 @@ void Compiler::impImportBlockCode(BasicBlock* block)
if (compIsForInlining())
{
+ if ((lclNum == 0) && compIsStructMethodThatOperatesOnCopy())
+ {
+ BADCODE("Illegal starg 0 in function");
+ }
+
op1 = impInlineFetchArg(impInlineInfo->inlArgInfo[lclNum], impInlineInfo->lclVarInfo[lclNum]);
noway_assert(op1->gtOper == GT_LCL_VAR);
lclNum = op1->AsLclVar()->GetLclNum();
@@ -6709,6 +6795,11 @@ void Compiler::impImportBlockCode(BasicBlock* block)
if (lclNum == info.compThisArg)
{
lclNum = lvaArg0Var;
+
+ if (compIsStructMethodThatOperatesOnCopy())
+ {
+ BADCODE("Illegal starg 0 in function");
+ }
}
// We should have seen this arg write in the prescan
@@ -6928,6 +7019,11 @@ void Compiler::impImportBlockCode(BasicBlock* block)
return;
}
+ if ((lclNum == 0) && compIsStructMethodThatOperatesOnCopy())
+ {
+ BADCODE("Illegal ldarga 0 in function");
+ }
+
op1->ChangeType(TYP_BYREF);
op1->SetOper(GT_LCL_ADDR);
op1->AsLclFld()->SetLclOffs(0);
@@ -6940,6 +7036,11 @@ void Compiler::impImportBlockCode(BasicBlock* block)
if (lclNum == info.compThisArg)
{
lclNum = lvaArg0Var;
+
+ if (compIsStructMethodThatOperatesOnCopy())
+ {
+ BADCODE("Illegal ldarga 0 in function");
+ }
}
goto ADRVAR;
@@ -8993,7 +9094,41 @@ void Compiler::impImportBlockCode(BasicBlock* block)
// many other places. We unfortunately embed that knowledge here.
if (opcode != CEE_CALLI)
{
- _impResolveToken(CORINFO_TOKENKIND_Method);
+ bool isAwait = false;
+ // TODO: The configVal should be wired to the actual implementation
+ // that control the flow of sync context.
+ // We do not have that yet.
+ int configVal = -1; // -1 not congigured, 0/1 configured to false/true
+ if (compIsAsync() && JitConfig.JitOptimizeAwait())
+ {
+ isAwait = impMatchAwaitPattern(codeAddr, codeEndp, &configVal);
+ }
+
+ if (isAwait)
+ {
+ _impResolveToken(CORINFO_TOKENKIND_Await);
+ if (resolvedToken.hMethod != NULL)
+ {
+ // There is a runtime async variant that is implicitly awaitable, just call that.
+ // if configured, skip {ldc call ConfigureAwait}
+ if (configVal >= 0)
+ codeAddr += 2 + sizeof(mdToken);
+
+ // Skip the call to `Await`
+ codeAddr += 1 + sizeof(mdToken);
+ }
+ else
+ {
+ // This can happen in rare cases when the Task-returning method is not a runtime Async
+ // function. For example "T M1(T arg) => arg" when called with a Task argument. Treat
+ // that as a regualr call that is Awaited
+ _impResolveToken(CORINFO_TOKENKIND_Method);
+ }
+ }
+ else
+ {
+ _impResolveToken(CORINFO_TOKENKIND_Method);
+ }
eeGetCallInfo(&resolvedToken,
(prefixFlags & PREFIX_CONSTRAINED) ? &constrainedResolvedToken : nullptr,
@@ -10949,6 +11084,14 @@ void Compiler::impLoadArg(unsigned ilArgNum, IL_OFFSET offset)
if (lclNum == info.compThisArg)
{
lclNum = lvaArg0Var;
+
+ // Redirect to copy in some struct instance methods
+ if (lvaThisCopyVar != BAD_VAR_NUM)
+ {
+ GenTree* lclAddr = gtNewLclVarAddrNode(lvaThisCopyVar, TYP_BYREF);
+ impPushOnStack(lclAddr, makeTypeInfoForLocal(lclNum));
+ return;
+ }
}
impLoadVar(lclNum, offset);
@@ -13280,7 +13423,8 @@ void Compiler::impInlineInitVars(InlineInfo* pInlineInfo)
switch (arg.GetWellKnownArg())
{
case WellKnownArg::RetBuffer:
- // This does not appear in the table of inline arg info; do not include them
+ case WellKnownArg::AsyncContinuation:
+ // These do not appear in the table of inline arg info; do not include them
continue;
case WellKnownArg::InstParam:
pInlineInfo->inlInstParamArgInfo = argInfo = new (this, CMK_Inlining) InlArgInfo{};
@@ -13297,6 +13441,17 @@ void Compiler::impInlineInitVars(InlineInfo* pInlineInfo)
{
return;
}
+
+ if ((arg.GetWellKnownArg() == WellKnownArg::ThisPointer) &&
+ ((methInfo->options & CORINFO_OPT_COPY_STRUCT_INSTANCE) != 0))
+ {
+ // Method call to a struct instance method that operates on a copy.
+ // We will load the instance as part of copying, so set up flags to
+ // indicate that there is a side effect.
+ argInfo->argIsByRefToCopy = true;
+ argInfo->argHasGlobRef = true;
+ argInfo->argHasSideEff = true;
+ }
}
#ifdef FEATURE_SIMD
@@ -13683,7 +13838,7 @@ GenTree* Compiler::impInlineFetchArg(InlArgInfo& argInfo, const InlLclVarInfo& l
{
// Cache the relevant arg and lcl info for this argument.
// We will modify argInfo but not lclVarInfo.
- const bool argCanBeModified = argInfo.argHasLdargaOp || argInfo.argHasStargOp;
+ const bool argCanBeModified = argInfo.argHasLdargaOp || argInfo.argHasStargOp || argInfo.argIsByRefToCopy;
const var_types lclTyp = lclInfo.lclTypeInfo;
GenTree* op1 = nullptr;
@@ -13760,7 +13915,7 @@ GenTree* Compiler::impInlineFetchArg(InlArgInfo& argInfo, const InlLclVarInfo& l
}
}
}
- else if (argInfo.argIsByRefToStructLocal && !argInfo.argHasStargOp)
+ else if (argInfo.argIsByRefToStructLocal && !argInfo.argHasStargOp && !argInfo.argIsByRefToCopy)
{
/* Argument is a by-ref address to a struct, a normed struct, or its field.
In these cases, don't spill the byref to a local, simply clone the tree and use it.
@@ -13857,7 +14012,7 @@ GenTree* Compiler::impInlineFetchArg(InlArgInfo& argInfo, const InlLclVarInfo& l
// if it is a struct, because it requires some additional handling.
if ((!varTypeIsStruct(lclTyp) && !argInfo.argHasSideEff && !argInfo.argHasGlobRef &&
- !argInfo.argHasCallerLocalRef))
+ !argInfo.argHasCallerLocalRef && !argInfo.argIsByRefToStructLocal))
{
/* Get a *LARGE* LCL_VAR node */
op1 = gtNewLclLNode(tmpNum, genActualType(lclTyp));
diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp
index 7c39a95917644b..9bec6d242e5748 100644
--- a/src/coreclr/jit/importercalls.cpp
+++ b/src/coreclr/jit/importercalls.cpp
@@ -96,7 +96,8 @@ var_types Compiler::impImportCall(OPCODE opcode,
bool bIntrinsicImported = false;
CORINFO_SIG_INFO calliSig;
- NewCallArg extraArg;
+ GenTree* varArgsCookie = nullptr;
+ GenTree* instParam = nullptr;
// Swift calls that might throw use a SwiftError* arg that requires additional IR to handle,
// so if we're importing a Swift call, look for this type in the signature
@@ -522,6 +523,18 @@ var_types Compiler::impImportCall(OPCODE opcode,
{
call->AsCall()->gtCallMoreFlags |= GTF_CALL_M_SPECIAL_INTRINSIC;
}
+
+ // Temporary hack since these functions have to be recognized as async2
+ // calls in JIT generated state machines only.
+ if (compIsAsync() &&
+ ((ni == NI_System_Runtime_CompilerServices_RuntimeHelpers_AwaitAwaiterFromRuntimeAsync) ||
+ (ni == NI_System_Runtime_CompilerServices_RuntimeHelpers_UnsafeAwaitAwaiterFromRuntimeAsync) ||
+ (ni == NI_System_Runtime_CompilerServices_RuntimeHelpers_Await)))
+ {
+ assert((call != nullptr) && call->OperIs(GT_CALL));
+ call->AsCall()->SetIsAsync();
+ JITDUMP("Marking [%06u] as a special-case async call\n", dspTreeID(call));
+ }
}
assert(sig);
assert(clsHnd || (opcode == CEE_CALLI)); // We're never verifying for CALLI, so this is not set.
@@ -711,12 +724,15 @@ var_types Compiler::impImportCall(OPCODE opcode,
}
}
- /*-------------------------------------------------------------------------
- * Create the argument list
- */
+ if (sig->isAsyncCall())
+ {
+ call->AsCall()->SetIsAsync();
+ }
+
+ // Now create the argument list.
//-------------------------------------------------------------------------
- // Special case - for varargs we have an implicit last argument
+ // Special case - for varargs we have an extra argument
if ((sig->callConv & CORINFO_CALLCONV_MASK) == CORINFO_CALLCONV_VARARG)
{
@@ -729,9 +745,7 @@ var_types Compiler::impImportCall(OPCODE opcode,
varCookie = info.compCompHnd->getVarArgsHandle(sig, &pVarCookie);
assert((!varCookie) != (!pVarCookie));
- GenTree* cookieNode = gtNewIconEmbHndNode(varCookie, pVarCookie, GTF_ICON_VARG_HDL, sig);
- assert(extraArg.Node == nullptr);
- extraArg = NewCallArg::Primitive(cookieNode).WellKnown(WellKnownArg::VarArgsCookie);
+ varArgsCookie = gtNewIconEmbHndNode(varCookie, pVarCookie, GTF_ICON_VARG_HDL, sig);
}
//-------------------------------------------------------------------------
@@ -751,7 +765,7 @@ var_types Compiler::impImportCall(OPCODE opcode,
// We also set the exact type context associated with the call so we can
// inline the call correctly later on.
- if (sig->callConv & CORINFO_CALLCONV_PARAMTYPE)
+ if (sig->hasTypeArg())
{
assert(call->AsCall()->gtCallType == CT_USER_FUNC);
if (clsHnd == nullptr)
@@ -761,8 +775,7 @@ var_types Compiler::impImportCall(OPCODE opcode,
assert(opcode != CEE_CALLI);
- GenTree* instParam;
- bool runtimeLookup;
+ bool runtimeLookup;
// Instantiated generic method
if (((SIZE_T)exactContextHnd & CORINFO_CONTEXTFLAGS_MASK) == CORINFO_CONTEXTFLAGS_METHOD)
@@ -852,9 +865,6 @@ var_types Compiler::impImportCall(OPCODE opcode,
}
}
}
-
- assert(extraArg.Node == nullptr);
- extraArg = NewCallArg::Primitive(instParam).WellKnown(WellKnownArg::InstParam);
}
if ((opcode == CEE_NEWOBJ) && ((clsFlags & CORINFO_FLG_DELEGATE) != 0))
@@ -890,18 +900,50 @@ var_types Compiler::impImportCall(OPCODE opcode,
}
impPopCallArgs(sig, call->AsCall());
- if (extraArg.Node != nullptr)
+
+ // Extra args
+ if ((instParam != nullptr) || call->AsCall()->IsAsync() || (varArgsCookie != nullptr))
{
if (Target::g_tgtArgOrder == Target::ARG_ORDER_R2L)
{
- call->AsCall()->gtArgs.PushFront(this, extraArg);
+ if (varArgsCookie != nullptr)
+ {
+ call->AsCall()->gtArgs.PushFront(this, NewCallArg::Primitive(varArgsCookie)
+ .WellKnown(WellKnownArg::VarArgsCookie));
+ }
+
+ if (call->AsCall()->IsAsync())
+ {
+ call->AsCall()->gtArgs.PushFront(this, NewCallArg::Primitive(gtNewNull(), TYP_REF)
+ .WellKnown(WellKnownArg::AsyncContinuation));
+ }
+
+ if (instParam != nullptr)
+ {
+ call->AsCall()->gtArgs.PushFront(this,
+ NewCallArg::Primitive(instParam).WellKnown(WellKnownArg::InstParam));
+ }
}
else
{
- call->AsCall()->gtArgs.PushBack(this, extraArg);
- }
+ if (instParam != nullptr)
+ {
+ call->AsCall()->gtArgs.PushBack(this,
+ NewCallArg::Primitive(instParam).WellKnown(WellKnownArg::InstParam));
+ }
- call->gtFlags |= extraArg.Node->gtFlags & GTF_GLOB_EFFECT;
+ if (call->AsCall()->IsAsync())
+ {
+ call->AsCall()->gtArgs.PushBack(this, NewCallArg::Primitive(gtNewNull(), TYP_REF)
+ .WellKnown(WellKnownArg::AsyncContinuation));
+ }
+
+ if (varArgsCookie != nullptr)
+ {
+ call->AsCall()->gtArgs.PushBack(this, NewCallArg::Primitive(varArgsCookie)
+ .WellKnown(WellKnownArg::VarArgsCookie));
+ }
+ }
}
//-------------------------------------------------------------------------
@@ -3303,6 +3345,35 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd,
return new (this, GT_LABEL) GenTree(GT_LABEL, TYP_I_IMPL);
}
+ if (ni == NI_System_StubHelpers_Async2CallContinuation)
+ {
+ GenTree* node = new (this, GT_ASYNC_CONTINUATION) GenTree(GT_ASYNC_CONTINUATION, TYP_REF);
+ node->SetHasOrderingSideEffect();
+ node->gtFlags |= GTF_CALL | GTF_GLOB_REF;
+ info.compUsesAsyncContinuation = true;
+ return node;
+ }
+
+ if (ni == NI_System_Runtime_CompilerServices_RuntimeHelpers_SuspendAsync2)
+ {
+ GenTree* node = gtNewOperNode(GT_RETURN_SUSPEND, TYP_VOID, impPopStack().val);
+ node->SetHasOrderingSideEffect();
+ node->gtFlags |= GTF_CALL | GTF_GLOB_REF;
+ return node;
+ }
+
+ if ((ni == NI_System_Runtime_CompilerServices_RuntimeHelpers_AwaitAwaiterFromRuntimeAsync) ||
+ (ni == NI_System_Runtime_CompilerServices_RuntimeHelpers_UnsafeAwaitAwaiterFromRuntimeAsync) ||
+ (ni == NI_System_Runtime_CompilerServices_RuntimeHelpers_Await))
+ {
+ // These are marked intrinsics simply to mark the call node as async,
+ // which the caller will do. Make sure we keep pIntrinsicName assigned
+ // (it would be overridden if we left this up to the rest of this
+ // function).
+ *pIntrinsicName = ni;
+ return nullptr;
+ }
+
bool betterToExpand = false;
// Allow some lightweight intrinsics in Tier0 which can improve throughput
@@ -10927,6 +10998,28 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
{
result = NI_System_Runtime_CompilerServices_RuntimeHelpers_GetMethodTable;
}
+ else if (strcmp(methodName, "AwaitAwaiterFromRuntimeAsync") == 0)
+ {
+ result = NI_System_Runtime_CompilerServices_RuntimeHelpers_AwaitAwaiterFromRuntimeAsync;
+ }
+ else if (strcmp(methodName, "UnsafeAwaitAwaiterFromRuntimeAsync") == 0)
+ {
+ result =
+ NI_System_Runtime_CompilerServices_RuntimeHelpers_UnsafeAwaitAwaiterFromRuntimeAsync;
+ }
+ else if (strcmp(methodName, "Await") == 0)
+ {
+ result = NI_System_Runtime_CompilerServices_RuntimeHelpers_Await;
+ }
+ else if (strcmp(methodName, "SuspendAsync2") == 0)
+ {
+ result = NI_System_Runtime_CompilerServices_RuntimeHelpers_SuspendAsync2;
+ }
+ else if (strcmp(methodName, "get_RuntimeAsyncViaJitGeneratedStateMachines") == 0)
+ {
+ result =
+ NI_System_Runtime_CompilerServices_RuntimeHelpers_get_RuntimeAsyncViaJitGeneratedStateMachines;
+ }
}
else if (strcmp(className, "StaticsHelpers") == 0)
{
@@ -11176,6 +11269,10 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
{
result = NI_System_StubHelpers_NextCallReturnAddress;
}
+ else if (strcmp(methodName, "Async2CallContinuation") == 0)
+ {
+ result = NI_System_StubHelpers_Async2CallContinuation;
+ }
}
}
else if (strcmp(namespaceName, "Text") == 0)
@@ -11253,6 +11350,17 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
}
}
}
+ else if (strcmp(namespaceName, "Threading.Tasks") == 0)
+ {
+ if (strcmp(methodName, "ConfigureAwait") == 0)
+ {
+ if (strcmp(className, "Task`1") == 0 || strcmp(className, "Task") == 0 ||
+ strcmp(className, "ValuTask`1") == 0 || strcmp(className, "ValueTask") == 0)
+ {
+ result = NI_System_Threading_Tasks_Task_ConfigureAwait;
+ }
+ }
+ }
}
}
else if (strcmp(namespaceName, "Internal.Runtime") == 0)
diff --git a/src/coreclr/jit/inline.def b/src/coreclr/jit/inline.def
index 44d6e83929e0ba..8ae779f7138819 100644
--- a/src/coreclr/jit/inline.def
+++ b/src/coreclr/jit/inline.def
@@ -115,6 +115,7 @@ INLINE_OBSERVATION(UNSUPPORTED_OPCODE, bool, "unsupported opcode",
INLINE_OBSERVATION(DEBUG_CODEGEN, bool, "debug codegen", FATAL, CALLER)
INLINE_OBSERVATION(IS_JIT_NOINLINE, bool, "noinline per JitNoInlineRange", FATAL, CALLER)
INLINE_OBSERVATION(USES_NEXT_CALL_RET_ADDR, bool, "uses NextCallReturnAddress intrinsic", FATAL, CALLER)
+INLINE_OBSERVATION(ASYNC2_USED_CONTINUATION, bool, "uses Async2CallContinuation intrinsic",FATAL, CALLER)
// ------ Caller Information -------
diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h
index 20bc4d4db4f319..17d7d59dc65f1f 100644
--- a/src/coreclr/jit/inline.h
+++ b/src/coreclr/jit/inline.h
@@ -656,7 +656,9 @@ struct InlArgInfo
unsigned argHasStargOp : 1; // Is there STARG(s) operation on this argument?
unsigned argIsByRefToStructLocal : 1; // Is this arg an address of a struct local or a normed struct local or a
// field in them?
- unsigned argIsExact : 1; // Is this arg of an exact class?
+ unsigned argIsExact : 1; // Is this arg of an exact class?
+ unsigned argIsByRefToCopy : 1; // Arg is a reference to a local that will be copied (used when inlining async2
+ // struct instance methods)
};
// InlLclVarInfo describes inline candidate argument and local variable properties.
diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h
index e7a71a4f1e3d02..fb0f9df00e2ecf 100644
--- a/src/coreclr/jit/jitconfigvalues.h
+++ b/src/coreclr/jit/jitconfigvalues.h
@@ -594,6 +594,8 @@ OPT_CONFIG_INTEGER(JitDoIfConversion, "JitDoIfConversion", 1)
OPT_CONFIG_INTEGER(JitDoOptimizeMaskConversions, "JitDoOptimizeMaskConversions", 1) // Perform optimization of mask
// conversions
+RELEASE_CONFIG_INTEGER(JitOptimizeAwait, "JitOptimizeAwait", 1) // Perform optimization of Await intrinsics
+
RELEASE_CONFIG_INTEGER(JitEnableOptRepeat, "JitEnableOptRepeat", 1) // If zero, do not allow JitOptRepeat
RELEASE_CONFIG_METHODSET(JitOptRepeat, "JitOptRepeat") // Runs optimizer multiple times on specified methods
RELEASE_CONFIG_INTEGER(JitOptRepeatCount, "JitOptRepeatCount", 2) // Number of times to repeat opts when repeating
diff --git a/src/coreclr/jit/jitee.h b/src/coreclr/jit/jitee.h
index d3a37a2c4d40c2..1e7a2ee740a5a1 100644
--- a/src/coreclr/jit/jitee.h
+++ b/src/coreclr/jit/jitee.h
@@ -44,6 +44,8 @@ class JitFlags
JIT_FLAG_SOFTFP_ABI = 30, // Enable armel calling convention
#endif
+ JIT_FLAG_RUNTIMEASYNCFUNCTION = 31, // Generate code for use as an async2 function
+
// Note: the mcs tool uses the currently unused upper flags bits when outputting SuperPMI MC file flags.
// See EXTRA_JIT_FLAGS and spmidumphelper.cpp. Currently, these are bits 56 through 63. If they overlap,
// something needs to change.
diff --git a/src/coreclr/jit/layout.cpp b/src/coreclr/jit/layout.cpp
index bb281674254790..2b9a0197d07a75 100644
--- a/src/coreclr/jit/layout.cpp
+++ b/src/coreclr/jit/layout.cpp
@@ -536,6 +536,31 @@ ClassLayout* ClassLayout::Create(Compiler* compiler, const ClassLayoutBuilder& b
return newLayout;
}
+//------------------------------------------------------------------------
+// HasGCByRef: // Check if this classlayout has a TYP_BYREF GC pointer in it.
+//
+// Return value:
+// True if so.
+//
+bool ClassLayout::HasGCByRef() const
+{
+ if (!HasGCPtr())
+ {
+ return false;
+ }
+
+ unsigned numSlots = GetSlotCount();
+ for (unsigned i = 0; i < numSlots; i++)
+ {
+ if (GetGCPtrType(i) == TYP_BYREF)
+ {
+ return true;
+ }
+ }
+
+ return false;
+}
+
//------------------------------------------------------------------------
// IsStackOnly: does the layout represent a block that can never be on the heap?
//
diff --git a/src/coreclr/jit/layout.h b/src/coreclr/jit/layout.h
index e92fee4ff43553..dad361c10ac0fd 100644
--- a/src/coreclr/jit/layout.h
+++ b/src/coreclr/jit/layout.h
@@ -222,6 +222,8 @@ class ClassLayout
return m_gcPtrCount != 0;
}
+ bool HasGCByRef() const;
+
bool IsStackOnly(Compiler* comp) const;
bool IsGCPtr(unsigned slot) const
diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp
index 29d5e07e7d3d47..bbd3fce509f652 100644
--- a/src/coreclr/jit/lclvars.cpp
+++ b/src/coreclr/jit/lclvars.cpp
@@ -36,19 +36,19 @@ void Compiler::lvaInitTypeRef()
{
/* x86 args look something like this:
- [this ptr] [hidden return buffer] [declared arguments]* [generic context] [var arg cookie]
+ [this ptr] [hidden return buffer] [declared arguments]* [generic context] [async continuation] [var arg cookie]
x64 is closer to the native ABI:
- [this ptr] [hidden return buffer] [generic context] [var arg cookie] [declared arguments]*
+ [this ptr] [hidden return buffer] [generic context] [async continuation] [var arg cookie] [declared arguments]*
(Note: prior to .NET Framework 4.5.1 for Windows 8.1 (but not .NET Framework 4.5.1 "downlevel"),
the "hidden return buffer" came before the "this ptr". Now, the "this ptr" comes first. This
is different from the C++ order, where the "hidden return buffer" always comes first.)
ARM and ARM64 are the same as the current x64 convention:
- [this ptr] [hidden return buffer] [generic context] [var arg cookie] [declared arguments]*
+ [this ptr] [hidden return buffer] [generic context] [async continuation] [var arg cookie] [declared arguments]*
Key difference:
- The var arg cookie and generic context are swapped with respect to the user arguments
+ The var arg cookie, generic context and async continuations are swapped with respect to the user arguments
*/
/* Set compArgsCount and compLocalsCount */
@@ -161,6 +161,11 @@ void Compiler::lvaInitTypeRef()
info.compTypeCtxtArg = BAD_VAR_NUM;
}
+ if (compIsAsync())
+ {
+ info.compArgsCount++;
+ }
+
lvaCount = info.compLocalsCount = info.compArgsCount + info.compMethodInfo->locals.numArgs;
info.compILlocalsCount = info.compILargsCount + info.compMethodInfo->locals.numArgs;
@@ -371,6 +376,8 @@ void Compiler::lvaInitArgs(bool hasRetBuffArg)
// and shared generic struct instance methods
lvaInitGenericsCtxt(&varNum);
+ lvaInitAsyncContinuation(&varNum);
+
/* If the method is varargs, process the varargs cookie */
lvaInitVarArgsHandle(&varNum);
#endif
@@ -384,6 +391,8 @@ void Compiler::lvaInitArgs(bool hasRetBuffArg)
// and shared generic struct instance methods
lvaInitGenericsCtxt(&varNum);
+ lvaInitAsyncContinuation(&varNum);
+
/* If the method is varargs, process the varargs cookie */
lvaInitVarArgsHandle(&varNum);
#endif
@@ -676,6 +685,33 @@ void Compiler::lvaInitGenericsCtxt(unsigned* curVarNum)
(*curVarNum)++;
}
+//-----------------------------------------------------------------------------
+// lvaInitAsyncContinuation:
+// Initialize the async continuation parameter.
+//
+// Type parameters:
+// curVarNum - [in, out] The current local variable number for parameters
+//
+void Compiler::lvaInitAsyncContinuation(unsigned* curVarNum)
+{
+ if (!compIsAsync())
+ {
+ return;
+ }
+
+ lvaAsyncContinuationArg = *curVarNum;
+ LclVarDsc* varDsc = lvaGetDesc(*curVarNum);
+ varDsc->lvType = TYP_REF;
+ varDsc->lvIsParam = true;
+
+ // The final home for this incoming register might be our local stack frame
+ varDsc->lvOnFrame = true;
+
+ INDEBUG(varDsc->lvReason = "Async continuation arg");
+
+ (*curVarNum)++;
+}
+
/*****************************************************************************/
void Compiler::lvaInitVarArgsHandle(unsigned* curVarNum)
{
diff --git a/src/coreclr/jit/lir.h b/src/coreclr/jit/lir.h
index a3271e832fa8de..99d011ea32e9bc 100644
--- a/src/coreclr/jit/lir.h
+++ b/src/coreclr/jit/lir.h
@@ -284,6 +284,20 @@ class LIR final
void InsertAtBeginning(Range&& range);
void InsertAtEnd(Range&& range);
+ template
+ void InsertAtBeginning(GenTree* tree, Trees&&... rest)
+ {
+ InsertAtBeginning(std::forward(rest)...);
+ InsertAtBeginning(tree);
+ }
+
+ template
+ void InsertAtEnd(GenTree* tree, Trees&&... rest)
+ {
+ InsertAtEnd(tree);
+ InsertAtEnd(std::forward(rest)...);
+ }
+
void Remove(GenTree* node, bool markOperandsUnused = false);
Range Remove(GenTree* firstNode, GenTree* lastNode);
Range Remove(ReadOnlyRange&& range);
diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp
index b199d9b6612509..f1c7c9a1d1ef9e 100644
--- a/src/coreclr/jit/liveness.cpp
+++ b/src/coreclr/jit/liveness.cpp
@@ -1454,6 +1454,7 @@ void Compiler::fgComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VARSET_VALAR
case GT_JCC:
case GT_JTRUE:
case GT_RETURN:
+ case GT_RETURN_SUSPEND:
case GT_SWITCH:
case GT_RETFILT:
case GT_START_NONGC:
diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp
index 9784dd6d55bd5d..0775a25a6a1028 100644
--- a/src/coreclr/jit/lower.cpp
+++ b/src/coreclr/jit/lower.cpp
@@ -732,6 +732,14 @@ GenTree* Lowering::LowerNode(GenTree* node)
return LowerArrLength(node->AsArrCommon());
break;
+ case GT_ASYNC_CONTINUATION:
+ LowerAsyncContinuation(node);
+ break;
+
+ case GT_RETURN_SUSPEND:
+ LowerReturnSuspend(node);
+ break;
+
default:
break;
}
@@ -5410,6 +5418,61 @@ void Lowering::LowerRetSingleRegStructLclVar(GenTreeUnOp* ret)
}
}
+//----------------------------------------------------------------------------------------------
+// LowerAsyncContinuation: Lower a GT_ASYNC_CONTINUATION node
+//
+// Arguments:
+// asyncCont - Async continuation node
+//
+void Lowering::LowerAsyncContinuation(GenTree* asyncCont)
+{
+ assert(asyncCont->OperIs(GT_ASYNC_CONTINUATION));
+
+ // When the ASYNC_CONTINUATION was created as a result of
+ // StubHelpers.Async2CallContinuation() the previous call hasn't been
+ // marked as an async call. We need to do that to get the right GC
+ // reporting behavior for the returned async continuation.
+ GenTree* node = asyncCont;
+ while (true)
+ {
+ node = node->gtPrev;
+ noway_assert((node != nullptr) && "Ran out of nodes while looking for call before async continuation");
+
+ if (node->IsCall())
+ {
+ if (!node->AsCall()->IsAsync())
+ {
+ JITDUMP("Marking the call [%06u] before async continuation [%06u] as an async call\n",
+ Compiler::dspTreeID(node), Compiler::dspTreeID(asyncCont));
+ node->AsCall()->SetIsAsync();
+ }
+
+ break;
+ }
+ }
+}
+
+//----------------------------------------------------------------------------------------------
+// LowerReturnSuspend:
+// Lower a GT_RETURN_SUSPEND by making it a terminator node.
+//
+// Arguments:
+// node - The node
+//
+void Lowering::LowerReturnSuspend(GenTree* node)
+{
+ assert(node->OperIs(GT_RETURN_SUSPEND));
+ while (BlockRange().LastNode() != node)
+ {
+ BlockRange().Remove(BlockRange().LastNode(), true);
+ }
+
+ if (comp->compMethodRequiresPInvokeFrame())
+ {
+ InsertPInvokeMethodEpilog(comp->compCurBB DEBUGARG(node));
+ }
+}
+
//----------------------------------------------------------------------------------------------
// LowerCallStruct: Lowers a call node that returns a struct.
//
diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h
index 388e136f3f7812..940144f38ce3bb 100644
--- a/src/coreclr/jit/lower.h
+++ b/src/coreclr/jit/lower.h
@@ -180,6 +180,8 @@ class Lowering final : public Phase
GenTree* LowerStoreLocCommon(GenTreeLclVarCommon* lclVar);
void LowerRetStruct(GenTreeUnOp* ret);
void LowerRetSingleRegStructLclVar(GenTreeUnOp* ret);
+ void LowerAsyncContinuation(GenTree* asyncCont);
+ void LowerReturnSuspend(GenTree* retSuspend);
void LowerRetFieldList(GenTreeOp* ret, GenTreeFieldList* fieldList);
bool IsFieldListCompatibleWithReturn(GenTreeFieldList* fieldList);
void LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldList);
diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp
index 6a1bc89c8c04aa..415cddc0fb064a 100644
--- a/src/coreclr/jit/lsra.cpp
+++ b/src/coreclr/jit/lsra.cpp
@@ -3902,6 +3902,11 @@ void LinearScan::processKills(RefPosition* killRefPosition)
regsBusyUntilKill &= ~killRefPosition->getKilledRegisters();
INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_KILL_REGS, nullptr, REG_NA, nullptr, NONE,
killRefPosition->getKilledRegisters()));
+
+ if (killRefPosition->busyUntilNextKill)
+ {
+ regsBusyUntilKill |= killRefPosition->getKilledRegisters();
+ }
}
//------------------------------------------------------------------------
diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h
index 2dcdc5db3f79fa..7cdb8f50b2a695 100644
--- a/src/coreclr/jit/lsra.h
+++ b/src/coreclr/jit/lsra.h
@@ -1030,7 +1030,7 @@ class LinearScan : public LinearScanInterface
// insert refpositions representing prolog zero-inits which will be added later
void insertZeroInitRefPositions();
- void addKillForRegs(regMaskTP mask, LsraLocation currentLoc);
+ RefPosition* addKillForRegs(regMaskTP mask, LsraLocation currentLoc);
void resolveConflictingDefAndUse(Interval* interval, RefPosition* defRefPosition);
@@ -2014,6 +2014,7 @@ class LinearScan : public LinearScanInterface
int BuildPutArgReg(GenTreeUnOp* node);
int BuildCall(GenTreeCall* call);
void MarkSwiftErrorBusyForCall(GenTreeCall* call);
+ void MarkAsyncContinuationBusyForCall(GenTreeCall* call);
int BuildCmp(GenTree* tree);
int BuildCmpOperands(GenTree* tree);
int BuildBlockStore(GenTreeBlk* blkNode);
@@ -2602,6 +2603,9 @@ class RefPosition
unsigned char liveVarUpperSave : 1;
#endif
+ // For a phys reg, indicates that it should be marked as busy until the next time it is killed.
+ unsigned char busyUntilNextKill : 1;
+
#ifdef DEBUG
// Minimum number registers that needs to be ensured while
// constraining candidates for this ref position under
@@ -2645,6 +2649,7 @@ class RefPosition
, isLocalDefUse(false)
, delayRegFree(false)
, outOfOrder(false)
+ , busyUntilNextKill(false)
#ifdef DEBUG
, minRegCandidateCount(1)
, rpNum(0)
diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp
index 815f0149aede11..286dbb00d0cb1f 100644
--- a/src/coreclr/jit/lsraarm.cpp
+++ b/src/coreclr/jit/lsraarm.cpp
@@ -631,6 +631,17 @@ int LinearScan::BuildNode(GenTree* tree)
BuildDef(tree, RBM_EXCEPTION_OBJECT.GetIntRegSet());
break;
+ case GT_ASYNC_CONTINUATION:
+ srcCount = 0;
+ assert(dstCount == 1);
+ // We kill the continuation arg here to communicate to the
+ // selection phase that the argument is no longer busy. This is a
+ // hack to make sure we do not overwrite the continuation between
+ // the call and this node.
+ addKillForRegs(RBM_ASYNC_CONTINUATION_RET, currentLoc);
+ BuildDef(tree, RBM_ASYNC_CONTINUATION_RET.GetIntRegSet());
+ break;
+
case GT_COPY:
srcCount = 1;
#ifdef TARGET_ARM
@@ -693,6 +704,7 @@ int LinearScan::BuildNode(GenTree* tree)
case GT_JCC:
case GT_SETCC:
case GT_MEMORYBARRIER:
+ case GT_RETURN_SUSPEND:
srcCount = BuildSimple(tree);
break;
diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp
index 9af6bef2f17f19..9bf2105a571761 100644
--- a/src/coreclr/jit/lsraarm64.cpp
+++ b/src/coreclr/jit/lsraarm64.cpp
@@ -1320,6 +1320,17 @@ int LinearScan::BuildNode(GenTree* tree)
BuildDef(tree, RBM_EXCEPTION_OBJECT.GetIntRegSet());
break;
+ case GT_ASYNC_CONTINUATION:
+ srcCount = 0;
+ assert(dstCount == 1);
+ // We kill the continuation arg here to communicate to the
+ // selection phase that the argument is no longer busy. This is a
+ // hack to make sure we do not overwrite the continuation between
+ // the call and this node.
+ addKillForRegs(RBM_ASYNC_CONTINUATION_RET, currentLoc);
+ BuildDef(tree, RBM_ASYNC_CONTINUATION_RET.GetIntRegSet());
+ break;
+
case GT_INDEX_ADDR:
assert(dstCount == 1);
srcCount = BuildBinaryUses(tree->AsOp());
diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp
index b3c6c7d4cf788d..2610501b4b74d4 100644
--- a/src/coreclr/jit/lsraarmarch.cpp
+++ b/src/coreclr/jit/lsraarmarch.cpp
@@ -302,6 +302,11 @@ int LinearScan::BuildCall(GenTreeCall* call)
}
#endif // SWIFT_SUPPORT
+ if (call->IsAsync() && compiler->compIsAsync())
+ {
+ MarkAsyncContinuationBusyForCall(call);
+ }
+
// No args are placed in registers anymore.
placedArgRegs = RBM_NONE;
numPlacedArgLocals = 0;
diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp
index b6b01ca38e63f8..e73c0d99ad45c1 100644
--- a/src/coreclr/jit/lsrabuild.cpp
+++ b/src/coreclr/jit/lsrabuild.cpp
@@ -686,7 +686,7 @@ bool LinearScan::isContainableMemoryOp(GenTree* node)
// mask - the mask (set) of registers.
// currentLoc - the location at which they should be added
//
-void LinearScan::addKillForRegs(regMaskTP mask, LsraLocation currentLoc)
+RefPosition* LinearScan::addKillForRegs(regMaskTP mask, LsraLocation currentLoc)
{
// The mask identifies a set of registers that will be used during
// codegen. Mark these as modified here, so when we do final frame
@@ -705,6 +705,8 @@ void LinearScan::addKillForRegs(regMaskTP mask, LsraLocation currentLoc)
*killTail = pos;
killTail = &pos->nextRefPosition;
+
+ return pos;
}
//------------------------------------------------------------------------
@@ -4741,3 +4743,27 @@ void LinearScan::MarkSwiftErrorBusyForCall(GenTreeCall* call)
setDelayFree(swiftErrorRegRecord->lastRefPosition);
}
#endif
+
+//------------------------------------------------------------------------
+// MarkAsyncContinuationBusyForCall:
+// Add a ref position that marks the async continuation register as busy
+// until it is killed.
+//
+// Arguments:
+// call - The call node
+//
+void LinearScan::MarkAsyncContinuationBusyForCall(GenTreeCall* call)
+{
+ // Async2 calls return an async continuation argument in a separate
+ // register. Since we do not have a flexible representation for
+ // multiple definitions (multi-reg support is tied into promotion) we
+ // have to utilize a hack here to make it work. We expect the return
+ // value to be consumed by an upcoming ASYNC_CONTINUATION node, but we
+ // must take care not to overwrite the register until we get to that
+ // node. To accomplish that we mark the register as "busy until next
+ // kill" when we see the call's kill, and then we have
+ // ASYNC_CONTINUATION insert its own kill to free up the register
+ // again.
+ RefPosition* refPos = addKillForRegs(RBM_ASYNC_CONTINUATION_RET, currentLoc + 1);
+ refPos->busyUntilNextKill = true;
+}
diff --git a/src/coreclr/jit/lsraloongarch64.cpp b/src/coreclr/jit/lsraloongarch64.cpp
index 529e6d8127b670..3dd551bd2757bc 100644
--- a/src/coreclr/jit/lsraloongarch64.cpp
+++ b/src/coreclr/jit/lsraloongarch64.cpp
@@ -563,6 +563,17 @@ int LinearScan::BuildNode(GenTree* tree)
BuildDef(tree, RBM_EXCEPTION_OBJECT.GetIntRegSet());
break;
+ case GT_ASYNC_CONTINUATION:
+ srcCount = 0;
+ assert(dstCount == 1);
+ // We kill the continuation arg here to communicate to the
+ // selection phase that the argument is no longer busy. This is a
+ // hack to make sure we do not overwrite the continuation between
+ // the call and this node.
+ addKillForRegs(RBM_ASYNC_CONTINUATION_RET, currentLoc);
+ BuildDef(tree, RBM_ASYNC_CONTINUATION_RET.GetIntRegSet());
+ break;
+
case GT_INDEX_ADDR:
assert(dstCount == 1);
srcCount = BuildBinaryUses(tree->AsOp());
@@ -803,6 +814,11 @@ int LinearScan::BuildCall(GenTreeCall* call)
BuildKills(call, killMask);
}
+ if (call->IsAsync() && compiler->compIsAsync())
+ {
+ MarkAsyncContinuationBusyForCall(call);
+ }
+
// No args are placed in registers anymore.
placedArgRegs = RBM_NONE;
numPlacedArgLocals = 0;
diff --git a/src/coreclr/jit/lsrariscv64.cpp b/src/coreclr/jit/lsrariscv64.cpp
index 186927b290290e..dc98161c0d5024 100644
--- a/src/coreclr/jit/lsrariscv64.cpp
+++ b/src/coreclr/jit/lsrariscv64.cpp
@@ -735,6 +735,17 @@ int LinearScan::BuildNode(GenTree* tree)
BuildDef(tree, RBM_EXCEPTION_OBJECT.GetIntRegSet());
break;
+ case GT_ASYNC_CONTINUATION:
+ srcCount = 0;
+ assert(dstCount == 1);
+ // We kill the continuation arg here to communicate to the
+ // selection phase that the argument is no longer busy. This is a
+ // hack to make sure we do not overwrite the continuation between
+ // the call and this node.
+ addKillForRegs(RBM_ASYNC_CONTINUATION_RET, currentLoc);
+ BuildDef(tree, RBM_ASYNC_CONTINUATION_RET.GetIntRegSet());
+ break;
+
case GT_INDEX_ADDR:
assert(dstCount == 1);
srcCount = BuildBinaryUses(tree->AsOp());
@@ -987,6 +998,11 @@ int LinearScan::BuildCall(GenTreeCall* call)
BuildKills(call, killMask);
}
+ if (call->IsAsync() && compiler->compIsAsync())
+ {
+ MarkAsyncContinuationBusyForCall(call);
+ }
+
// No args are placed in registers anymore.
placedArgRegs = RBM_NONE;
numPlacedArgLocals = 0;
diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp
index 6f92d25d2b23ed..e00e4345bd05ba 100644
--- a/src/coreclr/jit/lsraxarch.cpp
+++ b/src/coreclr/jit/lsraxarch.cpp
@@ -627,6 +627,17 @@ int LinearScan::BuildNode(GenTree* tree)
BuildDef(tree, RBM_EXCEPTION_OBJECT.GetIntRegSet());
break;
+ case GT_ASYNC_CONTINUATION:
+ srcCount = 0;
+ assert(dstCount == 1);
+ // We kill the continuation arg here to communicate to the
+ // selection phase that the argument is no longer busy. This is a
+ // hack to make sure we do not overwrite the continuation between
+ // the call and this node.
+ addKillForRegs(RBM_ASYNC_CONTINUATION_RET, currentLoc);
+ BuildDef(tree, RBM_ASYNC_CONTINUATION_RET.GetIntRegSet());
+ break;
+
#if defined(FEATURE_EH_WINDOWS_X86)
case GT_END_LFIN:
srcCount = 0;
@@ -1386,6 +1397,11 @@ int LinearScan::BuildCall(GenTreeCall* call)
}
#endif // SWIFT_SUPPORT
+ if (call->IsAsync() && compiler->compIsAsync())
+ {
+ MarkAsyncContinuationBusyForCall(call);
+ }
+
// No args are placed in registers anymore.
placedArgRegs = RBM_NONE;
numPlacedArgLocals = 0;
diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp
index 79b9d2743c9375..f2eed98cdc65dd 100644
--- a/src/coreclr/jit/morph.cpp
+++ b/src/coreclr/jit/morph.cpp
@@ -709,6 +709,8 @@ const char* getWellKnownArgName(WellKnownArg arg)
return "VarArgsCookie";
case WellKnownArg::InstParam:
return "InstParam";
+ case WellKnownArg::AsyncContinuation:
+ return "AsyncContinuation";
case WellKnownArg::RetBuffer:
return "RetBuffer";
case WellKnownArg::PInvokeFrame:
@@ -4490,6 +4492,12 @@ GenTree* Compiler::fgMorphPotentialTailCall(GenTreeCall* call)
}
#endif
+ if (compIsAsync() != call->IsAsync())
+ {
+ failTailCall("Caller and callee do not agree on async2-ness");
+ return nullptr;
+ }
+
// We have to ensure to pass the incoming retValBuf as the
// outgoing one. Using a temp will not do as this function will
// not regain control to do the copy. This can happen when inlining
diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h
index ddee451a200bea..50e863382cff6e 100644
--- a/src/coreclr/jit/namedintrinsiclist.h
+++ b/src/coreclr/jit/namedintrinsiclist.h
@@ -103,6 +103,7 @@ enum NamedIntrinsic : unsigned short
NI_System_RuntimeType_get_TypeHandle,
NI_System_StubHelpers_GetStubContext,
NI_System_StubHelpers_NextCallReturnAddress,
+ NI_System_StubHelpers_Async2CallContinuation,
NI_Array_Address,
NI_Array_Get,
@@ -118,6 +119,11 @@ enum NamedIntrinsic : unsigned short
NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant,
NI_System_Runtime_CompilerServices_RuntimeHelpers_IsReferenceOrContainsReferences,
NI_System_Runtime_CompilerServices_RuntimeHelpers_GetMethodTable,
+ NI_System_Runtime_CompilerServices_RuntimeHelpers_AwaitAwaiterFromRuntimeAsync,
+ NI_System_Runtime_CompilerServices_RuntimeHelpers_UnsafeAwaitAwaiterFromRuntimeAsync,
+ NI_System_Runtime_CompilerServices_RuntimeHelpers_Await,
+ NI_System_Runtime_CompilerServices_RuntimeHelpers_SuspendAsync2,
+ NI_System_Runtime_CompilerServices_RuntimeHelpers_get_RuntimeAsyncViaJitGeneratedStateMachines,
NI_System_Runtime_CompilerServices_StaticsHelpers_VolatileReadAsByref,
@@ -150,6 +156,8 @@ enum NamedIntrinsic : unsigned short
NI_System_Threading_Interlocked_ExchangeAdd,
NI_System_Threading_Interlocked_MemoryBarrier,
+ NI_System_Threading_Tasks_Task_ConfigureAwait,
+
// These two are special marker IDs so that we still get the inlining profitability boost
NI_System_Numerics_Intrinsic,
NI_System_Runtime_Intrinsics_Intrinsic,
diff --git a/src/coreclr/jit/optcse.cpp b/src/coreclr/jit/optcse.cpp
index 9afb109802e85f..711e1d91e9a109 100644
--- a/src/coreclr/jit/optcse.cpp
+++ b/src/coreclr/jit/optcse.cpp
@@ -999,6 +999,11 @@ void Compiler::optValnumCSE_InitDataFlow()
}
}
+ if (compIsAsync())
+ {
+ optValnumCSE_SetUpAsyncByrefKills();
+ }
+
for (BasicBlock* const block : Blocks())
{
// If the block doesn't contains a call then skip it...
@@ -1082,6 +1087,112 @@ void Compiler::optValnumCSE_InitDataFlow()
#endif // DEBUG
}
+//---------------------------------------------------------------------------
+// optValnumCSE_SetUpAsyncByrefKills:
+// Compute kills because of async calls requiring byrefs not to be live
+// across them.
+//
+void Compiler::optValnumCSE_SetUpAsyncByrefKills()
+{
+ bool anyAsyncKills = false;
+ cseAsyncKillsMask = BitVecOps::MakeFull(cseLivenessTraits);
+ for (unsigned inx = 1; inx <= optCSECandidateCount; inx++)
+ {
+ CSEdsc* dsc = optCSEtab[inx - 1];
+ assert(dsc->csdIndex == inx);
+ bool isByRef = false;
+ if (dsc->csdTree->TypeIs(TYP_BYREF))
+ {
+ isByRef = true;
+ }
+ else if (dsc->csdTree->TypeIs(TYP_STRUCT))
+ {
+ ClassLayout* layout = dsc->csdTree->GetLayout(this);
+ isByRef = layout->HasGCByRef();
+ }
+
+ if (isByRef)
+ {
+ // We generate a bit pattern like: 1111111100111100 where there
+ // are 0s only for the byref CSEs.
+ BitVecOps::RemoveElemD(cseLivenessTraits, cseAsyncKillsMask, getCSEAvailBit(inx));
+ BitVecOps::RemoveElemD(cseLivenessTraits, cseAsyncKillsMask, getCSEAvailCrossCallBit(inx));
+ anyAsyncKills = true;
+ }
+ }
+
+ if (!anyAsyncKills)
+ {
+ return;
+ }
+
+ for (BasicBlock* block : Blocks())
+ {
+ Statement* asyncCallStmt = nullptr;
+ GenTree* asyncCall = nullptr;
+ // Find last async call in block
+ Statement* stmt = block->lastStmt();
+ if (stmt == nullptr)
+ {
+ continue;
+ }
+
+ while (asyncCall == nullptr)
+ {
+ if ((stmt->GetRootNode()->gtFlags & GTF_CALL) != 0)
+ {
+ for (GenTree* tree = stmt->GetRootNode(); tree != nullptr; tree = tree->gtPrev)
+ {
+ if (tree->IsCall() && tree->AsCall()->IsAsync())
+ {
+ asyncCallStmt = stmt;
+ asyncCall = tree;
+ break;
+ }
+ }
+ }
+
+ if (stmt == block->firstStmt())
+ break;
+
+ stmt = stmt->GetPrevStmt();
+ }
+
+ if (asyncCall == nullptr)
+ {
+ continue;
+ }
+
+ // This block has a suspension point. Make all BYREF CSEs unavailable.
+ BitVecOps::IntersectionD(cseLivenessTraits, block->bbCseGen, cseAsyncKillsMask);
+ BitVecOps::IntersectionD(cseLivenessTraits, block->bbCseOut, cseAsyncKillsMask);
+
+ // Now make all byref CSEs after the suspension point available.
+ Statement* curStmt = asyncCallStmt;
+ GenTree* curTree = asyncCall;
+ while (true)
+ {
+ do
+ {
+ if (IS_CSE_INDEX(curTree->gtCSEnum))
+ {
+ unsigned CSEnum = GET_CSE_INDEX(curTree->gtCSEnum);
+ BitVecOps::AddElemD(cseLivenessTraits, block->bbCseGen, getCSEAvailBit(CSEnum));
+ BitVecOps::AddElemD(cseLivenessTraits, block->bbCseOut, getCSEAvailBit(CSEnum));
+ }
+
+ curTree = curTree->gtNext;
+ } while (curTree != nullptr);
+
+ curStmt = curStmt->GetNextStmt();
+ if (curStmt == nullptr)
+ break;
+
+ curTree = curStmt->GetTreeList();
+ }
+ }
+}
+
/*****************************************************************************
*
* CSE Dataflow, so that all helper methods for dataflow are in a single place
@@ -1577,7 +1688,7 @@ void Compiler::optValnumCSE_Availability()
// kill all of the cseAvailCrossCallBit for each CSE whenever we see a GT_CALL (unless the call
// generates a CSE).
//
- if (tree->OperGet() == GT_CALL)
+ if (tree->OperIs(GT_CALL))
{
// Check for the common case of an already empty available_cses set
// and thus nothing needs to be killed
@@ -1595,6 +1706,12 @@ void Compiler::optValnumCSE_Availability()
//
BitVecOps::IntersectionD(cseLivenessTraits, available_cses, cseCallKillsMask);
+ // In async state machines, make all byref CSEs unavailable after suspension points.
+ if (tree->AsCall()->IsAsync() && compIsAsync())
+ {
+ BitVecOps::IntersectionD(cseLivenessTraits, available_cses, cseAsyncKillsMask);
+ }
+
if (isDef)
{
// We can have a GT_CALL that produces a CSE,
diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h
index eaaf6fc4ccbc9d..e9c15f2b14805a 100644
--- a/src/coreclr/jit/targetamd64.h
+++ b/src/coreclr/jit/targetamd64.h
@@ -540,6 +540,9 @@
#define REG_VALIDATE_INDIRECT_CALL_ADDR REG_RCX
#define REG_DISPATCH_INDIRECT_CALL_ADDR REG_RAX
+ #define REG_ASYNC_CONTINUATION_RET REG_RCX
+ #define RBM_ASYNC_CONTINUATION_RET RBM_RCX
+
// What sort of reloc do we use for [disp32] address mode
#define IMAGE_REL_BASED_DISP32 IMAGE_REL_BASED_REL32
diff --git a/src/coreclr/jit/targetarm.h b/src/coreclr/jit/targetarm.h
index 95cb19a2291a49..cd3d29fafef33d 100644
--- a/src/coreclr/jit/targetarm.h
+++ b/src/coreclr/jit/targetarm.h
@@ -247,6 +247,9 @@
#define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH)
#define REG_VALIDATE_INDIRECT_CALL_ADDR REG_R0
+ #define REG_ASYNC_CONTINUATION_RET REG_R2
+ #define RBM_ASYNC_CONTINUATION_RET RBM_R2
+
#define REG_FPBASE REG_R11
#define RBM_FPBASE RBM_R11
#define STR_FPBASE "r11"
diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h
index 3e1dec49b4778a..678a05e181e40d 100644
--- a/src/coreclr/jit/targetarm64.h
+++ b/src/coreclr/jit/targetarm64.h
@@ -263,6 +263,9 @@
#define REG_VALIDATE_INDIRECT_CALL_ADDR REG_R15
#define REG_DISPATCH_INDIRECT_CALL_ADDR REG_R9
+ #define REG_ASYNC_CONTINUATION_RET REG_R2
+ #define RBM_ASYNC_CONTINUATION_RET RBM_R2
+
#define REG_FPBASE REG_FP
#define RBM_FPBASE RBM_FP
#define STR_FPBASE "fp"
diff --git a/src/coreclr/jit/targetloongarch64.h b/src/coreclr/jit/targetloongarch64.h
index 452778c31963a0..d691f4c8fd1ec2 100644
--- a/src/coreclr/jit/targetloongarch64.h
+++ b/src/coreclr/jit/targetloongarch64.h
@@ -246,6 +246,9 @@
#define REG_VALIDATE_INDIRECT_CALL_ADDR REG_T3
#define REG_DISPATCH_INDIRECT_CALL_ADDR REG_T0
+ #define REG_ASYNC_CONTINUATION_RET REG_A2
+ #define RBM_ASYNC_CONTINUATION_RET RBM_A2
+
#define REG_FPBASE REG_FP
#define RBM_FPBASE RBM_FP
#define STR_FPBASE "fp"
diff --git a/src/coreclr/jit/targetriscv64.h b/src/coreclr/jit/targetriscv64.h
index e5dcded3d878f5..ee6c6d22260c7c 100644
--- a/src/coreclr/jit/targetriscv64.h
+++ b/src/coreclr/jit/targetriscv64.h
@@ -222,6 +222,9 @@
#define REG_VALIDATE_INDIRECT_CALL_ADDR REG_T3
#define REG_DISPATCH_INDIRECT_CALL_ADDR REG_T0
+ #define REG_ASYNC_CONTINUATION_RET REG_A2
+ #define RBM_ASYNC_CONTINUATION_RET RBM_A2
+
#define REG_FPBASE REG_FP
#define RBM_FPBASE RBM_FP
#define STR_FPBASE "fp"
diff --git a/src/coreclr/jit/targetx86.h b/src/coreclr/jit/targetx86.h
index dd63766d631ac7..e630a1ae842120 100644
--- a/src/coreclr/jit/targetx86.h
+++ b/src/coreclr/jit/targetx86.h
@@ -294,6 +294,9 @@
#define RBM_VALIDATE_INDIRECT_CALL_TRASH (RBM_INT_CALLEE_TRASH & ~RBM_ECX)
#define REG_VALIDATE_INDIRECT_CALL_ADDR REG_ECX
+ #define REG_ASYNC_CONTINUATION_RET REG_ECX
+ #define RBM_ASYNC_CONTINUATION_RET RBM_ECX
+
#define REG_FPBASE REG_EBP
#define RBM_FPBASE RBM_EBP
#define STR_FPBASE "ebp"
diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp
index e240bd47fc50b2..e23b532427c15e 100644
--- a/src/coreclr/jit/valuenum.cpp
+++ b/src/coreclr/jit/valuenum.cpp
@@ -10369,8 +10369,8 @@ static genTreeOps genTreeOpsIllegalAsVNFunc[] = {GT_IND, // When we do heap memo
GT_NOP,
// These control-flow operations need no values.
- GT_JTRUE, GT_RETURN, GT_SWITCH, GT_RETFILT, GT_CKFINITE,
- GT_SWIFT_ERROR_RET};
+ GT_JTRUE, GT_RETURN, GT_RETURN_SUSPEND, GT_SWITCH, GT_RETFILT,
+ GT_CKFINITE, GT_SWIFT_ERROR_RET};
void ValueNumStore::ValidateValueNumStoreStatics()
{
@@ -12255,9 +12255,9 @@ void Compiler::fgValueNumberTree(GenTree* tree)
break;
case GT_CATCH_ARG:
+ case GT_ASYNC_CONTINUATION:
case GT_SWIFT_ERROR:
- // We know nothing about the value of a caught expression.
- // We also know nothing about the error register's value post-Swift call.
+ // We know nothing about the value of these.
tree->gtVNPair.SetBoth(vnStore->VNForExpr(compCurBB, tree->TypeGet()));
break;
@@ -12627,6 +12627,7 @@ void Compiler::fgValueNumberTree(GenTree* tree)
case GT_SWITCH:
case GT_RETURN:
case GT_RETFILT:
+ case GT_RETURN_SUSPEND:
case GT_NULLCHECK:
if (tree->gtGetOp1() != nullptr)
{
diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/CompatibilitySuppressions.xml b/src/coreclr/nativeaot/System.Private.CoreLib/src/CompatibilitySuppressions.xml
index 2de459555d114d..24c02cd1d575d6 100644
--- a/src/coreclr/nativeaot/System.Private.CoreLib/src/CompatibilitySuppressions.xml
+++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/CompatibilitySuppressions.xml
@@ -1,6 +1,10 @@
+
+ CP0001
+ T:Internal.Console
+ CP0001T:Internal.Metadata.NativeFormat.ArraySignature
@@ -53,14 +57,6 @@
CP0001T:Internal.Metadata.NativeFormat.ConstantBooleanValueHandle
-
- CP0001
- T:Internal.Metadata.NativeFormat.ConstantEnumValue
-
-
- CP0001
- T:Internal.Metadata.NativeFormat.ConstantEnumValueHandle
- CP0001T:Internal.Metadata.NativeFormat.ConstantByteArray
@@ -117,6 +113,14 @@
CP0001T:Internal.Metadata.NativeFormat.ConstantEnumArrayHandle
+
+ CP0001
+ T:Internal.Metadata.NativeFormat.ConstantEnumValue
+
+
+ CP0001
+ T:Internal.Metadata.NativeFormat.ConstantEnumValueHandle
+ CP0001T:Internal.Metadata.NativeFormat.ConstantHandleArray
@@ -725,6 +729,10 @@
CP0001T:Internal.TypeSystem.LockFreeReaderHashtable`2
+
+ CP0001
+ T:Internal.TypeSystem.LockFreeReaderHashtableOfPointers`2
+ CP0001T:System.Diagnostics.DebugAnnotations
@@ -735,11 +743,11 @@
CP0001
- T:System.MDArray
+ T:System.FieldHandleInfoCP0001
- T:System.FieldHandleInfo
+ T:System.MDArrayCP0001
@@ -810,8 +818,16 @@
T:System.Runtime.CompilerServices.StaticClassConstructionContext
- CP0001
- T:Internal.TypeSystem.LockFreeReaderHashtableOfPointers`2
+ CP0002
+ F:System.Resources.ResourceManager.BaseNameField
+
+
+ CP0002
+ F:System.Resources.ResourceSet.Reader
+
+
+ CP0002
+ M:System.Diagnostics.DiagnosticMethodInfo.#ctor(System.String,System.String,System.String)CP0002
@@ -819,10 +835,18 @@
CP0002
- M:System.Threading.Lock.#ctor(System.Boolean)
+ M:System.String.Trim(System.ReadOnlySpan{System.Char})CP0002
- M:System.Diagnostics.DiagnosticMethodInfo.#ctor(System.String,System.String,System.String)
+ M:System.String.TrimEnd(System.ReadOnlySpan{System.Char})
+
+
+ CP0002
+ M:System.String.TrimStart(System.ReadOnlySpan{System.Char})
+
+
+ CP0002
+ M:System.Threading.Lock.#ctor(System.Boolean)
\ No newline at end of file
diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs
index 04bca41e476671..050bbb17082715 100644
--- a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs
+++ b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs
@@ -96,6 +96,7 @@ which is the right helper to use to allocate an object of a given type. */
CORINFO_HELP_THROW, // Throw an exception object
CORINFO_HELP_RETHROW, // Rethrow the currently active exception
+ CORINFO_HELP_THROWEXACT, // Throw an exception object, preserving stack trace
CORINFO_HELP_USER_BREAKPOINT, // For a user program to break to the debugger
CORINFO_HELP_RNGCHKFAIL, // array bounds check failed
CORINFO_HELP_OVERFLOW, // throw an overflow exception
@@ -269,6 +270,7 @@ which is the right helper to use to allocate an object of a given type. */
CORINFO_HELP_PATCHPOINT, // Notify runtime that code has reached a patchpoint
CORINFO_HELP_PARTIAL_COMPILATION_PATCHPOINT, // Notify runtime that code has reached a part of the method that wasn't originally jitted.
+ CORINFO_HELP_RESUME_OSR, // Resume in an OSR version of the code for the specified IL offset
CORINFO_HELP_CLASSPROFILE32, // Update 32-bit class profile for a call site
CORINFO_HELP_CLASSPROFILE64, // Update 64-bit class profile for a call site
@@ -284,6 +286,10 @@ which is the right helper to use to allocate an object of a given type. */
CORINFO_HELP_VALIDATE_INDIRECT_CALL, // CFG: Validate function pointer
CORINFO_HELP_DISPATCH_INDIRECT_CALL, // CFG: Validate and dispatch to pointer
+ CORINFO_HELP_ALLOC_CONTINUATION,
+ CORINFO_HELP_ALLOC_CONTINUATION_METHOD,
+ CORINFO_HELP_ALLOC_CONTINUATION_CLASS,
+
CORINFO_HELP_COUNT,
}
}
diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs
index c4703ddf4516c7..495f9b85512261 100644
--- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs
+++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs
@@ -3359,6 +3359,11 @@ private void getEEInfo(ref CORINFO_EE_INFO pEEInfoOut)
pEEInfoOut.osType = TargetToOs(_compilation.NodeFactory.Target);
}
+ private void getAsync2Info(ref CORINFO_ASYNC2_INFO pAsync2InfoOut)
+ {
+ throw new NotImplementedException();
+ }
+
private mdToken getMethodDefFromMethod(CORINFO_METHOD_STRUCT_* hMethod)
{
MethodDesc method = HandleToObject(hMethod);
@@ -3686,6 +3691,13 @@ private bool getTailCallHelpers(ref CORINFO_RESOLVED_TOKEN callToken, CORINFO_SI
#endif
}
+#pragma warning disable CA1822 // Mark members as static
+ private CORINFO_METHOD_STRUCT_* getAsyncResumptionStub()
+#pragma warning restore CA1822 // Mark members as static
+ {
+ return null;
+ }
+
private byte[] _code;
private byte[] _coldCode;
private int _codeAlignment;
diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs
index 91df884c58272c..b760d4eaa71eac 100644
--- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs
+++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs
@@ -1784,6 +1784,20 @@ private static void _getEEInfo(IntPtr thisHandle, IntPtr* ppException, CORINFO_E
}
}
+ [UnmanagedCallersOnly]
+ private static void _getAsync2Info(IntPtr thisHandle, IntPtr* ppException, CORINFO_ASYNC2_INFO* pAsync2InfoOut)
+ {
+ var _this = GetThis(thisHandle);
+ try
+ {
+ _this.getAsync2Info(ref *pAsync2InfoOut);
+ }
+ catch (Exception ex)
+ {
+ *ppException = _this.AllocException(ex);
+ }
+ }
+
[UnmanagedCallersOnly]
private static mdToken _getMethodDefFromMethod(IntPtr thisHandle, IntPtr* ppException, CORINFO_METHOD_STRUCT_* hMethod)
{
@@ -2329,6 +2343,21 @@ private static byte _getTailCallHelpers(IntPtr thisHandle, IntPtr* ppException,
}
}
+ [UnmanagedCallersOnly]
+ private static CORINFO_METHOD_STRUCT_* _getAsyncResumptionStub(IntPtr thisHandle, IntPtr* ppException)
+ {
+ var _this = GetThis(thisHandle);
+ try
+ {
+ return _this.getAsyncResumptionStub();
+ }
+ catch (Exception ex)
+ {
+ *ppException = _this.AllocException(ex);
+ return default;
+ }
+ }
+
[UnmanagedCallersOnly]
private static byte _convertPInvokeCalliToCall(IntPtr thisHandle, IntPtr* ppException, CORINFO_RESOLVED_TOKEN* pResolvedToken, byte mustConvert)
{
@@ -2623,7 +2652,7 @@ private static uint _getJitFlags(IntPtr thisHandle, IntPtr* ppException, CORJIT_
private static IntPtr GetUnmanagedCallbacks()
{
- void** callbacks = (void**)Marshal.AllocCoTaskMem(sizeof(IntPtr) * 177);
+ void** callbacks = (void**)Marshal.AllocCoTaskMem(sizeof(IntPtr) * 179);
callbacks[0] = (delegate* unmanaged)&_isIntrinsic;
callbacks[1] = (delegate* unmanaged)&_notifyMethodInfoUsage;
@@ -2745,63 +2774,65 @@ private static IntPtr GetUnmanagedCallbacks()
callbacks[117] = (delegate* unmanaged)&_runWithErrorTrap;
callbacks[118] = (delegate* unmanaged)&_runWithSPMIErrorTrap;
callbacks[119] = (delegate* unmanaged)&_getEEInfo;
- callbacks[120] = (delegate* unmanaged)&_getMethodDefFromMethod;
- callbacks[121] = (delegate* unmanaged)&_printMethodName;
- callbacks[122] = (delegate* unmanaged)&_getMethodNameFromMetadata;
- callbacks[123] = (delegate* unmanaged)&_getMethodHash;
- callbacks[124] = (delegate* unmanaged)&_getSystemVAmd64PassStructInRegisterDescriptor;
- callbacks[125] = (delegate* unmanaged)&_getSwiftLowering;
- callbacks[126] = (delegate* unmanaged)&_getFpStructLowering;
- callbacks[127] = (delegate* unmanaged)&_getThreadTLSIndex;
- callbacks[128] = (delegate* unmanaged)&_getAddrOfCaptureThreadGlobal;
- callbacks[129] = (delegate* unmanaged)&_getHelperFtn;
- callbacks[130] = (delegate* unmanaged)&_getFunctionEntryPoint;
- callbacks[131] = (delegate* unmanaged)&_getFunctionFixedEntryPoint;
- callbacks[132] = (delegate* unmanaged)&_getMethodSync;
- callbacks[133] = (delegate* unmanaged)&_getLazyStringLiteralHelper;
- callbacks[134] = (delegate* unmanaged)&_embedModuleHandle;
- callbacks[135] = (delegate* unmanaged)&_embedClassHandle;
- callbacks[136] = (delegate* unmanaged)&_embedMethodHandle;
- callbacks[137] = (delegate* unmanaged)&_embedFieldHandle;
- callbacks[138] = (delegate* unmanaged)&_embedGenericHandle;
- callbacks[139] = (delegate* unmanaged)&_getLocationOfThisType;
- callbacks[140] = (delegate* unmanaged)&_getAddressOfPInvokeTarget;
- callbacks[141] = (delegate* unmanaged)&_GetCookieForPInvokeCalliSig;
- callbacks[142] = (delegate* unmanaged)&_canGetCookieForPInvokeCalliSig;
- callbacks[143] = (delegate* unmanaged)&_getJustMyCodeHandle;
- callbacks[144] = (delegate* unmanaged)&_GetProfilingHandle;
- callbacks[145] = (delegate* unmanaged)&_getCallInfo;
- callbacks[146] = (delegate* unmanaged)&_getStaticFieldContent;
- callbacks[147] = (delegate* unmanaged)&_getObjectContent;
- callbacks[148] = (delegate* unmanaged)&_getStaticFieldCurrentClass;
- callbacks[149] = (delegate* unmanaged)&_getVarArgsHandle;
- callbacks[150] = (delegate* unmanaged)&_canGetVarArgsHandle;
- callbacks[151] = (delegate* unmanaged)&_constructStringLiteral;
- callbacks[152] = (delegate* unmanaged)&_emptyStringLiteral;
- callbacks[153] = (delegate* unmanaged)&_getFieldThreadLocalStoreID;
- callbacks[154] = (delegate* unmanaged)&_GetDelegateCtor;
- callbacks[155] = (delegate* unmanaged)&_MethodCompileComplete;
- callbacks[156] = (delegate* unmanaged)&_getTailCallHelpers;
- callbacks[157] = (delegate* unmanaged)&_convertPInvokeCalliToCall;
- callbacks[158] = (delegate* unmanaged)&_notifyInstructionSetUsage;
- callbacks[159] = (delegate* unmanaged)&_updateEntryPointForTailCall;
- callbacks[160] = (delegate* unmanaged)&_allocMem;
- callbacks[161] = (delegate* unmanaged)&_reserveUnwindInfo;
- callbacks[162] = (delegate* unmanaged)&_allocUnwindInfo;
- callbacks[163] = (delegate* unmanaged)&_allocGCInfo;
- callbacks[164] = (delegate* unmanaged)&_setEHcount;
- callbacks[165] = (delegate* unmanaged)&_setEHinfo;
- callbacks[166] = (delegate* unmanaged)&_logMsg;
- callbacks[167] = (delegate* unmanaged)&_doAssert;
- callbacks[168] = (delegate* unmanaged)&_reportFatalError;
- callbacks[169] = (delegate* unmanaged)&_getPgoInstrumentationResults;
- callbacks[170] = (delegate* unmanaged)&_allocPgoInstrumentationBySchema;
- callbacks[171] = (delegate* unmanaged)&_recordCallSite;
- callbacks[172] = (delegate* unmanaged)&_recordRelocation;
- callbacks[173] = (delegate* unmanaged)&_getRelocTypeHint;
- callbacks[174] = (delegate* unmanaged)&_getExpectedTargetArchitecture;
- callbacks[175] = (delegate* unmanaged)&_getJitFlags;
- callbacks[176] = (delegate* unmanaged)&_getSpecialCopyHelper;
+ callbacks[120] = (delegate* unmanaged)&_getAsync2Info;
+ callbacks[121] = (delegate* unmanaged)&_getMethodDefFromMethod;
+ callbacks[122] = (delegate* unmanaged)&_printMethodName;
+ callbacks[123] = (delegate* unmanaged)&_getMethodNameFromMetadata;
+ callbacks[124] = (delegate* unmanaged)&_getMethodHash;
+ callbacks[125] = (delegate* unmanaged)&_getSystemVAmd64PassStructInRegisterDescriptor;
+ callbacks[126] = (delegate* unmanaged)&_getSwiftLowering;
+ callbacks[127] = (delegate* unmanaged)&_getFpStructLowering;
+ callbacks[128] = (delegate* unmanaged)&_getThreadTLSIndex;
+ callbacks[129] = (delegate* unmanaged)&_getAddrOfCaptureThreadGlobal;
+ callbacks[130] = (delegate* unmanaged)&_getHelperFtn;
+ callbacks[131] = (delegate* unmanaged)&_getFunctionEntryPoint;
+ callbacks[132] = (delegate* unmanaged)&_getFunctionFixedEntryPoint;
+ callbacks[133] = (delegate* unmanaged)&_getMethodSync;
+ callbacks[134] = (delegate* unmanaged)&_getLazyStringLiteralHelper;
+ callbacks[135] = (delegate* unmanaged)&_embedModuleHandle;
+ callbacks[136] = (delegate* unmanaged)&_embedClassHandle;
+ callbacks[137] = (delegate* unmanaged)&_embedMethodHandle;
+ callbacks[138] = (delegate* unmanaged)&_embedFieldHandle;
+ callbacks[139] = (delegate* unmanaged)&_embedGenericHandle;
+ callbacks[140] = (delegate* unmanaged)&_getLocationOfThisType;
+ callbacks[141] = (delegate* unmanaged)&_getAddressOfPInvokeTarget;
+ callbacks[142] = (delegate* unmanaged)&_GetCookieForPInvokeCalliSig;
+ callbacks[143] = (delegate* unmanaged)&_canGetCookieForPInvokeCalliSig;
+ callbacks[144] = (delegate* unmanaged)&_getJustMyCodeHandle;
+ callbacks[145] = (delegate* unmanaged)&_GetProfilingHandle;
+ callbacks[146] = (delegate* unmanaged)&_getCallInfo;
+ callbacks[147] = (delegate* unmanaged)&_getStaticFieldContent;
+ callbacks[148] = (delegate* unmanaged)&_getObjectContent;
+ callbacks[149] = (delegate* unmanaged)&_getStaticFieldCurrentClass;
+ callbacks[150] = (delegate* unmanaged)&_getVarArgsHandle;
+ callbacks[151] = (delegate* unmanaged)&_canGetVarArgsHandle;
+ callbacks[152] = (delegate* unmanaged)&_constructStringLiteral;
+ callbacks[153] = (delegate* unmanaged)&_emptyStringLiteral;
+ callbacks[154] = (delegate* unmanaged)&_getFieldThreadLocalStoreID;
+ callbacks[155] = (delegate* unmanaged)&_GetDelegateCtor;
+ callbacks[156] = (delegate* unmanaged)&_MethodCompileComplete;
+ callbacks[157] = (delegate* unmanaged)&_getTailCallHelpers;
+ callbacks[158] = (delegate* unmanaged)&_getAsyncResumptionStub;
+ callbacks[159] = (delegate* unmanaged)&_convertPInvokeCalliToCall;
+ callbacks[160] = (delegate* unmanaged)&_notifyInstructionSetUsage;
+ callbacks[161] = (delegate* unmanaged)&_updateEntryPointForTailCall;
+ callbacks[162] = (delegate* unmanaged)&_allocMem;
+ callbacks[163] = (delegate* unmanaged)&_reserveUnwindInfo;
+ callbacks[164] = (delegate* unmanaged)&_allocUnwindInfo;
+ callbacks[165] = (delegate* unmanaged)&_allocGCInfo;
+ callbacks[166] = (delegate* unmanaged)&_setEHcount;
+ callbacks[167] = (delegate* unmanaged)&_setEHinfo;
+ callbacks[168] = (delegate* unmanaged)&_logMsg;
+ callbacks[169] = (delegate* unmanaged)&_doAssert;
+ callbacks[170] = (delegate* unmanaged)&_reportFatalError;
+ callbacks[171] = (delegate* unmanaged)&_getPgoInstrumentationResults;
+ callbacks[172] = (delegate* unmanaged)&_allocPgoInstrumentationBySchema;
+ callbacks[173] = (delegate* unmanaged)&_recordCallSite;
+ callbacks[174] = (delegate* unmanaged)&_recordRelocation;
+ callbacks[175] = (delegate* unmanaged)&_getRelocTypeHint;
+ callbacks[176] = (delegate* unmanaged)&_getExpectedTargetArchitecture;
+ callbacks[177] = (delegate* unmanaged)&_getJitFlags;
+ callbacks[178] = (delegate* unmanaged)&_getSpecialCopyHelper;
return (IntPtr)callbacks;
}
diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs b/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs
index 124df22bdc7864..05e6c372766c88 100644
--- a/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs
+++ b/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs
@@ -873,6 +873,18 @@ public struct InlinedCallFrameInfo
public CORINFO_OS osType;
}
+ public unsafe struct CORINFO_ASYNC2_INFO
+ {
+ // Class handle for System.Runtime.CompilerServices.Continuation
+ public CORINFO_CLASS_STRUCT_* continuationClsHnd;
+ // 'Next' field
+ public CORINFO_FIELD_STRUCT_* continuationNextFldHnd;
+ // 'Data' field
+ public CORINFO_FIELD_STRUCT_* continuationDataFldHnd;
+ // 'GCData' field
+ public CORINFO_FIELD_STRUCT_* continuationGCDataFldHnd;
+ }
+
// Flags passed from JIT to runtime.
public enum CORINFO_GET_TAILCALL_HELPERS_FLAGS
{
diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt
index 3aaa80673334f4..860b38f505e1fc 100644
--- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt
+++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt
@@ -82,6 +82,7 @@ CORINFO_SIG_INFO*
CORINFO_RESOLVED_TOKEN*,ref CORINFO_RESOLVED_TOKEN
CORINFO_RESOLVED_TOKEN_PTR,CORINFO_RESOLVED_TOKEN*,CORINFO_RESOLVED_TOKEN*,CORINFO_RESOLVED_TOKEN*
CORINFO_EE_INFO*,ref CORINFO_EE_INFO
+CORINFO_ASYNC2_INFO*,ref CORINFO_ASYNC2_INFO
CORINFO_TAILCALL_HELPERS*,ref CORINFO_TAILCALL_HELPERS
CORINFO_SWIFT_LOWERING*,ref CORINFO_SWIFT_LOWERING
CORINFO_FPSTRUCT_LOWERING*,ref CORINFO_FPSTRUCT_LOWERING
@@ -284,6 +285,7 @@ FUNCTIONS
[ManualNativeWrapper] bool runWithErrorTrap(ICorJitInfo::errorTrapFunction function, void* parameter);
[ManualNativeWrapper] bool runWithSPMIErrorTrap(ICorJitInfo::errorTrapFunction function, void* parameter);
void getEEInfo(CORINFO_EE_INFO* pEEInfoOut);
+ void getAsync2Info(CORINFO_ASYNC2_INFO* pAsync2InfoOut);
mdMethodDef getMethodDefFromMethod(CORINFO_METHOD_HANDLE hMethod);
size_t printMethodName(CORINFO_METHOD_HANDLE ftn, char* buffer, size_t bufferSize, size_t* pRequiredBufferSize)
const char* getMethodNameFromMetadata(CORINFO_METHOD_HANDLE ftn, const char **className, const char **namespaceName, const char **enclosingClassNames, size_t maxEnclosingClassNames);
@@ -321,6 +323,7 @@ FUNCTIONS
CORINFO_METHOD_HANDLE GetDelegateCtor(CORINFO_METHOD_HANDLE methHnd, CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE targetMethodHnd, DelegateCtorArgs * pCtorData);
void MethodCompileComplete(CORINFO_METHOD_HANDLE methHnd);
bool getTailCallHelpers(CORINFO_RESOLVED_TOKEN* callToken, CORINFO_SIG_INFO* sig, CORINFO_GET_TAILCALL_HELPERS_FLAGS flags, CORINFO_TAILCALL_HELPERS* pResult);
+ CORINFO_METHOD_HANDLE getAsyncResumptionStub();
bool convertPInvokeCalliToCall(CORINFO_RESOLVED_TOKEN * pResolvedToken, bool mustConvert);
bool notifyInstructionSetUsage(CORINFO_InstructionSet instructionSet,bool supportEnabled);
void updateEntryPointForTailCall(CORINFO_CONST_LOOKUP* entryPoint);
diff --git a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h
index a1a6122037d27a..9117a99d6a2b3f 100644
--- a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h
+++ b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h
@@ -131,6 +131,7 @@ struct JitInterfaceCallbacks
bool (* runWithErrorTrap)(void * thisHandle, CorInfoExceptionClass** ppException, ICorJitInfo::errorTrapFunction function, void* parameter);
bool (* runWithSPMIErrorTrap)(void * thisHandle, CorInfoExceptionClass** ppException, ICorJitInfo::errorTrapFunction function, void* parameter);
void (* getEEInfo)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_EE_INFO* pEEInfoOut);
+ void (* getAsync2Info)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_ASYNC2_INFO* pAsync2InfoOut);
mdMethodDef (* getMethodDefFromMethod)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE hMethod);
size_t (* printMethodName)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE ftn, char* buffer, size_t bufferSize, size_t* pRequiredBufferSize);
const char* (* getMethodNameFromMetadata)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE ftn, const char** className, const char** namespaceName, const char** enclosingClassNames, size_t maxEnclosingClassNames);
@@ -168,6 +169,7 @@ struct JitInterfaceCallbacks
CORINFO_METHOD_HANDLE (* GetDelegateCtor)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE methHnd, CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE targetMethodHnd, DelegateCtorArgs* pCtorData);
void (* MethodCompileComplete)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE methHnd);
bool (* getTailCallHelpers)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* callToken, CORINFO_SIG_INFO* sig, CORINFO_GET_TAILCALL_HELPERS_FLAGS flags, CORINFO_TAILCALL_HELPERS* pResult);
+ CORINFO_METHOD_HANDLE (* getAsyncResumptionStub)(void * thisHandle, CorInfoExceptionClass** ppException);
bool (* convertPInvokeCalliToCall)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool mustConvert);
bool (* notifyInstructionSetUsage)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_InstructionSet instructionSet, bool supportEnabled);
void (* updateEntryPointForTailCall)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_CONST_LOOKUP* entryPoint);
@@ -1360,6 +1362,14 @@ class JitInterfaceWrapper : public ICorJitInfo
if (pException != nullptr) throw pException;
}
+ virtual void getAsync2Info(
+ CORINFO_ASYNC2_INFO* pAsync2InfoOut)
+{
+ CorInfoExceptionClass* pException = nullptr;
+ _callbacks->getAsync2Info(_thisHandle, &pException, pAsync2InfoOut);
+ if (pException != nullptr) throw pException;
+}
+
virtual mdMethodDef getMethodDefFromMethod(
CORINFO_METHOD_HANDLE hMethod)
{
@@ -1734,6 +1744,14 @@ class JitInterfaceWrapper : public ICorJitInfo
return temp;
}
+ virtual CORINFO_METHOD_HANDLE getAsyncResumptionStub()
+{
+ CorInfoExceptionClass* pException = nullptr;
+ CORINFO_METHOD_HANDLE temp = _callbacks->getAsyncResumptionStub(_thisHandle, &pException);
+ if (pException != nullptr) throw pException;
+ return temp;
+}
+
virtual bool convertPInvokeCalliToCall(
CORINFO_RESOLVED_TOKEN* pResolvedToken,
bool mustConvert)
diff --git a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h
index 18fd918a7a5cea..347a9b434c78b1 100644
--- a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h
+++ b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h
@@ -193,6 +193,18 @@ struct Agnostic_CORINFO_EE_INFO
DWORD osType;
};
+struct Agnostic_CORINFO_ASYNC2_INFO
+{
+ DWORDLONG continuationClsHnd;
+ DWORDLONG continuationNextFldHnd;
+ DWORDLONG continuationResumeFldHnd;
+ DWORDLONG continuationStateFldHnd;
+ DWORDLONG continuationFlagsFldHnd;
+ DWORDLONG continuationDataFldHnd;
+ DWORDLONG continuationGCDataFldHnd;
+ DWORD continuationsNeedMethodHandle;
+};
+
struct Agnostic_GetOSRInfo
{
DWORD index;
diff --git a/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h b/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h
index 50e9720aefaa2a..407ab9bd0303a2 100644
--- a/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h
+++ b/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h
@@ -79,6 +79,7 @@ LWM(GetDefaultEqualityComparerClass, DWORDLONG, DWORDLONG)
LWM(GetSZArrayHelperEnumeratorClass, DWORDLONG, DWORDLONG)
LWM(GetDelegateCtor, Agnostic_GetDelegateCtorIn, Agnostic_GetDelegateCtorOut)
LWM(GetEEInfo, DWORD, Agnostic_CORINFO_EE_INFO)
+LWM(GetAsync2Info, DWORD, Agnostic_CORINFO_ASYNC2_INFO)
LWM(GetEHinfo, DLD, Agnostic_CORINFO_EH_CLAUSE)
LWM(GetStaticFieldContent, DLDDD, DD)
LWM(GetObjectContent, DLDD, DD)
@@ -130,6 +131,7 @@ LWM(GetSystemVAmd64PassStructInRegisterDescriptor, DWORDLONG, Agnostic_GetSystem
LWM(GetSwiftLowering, DWORDLONG, Agnostic_GetSwiftLowering)
LWM(GetFpStructLowering, DWORDLONG, Agnostic_GetFpStructLowering)
LWM(GetTailCallHelpers, Agnostic_GetTailCallHelpers, Agnostic_CORINFO_TAILCALL_HELPERS)
+LWM(GetAsyncResumptionStub, DWORD, DWORDLONG)
LWM(UpdateEntryPointForTailCall, Agnostic_CORINFO_CONST_LOOKUP, Agnostic_CORINFO_CONST_LOOKUP)
LWM(GetSpecialCopyHelper, DWORDLONG, DWORDLONG)
LWM(GetThreadTLSIndex, DWORD, DLD)
diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp
index 0db57a809952d8..ca85d79f0eb677 100644
--- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp
+++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp
@@ -4459,6 +4459,48 @@ void MethodContext::repGetEEInfo(CORINFO_EE_INFO* pEEInfoOut)
pEEInfoOut->osType = (CORINFO_OS)value.osType;
}
+void MethodContext::recGetAsync2Info(const CORINFO_ASYNC2_INFO* pAsync2Info)
+{
+ if (GetAsync2Info == nullptr)
+ GetAsync2Info = new LightWeightMap();
+
+ Agnostic_CORINFO_ASYNC2_INFO value;
+ ZeroMemory(&value, sizeof(value));
+
+ value.continuationClsHnd = CastHandle(pAsync2Info->continuationClsHnd);
+ value.continuationNextFldHnd = CastHandle(pAsync2Info->continuationNextFldHnd);
+ value.continuationResumeFldHnd = CastHandle(pAsync2Info->continuationResumeFldHnd);
+ value.continuationStateFldHnd = CastHandle(pAsync2Info->continuationStateFldHnd);
+ value.continuationFlagsFldHnd = CastHandle(pAsync2Info->continuationFlagsFldHnd);
+ value.continuationDataFldHnd = CastHandle(pAsync2Info->continuationDataFldHnd);
+ value.continuationGCDataFldHnd = CastHandle(pAsync2Info->continuationGCDataFldHnd);
+ value.continuationsNeedMethodHandle = pAsync2Info->continuationsNeedMethodHandle ? 1 : 0;
+
+ GetAsync2Info->Add(0, value);
+ DEBUG_REC(dmpGetAsync2Info(0, value));
+}
+void MethodContext::dmpGetAsync2Info(DWORD key, const Agnostic_CORINFO_ASYNC2_INFO& value)
+{
+ printf("GetAsync2Info key %u value contClsHnd-%016" PRIX64 " contNextFldHnd-%016" PRIX64 " contResumeFldHnd-%016" PRIX64
+ " contStateFldHnd-%016" PRIX64 " contFlagsFldHnd-%016" PRIX64 " contDataFldHnd-%016" PRIX64 " contGCDataFldHnd-%016" PRIX64 " contsNeedMethodHandle-%d",
+ key, value.continuationClsHnd, value.continuationNextFldHnd, value.continuationResumeFldHnd,
+ value.continuationStateFldHnd, value.continuationFlagsFldHnd, value.continuationDataFldHnd,
+ value.continuationGCDataFldHnd, value.continuationsNeedMethodHandle);
+}
+void MethodContext::repGetAsync2Info(CORINFO_ASYNC2_INFO* pAsync2InfoOut)
+{
+ Agnostic_CORINFO_ASYNC2_INFO value = LookupByKeyOrMissNoMessage(GetAsync2Info, 0);
+ pAsync2InfoOut->continuationClsHnd = (CORINFO_CLASS_HANDLE)value.continuationClsHnd;
+ pAsync2InfoOut->continuationNextFldHnd = (CORINFO_FIELD_HANDLE)value.continuationNextFldHnd;
+ pAsync2InfoOut->continuationResumeFldHnd = (CORINFO_FIELD_HANDLE)value.continuationResumeFldHnd;
+ pAsync2InfoOut->continuationStateFldHnd = (CORINFO_FIELD_HANDLE)value.continuationStateFldHnd;
+ pAsync2InfoOut->continuationFlagsFldHnd = (CORINFO_FIELD_HANDLE)value.continuationFlagsFldHnd;
+ pAsync2InfoOut->continuationDataFldHnd = (CORINFO_FIELD_HANDLE)value.continuationDataFldHnd;
+ pAsync2InfoOut->continuationGCDataFldHnd = (CORINFO_FIELD_HANDLE)value.continuationGCDataFldHnd;
+ pAsync2InfoOut->continuationsNeedMethodHandle = value.continuationsNeedMethodHandle != 0;
+ DEBUG_REP(dmpGetAsync2Info(0, value));
+}
+
void MethodContext::recGetGSCookie(GSCookie* pCookieVal, GSCookie** ppCookieVal)
{
if (GetGSCookie == nullptr)
@@ -6941,6 +6983,25 @@ bool MethodContext::repGetTailCallHelpers(
return true;
}
+
+void MethodContext::recGetAsyncResumptionStub(CORINFO_METHOD_HANDLE hnd)
+{
+ if (GetAsyncResumptionStub == nullptr)
+ GetAsyncResumptionStub = new LightWeightMap();
+
+ GetAsyncResumptionStub->Add(0, CastHandle(hnd));
+ DEBUG_REC(dmpGetAsyncResumptionStub(CastHandle(hnd)));
+}
+void MethodContext::dmpGetAsyncResumptionStub(DWORD key, DWORDLONG hnd)
+{
+ printf("GetAsyncResumptionStub key-%u, value-%016" PRIX64, key, hnd);
+}
+CORINFO_METHOD_HANDLE MethodContext::repGetAsyncResumptionStub()
+{
+ DWORDLONG hnd = LookupByKeyOrMissNoMessage(GetAsyncResumptionStub, 0);
+ return (CORINFO_METHOD_HANDLE)hnd;
+}
+
void MethodContext::recUpdateEntryPointForTailCall(
const CORINFO_CONST_LOOKUP& origEntryPoint,
const CORINFO_CONST_LOOKUP& newEntryPoint)
diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h
index dd9f4abc0f0d4f..9a124380731859 100644
--- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h
+++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h
@@ -574,6 +574,10 @@ class MethodContext
void dmpGetEEInfo(DWORD key, const Agnostic_CORINFO_EE_INFO& value);
void repGetEEInfo(CORINFO_EE_INFO* pEEInfoOut);
+ void recGetAsync2Info(const CORINFO_ASYNC2_INFO* pAsync2Info);
+ void dmpGetAsync2Info(DWORD key, const Agnostic_CORINFO_ASYNC2_INFO& value);
+ void repGetAsync2Info(CORINFO_ASYNC2_INFO* pAsync2InfoOut);
+
void recGetGSCookie(GSCookie* pCookieVal, GSCookie** ppCookieVal);
void dmpGetGSCookie(DWORD key, DLDL value);
void repGetGSCookie(GSCookie* pCookieVal, GSCookie** ppCookieVal);
@@ -871,6 +875,10 @@ class MethodContext
CORINFO_GET_TAILCALL_HELPERS_FLAGS flags,
CORINFO_TAILCALL_HELPERS* pResult);
+ void recGetAsyncResumptionStub(CORINFO_METHOD_HANDLE hnd);
+ void dmpGetAsyncResumptionStub(DWORD key, DWORDLONG handle);
+ CORINFO_METHOD_HANDLE repGetAsyncResumptionStub();
+
void recUpdateEntryPointForTailCall(const CORINFO_CONST_LOOKUP& origEntryPoint, const CORINFO_CONST_LOOKUP& newEntryPoint);
void dmpUpdateEntryPointForTailCall(const Agnostic_CORINFO_CONST_LOOKUP& origEntryPoint, const Agnostic_CORINFO_CONST_LOOKUP& newEntryPoint);
void repUpdateEntryPointForTailCall(CORINFO_CONST_LOOKUP* entryPoint);
@@ -1208,6 +1216,8 @@ enum mcPackets
Packet_GetMethodInstantiationArgument = 227,
Packet_GetInstantiatedEntry = 228,
Packet_NotifyInstructionSetUsage = 229,
+ Packet_GetAsync2Info = 230,
+ Packet_GetAsyncResumptionStub = 231,
};
void SetDebugDumpVariables();
diff --git a/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp b/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp
index ed693cdf1b2a9f..d5621dc5a9032a 100644
--- a/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp
+++ b/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp
@@ -1373,6 +1373,13 @@ void interceptor_ICJI::getEEInfo(CORINFO_EE_INFO* pEEInfoOut)
mc->recGetEEInfo(pEEInfoOut);
}
+void interceptor_ICJI::getAsync2Info(CORINFO_ASYNC2_INFO* pAsync2Info)
+{
+ mc->cr->AddCall("getAsync2Info");
+ original_ICorJitInfo->getAsync2Info(pAsync2Info);
+ mc->recGetAsync2Info(pAsync2Info);
+}
+
/*********************************************************************************/
//
// Diagnostic methods
@@ -1788,6 +1795,14 @@ bool interceptor_ICJI::getTailCallHelpers(
return result;
}
+CORINFO_METHOD_HANDLE interceptor_ICJI::getAsyncResumptionStub()
+{
+ mc->cr->AddCall("getAsyncResumptionStub");
+ CORINFO_METHOD_HANDLE stub = original_ICorJitInfo->getAsyncResumptionStub();
+ mc->recGetAsyncResumptionStub(stub);
+ return stub;
+}
+
void interceptor_ICJI::updateEntryPointForTailCall(CORINFO_CONST_LOOKUP* entryPoint)
{
mc->cr->AddCall("updateEntryPointForTailCall");
diff --git a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp
index d14acec9674bb5..e91f7f00670ef5 100644
--- a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp
+++ b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp
@@ -969,6 +969,13 @@ void interceptor_ICJI::getEEInfo(
original_ICorJitInfo->getEEInfo(pEEInfoOut);
}
+void interceptor_ICJI::getAsync2Info(
+ CORINFO_ASYNC2_INFO* pAsync2InfoOut)
+{
+ mcs->AddCall("getAsync2Info");
+ original_ICorJitInfo->getAsync2Info(pAsync2InfoOut);
+}
+
mdMethodDef interceptor_ICJI::getMethodDefFromMethod(
CORINFO_METHOD_HANDLE hMethod)
{
@@ -1279,6 +1286,12 @@ bool interceptor_ICJI::getTailCallHelpers(
return original_ICorJitInfo->getTailCallHelpers(callToken, sig, flags, pResult);
}
+CORINFO_METHOD_HANDLE interceptor_ICJI::getAsyncResumptionStub()
+{
+ mcs->AddCall("getAsyncResumptionStub");
+ return original_ICorJitInfo->getAsyncResumptionStub();
+}
+
bool interceptor_ICJI::convertPInvokeCalliToCall(
CORINFO_RESOLVED_TOKEN* pResolvedToken,
bool mustConvert)
diff --git a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp
index ee04f7d948bb01..a124f6796a9c42 100644
--- a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp
+++ b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp
@@ -849,6 +849,12 @@ void interceptor_ICJI::getEEInfo(
original_ICorJitInfo->getEEInfo(pEEInfoOut);
}
+void interceptor_ICJI::getAsync2Info(
+ CORINFO_ASYNC2_INFO* pAsync2InfoOut)
+{
+ original_ICorJitInfo->getAsync2Info(pAsync2InfoOut);
+}
+
mdMethodDef interceptor_ICJI::getMethodDefFromMethod(
CORINFO_METHOD_HANDLE hMethod)
{
@@ -1122,6 +1128,11 @@ bool interceptor_ICJI::getTailCallHelpers(
return original_ICorJitInfo->getTailCallHelpers(callToken, sig, flags, pResult);
}
+CORINFO_METHOD_HANDLE interceptor_ICJI::getAsyncResumptionStub()
+{
+ return original_ICorJitInfo->getAsyncResumptionStub();
+}
+
bool interceptor_ICJI::convertPInvokeCalliToCall(
CORINFO_RESOLVED_TOKEN* pResolvedToken,
bool mustConvert)
diff --git a/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp b/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp
index d3792172ddd97f..96273f8c1a018a 100644
--- a/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp
+++ b/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp
@@ -1195,6 +1195,12 @@ void MyICJI::getEEInfo(CORINFO_EE_INFO* pEEInfoOut)
jitInstance->mc->repGetEEInfo(pEEInfoOut);
}
+void MyICJI::getAsync2Info(CORINFO_ASYNC2_INFO* pAsync2Info)
+{
+ jitInstance->mc->cr->AddCall("getAsync2Info");
+ jitInstance->mc->repGetAsync2Info(pAsync2Info);
+}
+
/*********************************************************************************/
//
// Diagnostic methods
@@ -1520,6 +1526,12 @@ bool MyICJI::getTailCallHelpers(
return jitInstance->mc->repGetTailCallHelpers(callToken, sig, flags, pResult);
}
+CORINFO_METHOD_HANDLE MyICJI::getAsyncResumptionStub()
+{
+ jitInstance->mc->cr->AddCall("getAsyncResumptionStub");
+ return jitInstance->mc->repGetAsyncResumptionStub();;
+}
+
bool MyICJI::convertPInvokeCalliToCall(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fMustConvert)
{
jitInstance->mc->cr->AddCall("convertPInvokeCalliToCall");
diff --git a/src/coreclr/vm/amd64/AsmHelpers.asm b/src/coreclr/vm/amd64/AsmHelpers.asm
index b0d1b7b4093312..b43b7912e8e959 100644
--- a/src/coreclr/vm/amd64/AsmHelpers.asm
+++ b/src/coreclr/vm/amd64/AsmHelpers.asm
@@ -203,10 +203,11 @@ NESTED_ENTRY OnHijackTripThread, _TEXT
push rax ; make room for the real return address (Rip)
push rdx
PUSH_CALLEE_SAVED_REGISTERS
+ push_vol_reg rcx
push_vol_reg rax
mov rcx, rsp
- alloc_stack 38h ; make extra room for xmm0, argument home slots and align the SP
+ alloc_stack 30h ; make extra room for xmm0 and argument home slots
save_xmm128_postrsp xmm0, 20h
@@ -216,8 +217,9 @@ NESTED_ENTRY OnHijackTripThread, _TEXT
movdqa xmm0, [rsp + 20h]
- add rsp, 38h
+ add rsp, 30h
pop rax
+ pop rcx
POP_CALLEE_SAVED_REGISTERS
pop rdx
ret ; return to the correct place, adjusted by our caller
@@ -469,6 +471,18 @@ LEAF_ENTRY JIT_PartialCompilationPatchpoint, _TEXT
jmp JIT_Patchpoint
LEAF_END JIT_PartialCompilationPatchpoint, _TEXT
+extern JIT_ResumeOSRWorker:proc
+
+NESTED_ENTRY JIT_ResumeOSR, _TEXT
+ PROLOG_WITH_TRANSITION_BLOCK
+
+ lea rcx, [rsp + __PWTB_TransitionBlock] ; TransitionBlock *
+ call JIT_ResumeOSRWorker
+
+ EPILOG_WITH_TRANSITION_BLOCK_RETURN
+ TAILJMP_RAX
+NESTED_END JIT_ResumeOSR, _TEXT
+
endif ; FEATURE_TIERED_COMPILATION
LEAF_ENTRY JIT_PollGC, _TEXT
diff --git a/src/coreclr/vm/amd64/cgenamd64.cpp b/src/coreclr/vm/amd64/cgenamd64.cpp
index 1ba7aae2916bd7..a97b9439236080 100644
--- a/src/coreclr/vm/amd64/cgenamd64.cpp
+++ b/src/coreclr/vm/amd64/cgenamd64.cpp
@@ -339,7 +339,6 @@ void HijackFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFloats
pRD->pCurrentContextPointers->Rsi = NULL;
pRD->pCurrentContextPointers->Rdi = NULL;
#endif
- pRD->pCurrentContextPointers->Rcx = NULL;
#ifdef UNIX_AMD64_ABI
pRD->pCurrentContextPointers->Rdx = (PULONG64)&m_Args->Rdx;
#else // UNIX_AMD64_ABI
@@ -351,6 +350,7 @@ void HijackFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFloats
pRD->pCurrentContextPointers->R11 = NULL;
pRD->pCurrentContextPointers->Rax = (PULONG64)&m_Args->Rax;
+ pRD->pCurrentContextPointers->Rcx = (PULONG64)&m_Args->Rcx;
SyncRegDisplayToCurrentContext(pRD);
}
diff --git a/src/coreclr/vm/amd64/cgencpu.h b/src/coreclr/vm/amd64/cgencpu.h
index 825b116095a76e..1f118ca0e806a2 100644
--- a/src/coreclr/vm/amd64/cgencpu.h
+++ b/src/coreclr/vm/amd64/cgencpu.h
@@ -524,6 +524,11 @@ struct HijackArgs
ULONG64 ReturnValue[2];
};
#endif // !FEATURE_MULTIREG_RETURN
+ union
+ {
+ ULONG64 Rcx;
+ ULONG64 AsyncRet;
+ };
CalleeSavedRegisters Regs;
#ifdef TARGET_WINDOWS
ULONG64 Rsp;
diff --git a/src/coreclr/vm/amd64/unixasmhelpers.S b/src/coreclr/vm/amd64/unixasmhelpers.S
index 10ab11933caee4..78e107233f4594 100644
--- a/src/coreclr/vm/amd64/unixasmhelpers.S
+++ b/src/coreclr/vm/amd64/unixasmhelpers.S
@@ -152,13 +152,16 @@ NESTED_ENTRY OnHijackTripThread, _TEXT, NoHandler
PUSH_CALLEE_SAVED_REGISTERS
+ // Push rcx for the async continuation
+ push_register rcx
+ // Push rdx for the second half of the return value
push_register rdx
// Push rax again - this is where integer/pointer return values are returned
push_register rax
mov rdi, rsp
- alloc_stack 0x28
+ alloc_stack 0x20
// First float return register
movdqa [rsp], xmm0
@@ -171,9 +174,11 @@ NESTED_ENTRY OnHijackTripThread, _TEXT, NoHandler
movdqa xmm0, [rsp]
movdqa xmm1, [rsp+0x10]
- free_stack 0x28
+ free_stack 0x20
+
pop_register rax
pop_register rdx
+ pop_register rcx
POP_CALLEE_SAVED_REGISTERS
ret
@@ -211,4 +216,13 @@ LEAF_ENTRY JIT_PartialCompilationPatchpoint, _TEXT
jmp C_FUNC(JIT_Patchpoint)
LEAF_END JIT_PartialCompilationPatchpoint, _TEXT
+NESTED_ENTRY JIT_ResumeOSR, _TEXT, NoHandler
+ PROLOG_WITH_TRANSITION_BLOCK
+
+ lea rdi, [rsp + __PWTB_TransitionBlock] // TransitionBlock *
+ call C_FUNC(JIT_ResumeOSRWorker)
+
+ EPILOG_WITH_TRANSITION_BLOCK_RETURN
+NESTED_END JIT_ResumeOSR, _TEXT
+
#endif // FEATURE_TIERED_COMPILATION
diff --git a/src/coreclr/vm/arm/asmhelpers.S b/src/coreclr/vm/arm/asmhelpers.S
index e577b1828ae8a3..973de5db340ea7 100644
--- a/src/coreclr/vm/arm/asmhelpers.S
+++ b/src/coreclr/vm/arm/asmhelpers.S
@@ -884,22 +884,19 @@ DelayLoad_Helper\suffix:
// ------------------------------------------------------------------
// Hijack function for functions which return a value type
NESTED_ENTRY OnHijackTripThread, _TEXT, NoHandler
- PROLOG_PUSH "{r0,r4-r11,lr}"
+ // saving r1 as well, as it can have partial return value when return is > 32 bits
+ PROLOG_PUSH "{r0,r1,r2,r4-r11,lr}"
PROLOG_VPUSH "{d0-d3}" // saving as d0-d3 can have the floating point return value
- PROLOG_PUSH "{r1}" // saving as r1 can have partial return value when return is > 32 bits
- alloc_stack 4 // 8 byte align
CHECK_STACK_ALIGNMENT
- add r0, sp, #40
+ add r0, sp, #32
bl C_FUNC(OnHijackWorker)
- free_stack 4
- EPILOG_POP "{r1}"
EPILOG_VPOP "{d0-d3}"
- EPILOG_POP "{r0,r4-r11,pc}"
+ EPILOG_POP "{r0,r1,r2,r4-r11,pc}"
NESTED_END OnHijackTripThread, _TEXT
#endif
diff --git a/src/coreclr/vm/arm/cgencpu.h b/src/coreclr/vm/arm/cgencpu.h
index 914c37d6fea63d..6ae9d7e3b580d5 100644
--- a/src/coreclr/vm/arm/cgencpu.h
+++ b/src/coreclr/vm/arm/cgencpu.h
@@ -919,6 +919,16 @@ struct HijackArgs
// this is only used by functions OnHijackWorker()
};
+ // saving r1 as well, as it can have partial return value when return is > 32 bits
+ // also keeps the struct size 8-byte aligned.
+ DWORD R1;
+
+ union
+ {
+ DWORD R2;
+ size_t AsyncRet;
+ };
+
//
// Non-volatile Integer registers
//
diff --git a/src/coreclr/vm/arm/stubs.cpp b/src/coreclr/vm/arm/stubs.cpp
index cb665654d0cac4..e28af8bcd67aaa 100644
--- a/src/coreclr/vm/arm/stubs.cpp
+++ b/src/coreclr/vm/arm/stubs.cpp
@@ -1592,10 +1592,17 @@ void HijackFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFloats
pRD->IsCallerSPValid = FALSE;
pRD->pCurrentContext->Pc = m_ReturnAddress;
- pRD->pCurrentContext->Sp = PTR_TO_TADDR(m_Args) + sizeof(struct HijackArgs);
+ size_t s = sizeof(struct HijackArgs);
+ _ASSERTE(s%4 == 0); // HijackArgs contains register values and hence will be a multiple of 4
+ // stack must be multiple of 8. So if s is not multiple of 8 then there must be padding of 4 bytes
+ s = s + s%8;
+ pRD->pCurrentContext->Sp = PTR_TO_TADDR(m_Args) + s ;
pRD->pCurrentContext->R0 = m_Args->R0;
+ pRD->pCurrentContext->R2 = m_Args->R2;
+
pRD->volatileCurrContextPointers.R0 = &m_Args->R0;
+ pRD->volatileCurrContextPointers.R2 = &m_Args->R2;
pRD->pCurrentContext->R4 = m_Args->R4;
pRD->pCurrentContext->R5 = m_Args->R5;
diff --git a/src/coreclr/vm/arm64/asmhelpers.S b/src/coreclr/vm/arm64/asmhelpers.S
index 5fac7fae7aa838..16888164cf6f5a 100644
--- a/src/coreclr/vm/arm64/asmhelpers.S
+++ b/src/coreclr/vm/arm64/asmhelpers.S
@@ -301,7 +301,7 @@ NESTED_END TheUMEntryPrestub, _TEXT
// ------------------------------------------------------------------
// Hijack function for functions which return a scalar type or a struct (value type)
NESTED_ENTRY OnHijackTripThread, _TEXT, NoHandler
- PROLOG_SAVE_REG_PAIR_INDEXED fp, lr, -176
+ PROLOG_SAVE_REG_PAIR_INDEXED fp, lr, -192
// Spill callee saved registers
PROLOG_SAVE_REG_PAIR x19, x20, 16
PROLOG_SAVE_REG_PAIR x21, x22, 32
@@ -312,9 +312,12 @@ NESTED_ENTRY OnHijackTripThread, _TEXT, NoHandler
// save any integral return value(s)
stp x0, x1, [sp, #96]
+ // save async continuation return value
+ str x2, [sp, #112]
+
// save any FP/HFA return value(s)
- stp q0, q1, [sp, #112]
- stp q2, q3, [sp, #144]
+ stp q0, q1, [sp, #128]
+ stp q2, q3, [sp, #160]
mov x0, sp
bl C_FUNC(OnHijackWorker)
@@ -322,16 +325,19 @@ NESTED_ENTRY OnHijackTripThread, _TEXT, NoHandler
// restore any integral return value(s)
ldp x0, x1, [sp, #96]
+ // restore async continuation return value
+ ldr x2, [sp, #112]
+
// restore any FP/HFA return value(s)
- ldp q0, q1, [sp, #112]
- ldp q2, q3, [sp, #144]
+ ldp q0, q1, [sp, #128]
+ ldp q2, q3, [sp, #160]
EPILOG_RESTORE_REG_PAIR x19, x20, 16
EPILOG_RESTORE_REG_PAIR x21, x22, 32
EPILOG_RESTORE_REG_PAIR x23, x24, 48
EPILOG_RESTORE_REG_PAIR x25, x26, 64
EPILOG_RESTORE_REG_PAIR x27, x28, 80
- EPILOG_RESTORE_REG_PAIR_INDEXED fp, lr, 176
+ EPILOG_RESTORE_REG_PAIR_INDEXED fp, lr, 192
EPILOG_RETURN
NESTED_END OnHijackTripThread, _TEXT
@@ -764,6 +770,15 @@ LEAF_ENTRY JIT_PartialCompilationPatchpoint, _TEXT
b C_FUNC(JIT_Patchpoint)
LEAF_END JIT_PartialCompilationPatchpoint, _TEXT
+NESTED_ENTRY JIT_ResumeOSR, _TEXT, NoHandler
+ PROLOG_WITH_TRANSITION_BLOCK
+
+ add x0, sp, #__PWTB_TransitionBlock // TransitionBlock *
+ bl C_FUNC(JIT_ResumeOSRWorker)
+
+ EPILOG_WITH_TRANSITION_BLOCK_RETURN
+NESTED_END JIT_ResumeOSR, _TEXT
+
#endif // FEATURE_TIERED_COMPILATION
LEAF_ENTRY JIT_ValidateIndirectCall, _TEXT
diff --git a/src/coreclr/vm/arm64/asmhelpers.asm b/src/coreclr/vm/arm64/asmhelpers.asm
index 3dd6edcb34ab2b..fd8ee9f0bde223 100644
--- a/src/coreclr/vm/arm64/asmhelpers.asm
+++ b/src/coreclr/vm/arm64/asmhelpers.asm
@@ -640,7 +640,7 @@ COMToCLRDispatchHelper_RegSetup
; ------------------------------------------------------------------
; Hijack function for functions which return a scalar type or a struct (value type)
NESTED_ENTRY OnHijackTripThread
- PROLOG_SAVE_REG_PAIR fp, lr, #-176!
+ PROLOG_SAVE_REG_PAIR fp, lr, #-192!
; Spill callee saved registers
PROLOG_SAVE_REG_PAIR x19, x20, #16
PROLOG_SAVE_REG_PAIR x21, x22, #32
@@ -651,9 +651,12 @@ COMToCLRDispatchHelper_RegSetup
; save any integral return value(s)
stp x0, x1, [sp, #96]
+ ; save async continuation return value
+ str x2, [sp, #112]
+
; save any FP/HFA/HVA return value(s)
- stp q0, q1, [sp, #112]
- stp q2, q3, [sp, #144]
+ stp q0, q1, [sp, #128]
+ stp q2, q3, [sp, #160]
mov x0, sp
bl OnHijackWorker
@@ -661,16 +664,19 @@ COMToCLRDispatchHelper_RegSetup
; restore any integral return value(s)
ldp x0, x1, [sp, #96]
+ ; restore async continuation return value
+ ldr x2, [sp, #112]
+
; restore any FP/HFA/HVA return value(s)
- ldp q0, q1, [sp, #112]
- ldp q2, q3, [sp, #144]
+ ldp q0, q1, [sp, #128]
+ ldp q2, q3, [sp, #160]
EPILOG_RESTORE_REG_PAIR x19, x20, #16
EPILOG_RESTORE_REG_PAIR x21, x22, #32
EPILOG_RESTORE_REG_PAIR x23, x24, #48
EPILOG_RESTORE_REG_PAIR x25, x26, #64
EPILOG_RESTORE_REG_PAIR x27, x28, #80
- EPILOG_RESTORE_REG_PAIR fp, lr, #176!
+ EPILOG_RESTORE_REG_PAIR fp, lr, #192!
EPILOG_RETURN
NESTED_END
@@ -1166,6 +1172,17 @@ __HelperNakedFuncName SETS "$helper":CC:"Naked"
b JIT_Patchpoint
LEAF_END
+ IMPORT JIT_ResumeOSRWorker
+
+ NESTED_ENTRY JIT_ResumeOSR
+ PROLOG_WITH_TRANSITION_BLOCK
+
+ add x0, sp, #__PWTB_TransitionBlock ; TransitionBlock *
+ bl JIT_ResumeOSRWorker
+
+ EPILOG_WITH_TRANSITION_BLOCK_RETURN
+ NESTED_END
+
#endif ; FEATURE_TIERED_COMPILATION
LEAF_ENTRY JIT_ValidateIndirectCall
diff --git a/src/coreclr/vm/arm64/cgencpu.h b/src/coreclr/vm/arm64/cgencpu.h
index c016515df827ce..9f6f86532ffddb 100644
--- a/src/coreclr/vm/arm64/cgencpu.h
+++ b/src/coreclr/vm/arm64/cgencpu.h
@@ -533,6 +533,12 @@ struct HijackArgs
size_t ReturnValue[2];
};
union
+ {
+ DWORD64 X2;
+ size_t AsyncRet;
+ };
+ DWORD64 Pad;
+ union
{
struct {
NEON128 Q0;
diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp
index a6695fb0895b1b..751595b9f528b3 100644
--- a/src/coreclr/vm/arm64/stubs.cpp
+++ b/src/coreclr/vm/arm64/stubs.cpp
@@ -768,9 +768,11 @@ void HijackFrame::UpdateRegDisplay_Impl(const PREGDISPLAY pRD, bool updateFloats
pRD->pCurrentContext->X0 = m_Args->X0;
pRD->pCurrentContext->X1 = m_Args->X1;
+ pRD->pCurrentContext->X2 = m_Args->X2;
pRD->volatileCurrContextPointers.X0 = &m_Args->X0;
pRD->volatileCurrContextPointers.X1 = &m_Args->X1;
+ pRD->volatileCurrContextPointers.X1 = &m_Args->X2;
pRD->pCurrentContext->X19 = m_Args->X19;
pRD->pCurrentContext->X20 = m_Args->X20;
diff --git a/src/coreclr/vm/array.cpp b/src/coreclr/vm/array.cpp
index 9c2f495a844c04..698e3a7697eda2 100644
--- a/src/coreclr/vm/array.cpp
+++ b/src/coreclr/vm/array.cpp
@@ -465,7 +465,7 @@ MethodTable* Module::CreateArrayMethodTable(TypeHandle elemTypeHnd, CorElementTy
+ 3; // for rank specific Get, Set, Address
MethodDescChunk * pChunks = MethodDescChunk::CreateChunk(pAllocator->GetHighFrequencyHeap(),
- dwMethodDescs, mcArray, FALSE /* fNonVtableSlot*/, FALSE /* fNativeCodeSlot */,
+ dwMethodDescs, mcArray, FALSE /* fNonVtableSlot*/, FALSE /* fNativeCodeSlot */, FALSE /* HasAsyncMethodData */,
pMT, pamTracker);
pClass->SetChunks(pChunks);
diff --git a/src/coreclr/vm/binder.cpp b/src/coreclr/vm/binder.cpp
index c60ced8f4cb26c..939aa29359649f 100644
--- a/src/coreclr/vm/binder.cpp
+++ b/src/coreclr/vm/binder.cpp
@@ -441,6 +441,12 @@ void CoreLibBinder::BuildConvertedSignature(const BYTE* pSig, SigBuilder * pSigB
argCount = 0;
}
+ if ((callConv & IMAGE_CEE_CS_CALLCONV_GENERIC) != 0)
+ {
+ unsigned genericArgCount = *pSig++;
+ pSigBuilder->AppendData(genericArgCount);
+ }
+
// <= because we want to include the return value or the field
for (unsigned i = 0; i <= argCount; i++) {
if (ConvertType(pSig, pSigBuilder))
diff --git a/src/coreclr/vm/callhelpers.cpp b/src/coreclr/vm/callhelpers.cpp
index 26c2ff0c03c229..f575a21899185c 100644
--- a/src/coreclr/vm/callhelpers.cpp
+++ b/src/coreclr/vm/callhelpers.cpp
@@ -327,7 +327,7 @@ void MethodDescCallSite::CallTargetWorker(const ARG_SLOT *pArguments, ARG_SLOT *
ENABLE_FORBID_GC_LOADER_USE_IN_THIS_SCOPE();
_ASSERTE(isCallConv(m_methodSig.GetCallingConvention(), IMAGE_CEE_CS_CALLCONV_DEFAULT));
- _ASSERTE(!(m_methodSig.GetCallingConventionInfo() & CORINFO_CALLCONV_PARAMTYPE));
+ _ASSERTE(!m_methodSig.HasGenericContextArg());
#ifdef DEBUGGING_SUPPORTED
if (CORDebuggerTraceCall())
diff --git a/src/coreclr/vm/callingconvention.h b/src/coreclr/vm/callingconvention.h
index 5b167300ec856d..98e6eee147fcd6 100644
--- a/src/coreclr/vm/callingconvention.h
+++ b/src/coreclr/vm/callingconvention.h
@@ -2148,6 +2148,18 @@ class ArgIteratorBase
return m_pSig->IsVarArg() || m_pSig->IsTreatAsVarArg();
}
+ BOOL IsAsyncCall()
+ {
+ LIMITED_METHOD_CONTRACT;
+ return m_pSig->IsAsyncCall();
+ }
+
+ BOOL HasAsyncContinuation()
+ {
+ LIMITED_METHOD_CONTRACT;
+ return m_pSig->HasAsyncContinuation();
+ }
+
DWORD NumFixedArgs()
{
LIMITED_METHOD_CONTRACT;
diff --git a/src/coreclr/vm/class.cpp b/src/coreclr/vm/class.cpp
index 1834d34953b156..d135e2c364dfa7 100644
--- a/src/coreclr/vm/class.cpp
+++ b/src/coreclr/vm/class.cpp
@@ -679,6 +679,7 @@ HRESULT EEClass::AddMethodDesc(
classification,
TRUE, // fNonVtableSlot
TRUE, // fNativeCodeSlot
+ FALSE, /* HasAsyncMethodData */
pMT,
&dummyAmTracker);
@@ -726,7 +727,9 @@ HRESULT EEClass::AddMethodDesc(
TRUE, // fEnC
0, // RVA - non-zero only for NDirect
pImport,
- NULL
+ NULL,
+ Signature(),
+ AsyncMethodKind::NotAsync
COMMA_INDEBUG(debug_szMethodName)
COMMA_INDEBUG(pMT->GetDebugClassName())
COMMA_INDEBUG(NULL)
diff --git a/src/coreclr/vm/clrtocomcall.cpp b/src/coreclr/vm/clrtocomcall.cpp
index a9505696385123..23c10df106b239 100644
--- a/src/coreclr/vm/clrtocomcall.cpp
+++ b/src/coreclr/vm/clrtocomcall.cpp
@@ -230,6 +230,11 @@ I4ARRAYREF SetUpWrapperInfo(MethodDesc *pMD)
GCX_PREEMP();
+ if (pMD->IsAsync2Method())
+ {
+ ThrowHR(COR_E_NOTSUPPORTED);
+ }
+
// Collects ParamDef information in an indexed array where element 0 represents
// the return type.
mdParamDef *params = (mdParamDef*)_alloca((numArgs+1) * sizeof(mdParamDef));
@@ -504,6 +509,11 @@ UINT32 CLRToCOMLateBoundWorker(
LPCUTF8 strMemberName;
ULONG uSemantic;
+ if (pItfMD->IsAsync2Method())
+ {
+ ThrowHR(COR_E_NOTSUPPORTED);
+ }
+
// See if there is property information for this member.
hr = pItfMT->GetMDImport()->GetPropertyInfoForMethodDef(pItfMD->GetMemberDef(), &propToken, &strMemberName, &uSemantic);
if (hr != S_OK)
diff --git a/src/coreclr/vm/clsload.cpp b/src/coreclr/vm/clsload.cpp
index 1a3a6ad4343619..d02505d240850b 100644
--- a/src/coreclr/vm/clsload.cpp
+++ b/src/coreclr/vm/clsload.cpp
@@ -2801,7 +2801,7 @@ TypeHandle ClassLoader::PublishType(const TypeKey *pTypeKey, TypeHandle typeHnd)
{
MethodDesc * pMD = it.GetMethodDesc();
CONSISTENCY_CHECK(pMD != NULL && pMD->GetMethodTable() == pMT);
- if (!pMD->IsUnboxingStub())
+ if (!pMD->IsUnboxingStub() && !pMD->IsAsync2VariantMethod())
{
pModule->EnsuredStoreMethodDef(pMD->GetMemberDef(), pMD);
}
diff --git a/src/coreclr/vm/codeversion.cpp b/src/coreclr/vm/codeversion.cpp
index cc8a516a7dc4a2..f67f3a4b72066d 100644
--- a/src/coreclr/vm/codeversion.cpp
+++ b/src/coreclr/vm/codeversion.cpp
@@ -362,7 +362,7 @@ void NativeCodeVersion::SetOptimizationTier(OptimizationTier tier)
#ifdef FEATURE_ON_STACK_REPLACEMENT
-PatchpointInfo * NativeCodeVersion::GetOSRInfo(unsigned * ilOffset)
+PatchpointInfo * NativeCodeVersion::GetOSRInfo(unsigned * ilOffset) const
{
LIMITED_METHOD_DAC_CONTRACT;
if (m_storageKind == StorageKind::Explicit)
diff --git a/src/coreclr/vm/codeversion.h b/src/coreclr/vm/codeversion.h
index b9dc76b90e3066..d4e973bdfbdcac 100644
--- a/src/coreclr/vm/codeversion.h
+++ b/src/coreclr/vm/codeversion.h
@@ -90,7 +90,7 @@ class NativeCodeVersion
#endif // FEATURE_TIERED_COMPILATION
#ifdef FEATURE_ON_STACK_REPLACEMENT
- PatchpointInfo * GetOSRInfo(unsigned * iloffset);
+ PatchpointInfo * GetOSRInfo(unsigned * iloffset) const;
#endif // FEATURE_ON_STACK_REPLACEMENT
#ifdef HAVE_GCCOVER
diff --git a/src/coreclr/vm/comdelegate.cpp b/src/coreclr/vm/comdelegate.cpp
index cdc89f491641b3..d9c5749ede5dd5 100644
--- a/src/coreclr/vm/comdelegate.cpp
+++ b/src/coreclr/vm/comdelegate.cpp
@@ -1963,7 +1963,7 @@ Stub* COMDelegate::GetInvokeMethodStub(EEImplMethodDesc* pMD)
MetaSig sig(pMD);
- BYTE callConv = sig.GetCallingConventionInfo();
+ unsigned callConv = sig.GetCallingConventionInfo();
if (callConv != (IMAGE_CEE_CS_CALLCONV_HASTHIS | IMAGE_CEE_CS_CALLCONV_DEFAULT))
COMPlusThrow(kInvalidProgramException);
diff --git a/src/coreclr/vm/commodule.cpp b/src/coreclr/vm/commodule.cpp
index e9850d49a678e0..e89b5cf3362a25 100644
--- a/src/coreclr/vm/commodule.cpp
+++ b/src/coreclr/vm/commodule.cpp
@@ -302,7 +302,13 @@ extern "C" INT32 QCALLTYPE ModuleBuilder_GetMemberRefOfMethodInfo(QCall::ModuleH
COMPlusThrow(kNotSupportedException);
}
- if (pMeth->GetMethodTable()->GetModule() == pModule)
+ if (pMeth->IsAsync2VariantMethod())
+ {
+ _ASSERTE(!"Should not have come here!");
+ COMPlusThrow(kNotSupportedException);
+ }
+
+ if ((pMeth->GetMethodTable()->GetModule() == pModule))
{
// If the passed in method is defined in the same module, just return the MethodDef token
memberRefE = pMeth->GetMemberDef();
diff --git a/src/coreclr/vm/commtmemberinfomap.cpp b/src/coreclr/vm/commtmemberinfomap.cpp
index df96c0f44f74d5..8126f9b673176b 100644
--- a/src/coreclr/vm/commtmemberinfomap.cpp
+++ b/src/coreclr/vm/commtmemberinfomap.cpp
@@ -688,6 +688,9 @@ void ComMTMemberInfoMap::GetMethodPropsForMeth(
// Generally don't munge function into a getter.
rProps[ix].bFunction2Getter = FALSE;
+ if (pMeth->IsAsync2Method())
+ ThrowHR(COR_E_NOTSUPPORTED);
+
// See if there is property information for this member.
hr = pMeth->GetMDImport()->GetPropertyInfoForMethodDef(pMeth->GetMemberDef(), &pd, &pPropName, &uSemantic);
IfFailThrow(hr);
@@ -1604,6 +1607,10 @@ void ComMTMemberInfoMap::PopulateMemberHashtable()
// We are dealing with a method.
MethodDesc *pMD = pProps->pMeth;
+ if (pMD->IsAsync2Method())
+ {
+ ThrowHR(COR_E_NOTSUPPORTED); // Probably this isn't right, and instead should be a skip, but a throw makes it easier to find if this is wrong
+ }
EEModuleTokenPair Key(pMD->GetMemberDef(), pMD->GetModule());
m_TokenToComMTMethodPropsMap.InsertValue(&Key, (HashDatum)pProps);
}
diff --git a/src/coreclr/vm/comtoclrcall.cpp b/src/coreclr/vm/comtoclrcall.cpp
index 0f51af428e8b04..1beae2cd959656 100644
--- a/src/coreclr/vm/comtoclrcall.cpp
+++ b/src/coreclr/vm/comtoclrcall.cpp
@@ -986,7 +986,10 @@ void ComCallMethodDesc::InitNativeInfo()
MethodTable * pMT = pMD->GetMethodTable();
IMDInternalImport * pInternalImport = pMT->GetMDImport();
-
+ if (pMD->IsAsync2Method())
+ {
+ ThrowHR(COR_E_NOTSUPPORTED);
+ }
mdMethodDef md = pMD->GetMemberDef();
ULONG ulCodeRVA;
diff --git a/src/coreclr/vm/comutilnative.cpp b/src/coreclr/vm/comutilnative.cpp
index d039226c328cad..93992773a4a387 100644
--- a/src/coreclr/vm/comutilnative.cpp
+++ b/src/coreclr/vm/comutilnative.cpp
@@ -1851,6 +1851,14 @@ FCIMPL1(MethodTable*, MethodTableNative::InstantiationArg0, MethodTable* mt);
}
FCIMPLEND
+FCIMPL1(OBJECTHANDLE, MethodTableNative::GetLoaderAllocatorHandle, MethodTable *mt)
+{
+ FCALL_CONTRACT;
+
+ return mt->GetLoaderAllocatorObjectHandle();
+}
+FCIMPLEND
+
extern "C" BOOL QCALLTYPE MethodTable_AreTypesEquivalent(MethodTable* mta, MethodTable* mtb)
{
QCALL_CONTRACT;
diff --git a/src/coreclr/vm/comutilnative.h b/src/coreclr/vm/comutilnative.h
index 1f5fda9ed02ccc..8f229352b439a4 100644
--- a/src/coreclr/vm/comutilnative.h
+++ b/src/coreclr/vm/comutilnative.h
@@ -264,6 +264,7 @@ class MethodTableNative {
static FCDECL1(CorElementType, GetPrimitiveCorElementType, MethodTable* mt);
static FCDECL2(MethodTable*, GetMethodTableMatchingParentClass, MethodTable* mt, MethodTable* parent);
static FCDECL1(MethodTable*, InstantiationArg0, MethodTable* mt);
+ static FCDECL1(OBJECTHANDLE, GetLoaderAllocatorHandle, MethodTable* mt);
};
extern "C" BOOL QCALLTYPE MethodTable_AreTypesEquivalent(MethodTable* mta, MethodTable* mtb);
diff --git a/src/coreclr/vm/corelib.cpp b/src/coreclr/vm/corelib.cpp
index c271a057d12d68..1e28694d8c5935 100644
--- a/src/coreclr/vm/corelib.cpp
+++ b/src/coreclr/vm/corelib.cpp
@@ -157,6 +157,11 @@ enum _gsigc {
const BYTE gsige_IM_ ## varname[] = { (BYTE) -gsigl_IM_ ## varname, \
IMAGE_CEE_CS_CALLCONV_HASTHIS, gsigc_IM_ ## varname, retval args };
+#define _GM(varname, conv, n, args, retval) extern const BYTE gsige_GM_ ## varname[]; \
+ HardCodedMetaSig gsig_GM_ ## varname = { gsige_GM_ ## varname }; \
+ const BYTE gsige_GM_ ## varname[] = { (BYTE) -gsigl_GM_ ## varname, \
+ conv | IMAGE_CEE_CS_CALLCONV_GENERIC, n, gsigc_GM_ ## varname, retval args };
+
#define _Fld(varname, val) extern const BYTE gsige_Fld_ ## varname[]; \
HardCodedMetaSig gsig_Fld_ ## varname = { gsige_Fld_ ## varname }; \
const BYTE gsige_Fld_ ## varname[] = { (BYTE) -gsigl_Fld_ ## varname, \
@@ -166,6 +171,7 @@ enum _gsigc {
#undef _SM
#undef _IM
+#undef _GM
#undef _Fld
#ifdef _DEBUG
diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h
index bb71255f382cd5..856ef83e827d01 100644
--- a/src/coreclr/vm/corelib.h
+++ b/src/coreclr/vm/corelib.h
@@ -347,6 +347,32 @@ DEFINE_METHOD(TYPE_INIT_EXCEPTION, STR_EX_CTOR, .ctor,
DEFINE_CLASS(THREAD_START_EXCEPTION,Threading, ThreadStartException)
DEFINE_METHOD(THREAD_START_EXCEPTION,EX_CTOR, .ctor, IM_Exception_RetVoid)
+DEFINE_CLASS(VALUETASK_1, Tasks, ValueTask`1)
+
+DEFINE_CLASS(VALUETASK, Tasks, ValueTask)
+DEFINE_METHOD(VALUETASK, FROM_EXCEPTION, FromException, SM_Exception_RetValueTask)
+DEFINE_METHOD(VALUETASK, FROM_EXCEPTION_1, FromException, GM_Exception_RetValueTaskOfT)
+DEFINE_METHOD(VALUETASK, FROM_RESULT_T, FromResult, GM_T_RetValueTaskOfT)
+DEFINE_METHOD(VALUETASK, GET_COMPLETED_TASK, get_CompletedTask, SM_RetValueTask)
+
+DEFINE_CLASS(TASK_1, Tasks, Task`1)
+DEFINE_METHOD(TASK_1, GET_AWAITER, GetAwaiter, NoSig)
+
+DEFINE_CLASS(TASK, Tasks, Task)
+DEFINE_METHOD(TASK, FROM_EXCEPTION, FromException, SM_Exception_RetTask)
+DEFINE_METHOD(TASK, FROM_EXCEPTION_1, FromException, GM_Exception_RetTaskOfT)
+DEFINE_METHOD(TASK, FROM_RESULT_T, FromResult, GM_T_RetTaskOfT)
+DEFINE_METHOD(TASK, GET_COMPLETED_TASK, get_CompletedTask, SM_RetTask)
+DEFINE_METHOD(TASK, GET_AWAITER, GetAwaiter, NoSig)
+
+DEFINE_CLASS(TASK_AWAITER_1, CompilerServices, TaskAwaiter`1)
+DEFINE_METHOD(TASK_AWAITER_1, GET_ISCOMPLETED, get_IsCompleted, NoSig)
+DEFINE_METHOD(TASK_AWAITER_1, GET_RESULT, GetResult, NoSig)
+
+DEFINE_CLASS(TASK_AWAITER, CompilerServices, TaskAwaiter)
+DEFINE_METHOD(TASK_AWAITER, GET_ISCOMPLETED, get_IsCompleted, NoSig)
+DEFINE_METHOD(TASK_AWAITER, GET_RESULT, GetResult, NoSig)
+
DEFINE_CLASS(TYPE_HANDLE, System, RuntimeTypeHandle)
DEFINE_CLASS(RT_TYPE_HANDLE, System, RuntimeTypeHandle)
DEFINE_METHOD(RT_TYPE_HANDLE, PVOID_CTOR, .ctor, IM_RuntimeType_RetVoid)
@@ -653,6 +679,10 @@ DEFINE_CLASS(RESOURCE_MANAGER, Resources, ResourceManager)
DEFINE_CLASS(RTFIELD, Reflection, RtFieldInfo)
DEFINE_METHOD(RTFIELD, GET_FIELDESC, GetFieldDesc, IM_RetIntPtr)
+DEFINE_CLASS(EXECUTIONANDSYNCBLOCKSTORE, CompilerServices, ExecutionAndSyncBlockStore)
+DEFINE_METHOD(EXECUTIONANDSYNCBLOCKSTORE, PUSH, Push, NoSig)
+DEFINE_METHOD(EXECUTIONANDSYNCBLOCKSTORE, POP, Pop, NoSig)
+
DEFINE_CLASS(RUNTIME_HELPERS, CompilerServices, RuntimeHelpers)
DEFINE_METHOD(RUNTIME_HELPERS, IS_BITWISE_EQUATABLE, IsBitwiseEquatable, NoSig)
DEFINE_METHOD(RUNTIME_HELPERS, GET_RAW_DATA, GetRawData, NoSig)
@@ -665,6 +695,16 @@ DEFINE_METHOD(RUNTIME_HELPERS, DISPATCH_TAILCALLS, DispatchTailCalls,
#ifdef FEATURE_IJW
DEFINE_METHOD(RUNTIME_HELPERS, COPY_CONSTRUCT, CopyConstruct, NoSig)
#endif // FEATURE_IJW
+DEFINE_METHOD(RUNTIME_HELPERS, ALLOC_CONTINUATION, AllocContinuation, NoSig)
+DEFINE_METHOD(RUNTIME_HELPERS, ALLOC_CONTINUATION_METHOD, AllocContinuationMethod, NoSig)
+DEFINE_METHOD(RUNTIME_HELPERS, ALLOC_CONTINUATION_CLASS, AllocContinuationClass, NoSig)
+DEFINE_METHOD(RUNTIME_HELPERS, ALLOC_CONTINUATION_RESULT_BOX, AllocContinuationResultBox, SM_VoidPtr_RetObj)
+DEFINE_METHOD(RUNTIME_HELPERS, FINALIZE_TASK_RETURNING_THUNK, FinalizeTaskReturningThunk, SM_Continuation_RetTask)
+DEFINE_METHOD(RUNTIME_HELPERS, FINALIZE_TASK_RETURNING_THUNK_1, FinalizeTaskReturningThunk, GM_Continuation_RetTaskOfT)
+DEFINE_METHOD(RUNTIME_HELPERS, FINALIZE_VALUETASK_RETURNING_THUNK, FinalizeValueTaskReturningThunk, SM_Continuation_RetValueTask)
+DEFINE_METHOD(RUNTIME_HELPERS, FINALIZE_VALUETASK_RETURNING_THUNK_1, FinalizeValueTaskReturningThunk, GM_Continuation_RetValueTaskOfT)
+DEFINE_METHOD(RUNTIME_HELPERS, UNSAFE_AWAIT_AWAITER_FROM_RUNTIME_ASYNC_1, UnsafeAwaitAwaiterFromRuntimeAsync, GM_T_RetVoid)
+DEFINE_METHOD(RUNTIME_HELPERS, AWAIT_AWAITER_FROM_RUNTIME_ASYNC_1, AwaitAwaiterFromRuntimeAsync, GM_T_RetVoid)
DEFINE_CLASS(SPAN_HELPERS, System, SpanHelpers)
DEFINE_METHOD(SPAN_HELPERS, MEMSET, Fill, SM_RefByte_Byte_UIntPtr_RetVoid)
@@ -767,6 +807,14 @@ DEFINE_CLASS_U(CompilerServices, TailCallTls, TailCallTls)
DEFINE_FIELD_U(Frame, TailCallTls, m_frame)
DEFINE_FIELD_U(ArgBuffer, TailCallTls, m_argBuffer)
+DEFINE_CLASS(CONTINUATION, CompilerServices, Continuation)
+DEFINE_FIELD(CONTINUATION, NEXT, Next)
+DEFINE_FIELD(CONTINUATION, RESUME, Resume)
+DEFINE_FIELD(CONTINUATION, STATE, State)
+DEFINE_FIELD(CONTINUATION, FLAGS, Flags)
+DEFINE_FIELD(CONTINUATION, DATA, Data)
+DEFINE_FIELD(CONTINUATION, GCDATA, GCData)
+
DEFINE_CLASS(RUNTIME_WRAPPED_EXCEPTION, CompilerServices, RuntimeWrappedException)
DEFINE_METHOD(RUNTIME_WRAPPED_EXCEPTION, OBJ_CTOR, .ctor, IM_Obj_RetVoid)
DEFINE_FIELD(RUNTIME_WRAPPED_EXCEPTION, WRAPPED_EXCEPTION, _wrappedException)
@@ -986,6 +1034,7 @@ DEFINE_METHOD(STUBHELPERS, VALIDATE_BYREF, Validate
DEFINE_METHOD(STUBHELPERS, GET_STUB_CONTEXT, GetStubContext, SM_RetIntPtr)
DEFINE_METHOD(STUBHELPERS, LOG_PINNED_ARGUMENT, LogPinnedArgument, SM_IntPtr_IntPtr_RetVoid)
DEFINE_METHOD(STUBHELPERS, NEXT_CALL_RETURN_ADDRESS, NextCallReturnAddress, SM_RetIntPtr)
+DEFINE_METHOD(STUBHELPERS, ASYNC2_CALL_CONTINUATION, Async2CallContinuation, SM_RetContinuation)
DEFINE_METHOD(STUBHELPERS, SAFE_HANDLE_ADD_REF, SafeHandleAddRef, SM_SafeHandle_RefBool_RetIntPtr)
DEFINE_METHOD(STUBHELPERS, SAFE_HANDLE_RELEASE, SafeHandleRelease, SM_SafeHandle_RetVoid)
diff --git a/src/coreclr/vm/dispatchinfo.cpp b/src/coreclr/vm/dispatchinfo.cpp
index bade7f6ce88636..00549cdc341786 100644
--- a/src/coreclr/vm/dispatchinfo.cpp
+++ b/src/coreclr/vm/dispatchinfo.cpp
@@ -448,7 +448,13 @@ ComMTMethodProps * DispatchMemberInfo::GetMemberProps(OBJECTREF MemberInfoObj, C
ARG_SLOT GetMethodHandleArg = ObjToArgSlot(MemberInfoObj);
MethodDesc* pMeth = (MethodDesc*) getMethodHandle.Call_RetLPVOID(&GetMethodHandleArg);
if (pMeth)
+ {
+ if (pMeth->IsAsync2Method())
+ {
+ ThrowHR(COR_E_NOTSUPPORTED);
+ }
pMemberProps = pMemberMap->GetMethodProps(pMeth->GetMemberDef(), pMeth->GetModule());
+ }
}
else if (CoreLibBinder::IsClass(pMemberInfoClass, CLASS__RT_FIELD_INFO))
{
@@ -844,6 +850,10 @@ void DispatchMemberInfo::SetUpMethodMarshalerInfo(MethodDesc *pMD, BOOL bReturnV
//
// Initialize the parameter definition enum.
//
+ if (pMD->IsAsync2Method())
+ {
+ ThrowHR(COR_E_NOTSUPPORTED);
+ }
hEnumParams.EnumInit(mdtParamDef, pMD->GetMemberDef());
//
@@ -2578,6 +2588,11 @@ bool DispatchInfo::IsPropertyAccessorVisible(bool fIsSetter, OBJECTREF* pMemberI
// Check to see if the new method is a property accessor.
mdToken tkMember = mdTokenNil;
+ if (pMDForProperty->IsAsync2VariantMethod())
+ {
+ return false;
+ }
+
if (pMDForProperty->GetMDImport()->GetPropertyInfoForMethodDef(pMDForProperty->GetMemberDef(), &tkMember, NULL, NULL) == S_OK)
{
if (IsMemberVisibleFromCom(pMDForProperty->GetMethodTable(), tkMember, pMDForProperty->GetMemberDef()))
diff --git a/src/coreclr/vm/dllimport.cpp b/src/coreclr/vm/dllimport.cpp
index 858c42329c0720..4ed1a2df875530 100644
--- a/src/coreclr/vm/dllimport.cpp
+++ b/src/coreclr/vm/dllimport.cpp
@@ -105,6 +105,8 @@ StubSigDesc::StubSigDesc(MethodDesc *pMD)
m_sig = pMD->GetSignature();
m_pModule = pMD->GetModule(); // Used for token resolution.
+ _ASSERTE(!pMD->IsAsync2Method());
+
m_tkMethodDef = pMD->GetMemberDef();
SigTypeContext::InitTypeContext(pMD, &m_typeContext);
m_pMetadataModule = pMD->GetModule();
@@ -132,6 +134,7 @@ StubSigDesc::StubSigDesc(MethodDesc* pMD, const Signature& sig, Module* pModule,
if (pMD != NULL)
{
+ _ASSERTE(!pMD->IsAsync2Method());
m_tkMethodDef = pMD->GetMemberDef();
SigTypeContext::InitTypeContext(pMD, &m_typeContext);
m_pMetadataModule = pMD->GetModule();
@@ -1086,7 +1089,10 @@ class ILStubState : public StubState
DWORD dwToken = 0;
if (pTargetMD)
+ {
+ _ASSERTE(!pTargetMD->IsAsync2VariantMethod());
dwToken = pTargetMD->GetMemberDef();
+ }
//
@@ -2710,6 +2716,10 @@ void PInvokeStaticSigInfo::DllImportInit(
IMDInternalImport *pInternalImport = pMD->GetMDImport();
CorPinvokeMap mappingFlags = pmMaxValue;
mdModuleRef modref = mdModuleRefNil;
+ if (pMD->IsAsync2Method())
+ {
+ ThrowHR(COR_E_NOTSUPPORTED);
+ }
if (FAILED(pInternalImport->GetPinvokeMap(pMD->GetMemberDef(), (DWORD*)&mappingFlags, ppEntryPointName, &modref)))
{
InitCallConv(CallConvWinApiSentinel, pMD);
@@ -2987,6 +2997,10 @@ namespace
CorInfoCallConvExtension callConvLocal;
IMDInternalImport* pInternalImport = pMD->GetMDImport();
CorPinvokeMap mappingFlags = pmMaxValue;
+ if (pMD->IsAsync2Method())
+ {
+ ThrowHR(COR_E_NOTSUPPORTED);
+ }
HRESULT hr = pInternalImport->GetPinvokeMap(pMD->GetMemberDef(), (DWORD*)&mappingFlags, NULL /*pszImportName*/, NULL /*pmrImportDLL*/);
if (FAILED(hr))
return false;
@@ -3247,6 +3261,10 @@ BOOL NDirect::MarshalingRequired(
mdMethodDef methodToken = mdMethodDefNil;
if (pMD != NULL)
{
+ if (pMD->IsAsync2Method())
+ {
+ ThrowHR(COR_E_NOTSUPPORTED);
+ }
methodToken = pMD->GetMemberDef();
}
CollateParamTokens(pMDImport, methodToken, numArgs - 1, pParamTokenArray);
@@ -6040,6 +6058,10 @@ PCODE GetILStubForCalli(VASigCookie *pVASigCookie, MethodDesc *pMD)
{
PInvokeStaticSigInfo sigInfo(pMD);
+ if (pMD->IsAsync2Method())
+ {
+ ThrowHR(COR_E_NOTSUPPORTED);
+ }
md = pMD->GetMemberDef();
nlFlags = sigInfo.GetLinkFlags();
nlType = sigInfo.GetCharSet();
diff --git a/src/coreclr/vm/dllimport.h b/src/coreclr/vm/dllimport.h
index 4f6b791748ea93..b53222626757c8 100644
--- a/src/coreclr/vm/dllimport.h
+++ b/src/coreclr/vm/dllimport.h
@@ -204,6 +204,7 @@ enum ILStubTypes
ILSTUB_STATIC_VIRTUAL_DISPATCH_STUB = 0x8000000A,
ILSTUB_DELEGATE_INVOKE_METHOD = 0x8000000B,
ILSTUB_DELEGATE_SHUFFLE_THUNK = 0x8000000C,
+ ILSTUB_ASYNC_RESUME = 0x8000000D,
};
#ifdef FEATURE_COMINTEROP
@@ -242,6 +243,7 @@ inline bool SF_IsInstantiatingStub (DWORD dwStubFlags) { LIMITED_METHOD_CON
inline bool SF_IsTailCallStoreArgsStub (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags == ILSTUB_TAILCALL_STOREARGS); }
inline bool SF_IsTailCallCallTargetStub (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags == ILSTUB_TAILCALL_CALLTARGET); }
inline bool SF_IsDelegateShuffleThunk (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags == ILSTUB_DELEGATE_SHUFFLE_THUNK); }
+inline bool SF_IsAsyncResumeStub (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return (dwStubFlags == ILSTUB_ASYNC_RESUME); }
inline bool SF_IsCOMStub (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return COM_ONLY(dwStubFlags < NDIRECTSTUB_FL_INVALID && 0 != (dwStubFlags & NDIRECTSTUB_FL_COM)); }
inline bool SF_IsCOMLateBoundStub (DWORD dwStubFlags) { LIMITED_METHOD_CONTRACT; return COM_ONLY(dwStubFlags < NDIRECTSTUB_FL_INVALID && 0 != (dwStubFlags & NDIRECTSTUB_FL_COMLATEBOUND)); }
@@ -263,6 +265,11 @@ inline bool SF_IsSharedStub(DWORD dwStubFlags)
return false;
}
+ if (SF_IsAsyncResumeStub(dwStubFlags))
+ {
+ return false;
+ }
+
return true;
}
diff --git a/src/coreclr/vm/dynamicmethod.cpp b/src/coreclr/vm/dynamicmethod.cpp
index 2fc4698f2b3b4e..4b9df31b3e21e7 100644
--- a/src/coreclr/vm/dynamicmethod.cpp
+++ b/src/coreclr/vm/dynamicmethod.cpp
@@ -164,7 +164,7 @@ void DynamicMethodTable::AddMethodsToList()
// allocate as many chunks as needed to hold the methods
//
MethodDescChunk* pChunk = MethodDescChunk::CreateChunk(pHeap, 0 /* one chunk of maximum size */,
- mcDynamic, TRUE /* fNonVtableSlot */, TRUE /* fNativeCodeSlot */, m_pMethodTable, &amt);
+ mcDynamic, TRUE /* fNonVtableSlot */, TRUE /* fNativeCodeSlot */, FALSE /* HasAsyncMethodData */, m_pMethodTable, &amt);
if (m_DynamicMethodList) RETURN;
int methodCount = pChunk->GetCount();
diff --git a/src/coreclr/vm/ecalllist.h b/src/coreclr/vm/ecalllist.h
index 9fcd6082b5edc6..7fa6ae288bb697 100644
--- a/src/coreclr/vm/ecalllist.h
+++ b/src/coreclr/vm/ecalllist.h
@@ -345,6 +345,7 @@ FCFuncStart(gMethodTableFuncs)
FCFuncElement("GetPrimitiveCorElementType", MethodTableNative::GetPrimitiveCorElementType)
FCFuncElement("GetMethodTableMatchingParentClass", MethodTableNative::GetMethodTableMatchingParentClass)
FCFuncElement("InstantiationArg0", MethodTableNative::InstantiationArg0)
+ FCFuncElement("GetLoaderAllocatorHandle", MethodTableNative::GetLoaderAllocatorHandle)
FCFuncEnd()
FCFuncStart(gStubHelperFuncs)
diff --git a/src/coreclr/vm/frames.cpp b/src/coreclr/vm/frames.cpp
index 3f9bdb122c5bf1..b542538ed6ba58 100644
--- a/src/coreclr/vm/frames.cpp
+++ b/src/coreclr/vm/frames.cpp
@@ -1357,7 +1357,8 @@ void HijackFrame::GcScanRoots_Impl(promote_func *fn, ScanContext* sc)
{
LIMITED_METHOD_CONTRACT;
- ReturnKind returnKind = m_Thread->GetHijackReturnKind();
+ bool hasAsyncRet;
+ ReturnKind returnKind = m_Thread->GetHijackReturnKind(&hasAsyncRet);
_ASSERTE(IsValidReturnKind(returnKind));
int regNo = 0;
@@ -1395,6 +1396,15 @@ void HijackFrame::GcScanRoots_Impl(promote_func *fn, ScanContext* sc)
regNo++;
} while (moreRegisters);
+
+ if (hasAsyncRet)
+ {
+ PTR_PTR_Object objPtr = dac_cast(&m_Args->AsyncRet);
+ LOG((LF_GC, INFO3, "Hijack Frame Promoting Async Continuation Return" FMT_ADDR "to",
+ DBG_ADDR(OBJECTREF_TO_UNCHECKED_OBJECTREF(*objPtr))));
+ (*fn)(objPtr, sc, CHECK_APP_DOMAIN);
+ LOG((LF_GC, INFO3, FMT_ADDR "\n", DBG_ADDR(OBJECTREF_TO_UNCHECKED_OBJECTREF(*objPtr))));
+ }
}
#endif // TARGET_X86
#endif // FEATURE_HIJACK
@@ -1506,6 +1516,9 @@ void TransitionFrame::PromoteCallerStack(promote_func* fn, ScanContext* sc)
if (pFunction->RequiresInstArg() && !SuppressParamTypeArg())
msig.SetHasParamTypeArg();
+ if (pFunction->IsAsync2Method())
+ msig.SetIsAsyncCall();
+
PromoteCallerStackHelper (fn, sc, pFunction, &msig);
}
else
@@ -2339,9 +2352,17 @@ void ComputeCallRefMap(MethodDesc* pMD,
// See code:getMethodSigInternal
//
assert(!isDispatchCell || !pMD->RequiresInstArg() || pMD->GetMethodTable()->IsInterface());
- if (pMD->RequiresInstArg() && !isDispatchCell)
+ if (!isDispatchCell)
{
- msig.SetHasParamTypeArg();
+ if (pMD->RequiresInstArg())
+ {
+ msig.SetHasParamTypeArg();
+ }
+
+ if (pMD->IsAsync2Method())
+ {
+ msig.SetIsAsyncCall();
+ }
}
ArgIterator argit(&msig);
diff --git a/src/coreclr/vm/gcenv.ee.common.cpp b/src/coreclr/vm/gcenv.ee.common.cpp
index 427d02ec20bec2..5d7fe777582aad 100644
--- a/src/coreclr/vm/gcenv.ee.common.cpp
+++ b/src/coreclr/vm/gcenv.ee.common.cpp
@@ -441,7 +441,7 @@ StackWalkAction GcStackCrawlCallBack(CrawlFrame* pCF, VOID* pData)
if (paramContextType == GENERIC_PARAM_CONTEXT_METHODDESC)
{
MethodDesc *pMDReal = dac_cast(pCF->GetParamTypeArg());
- _ASSERTE((pMDReal != NULL) || !pCF->IsFrameless());
+ _ASSERTE((pMDReal != NULL) || !pCF->IsFrameless() || pMD->IsAsync2Method());
if (pMDReal != NULL)
{
GcReportLoaderAllocator(gcctx->f, gcctx->sc, pMDReal->GetLoaderAllocator());
@@ -450,7 +450,7 @@ StackWalkAction GcStackCrawlCallBack(CrawlFrame* pCF, VOID* pData)
else if (paramContextType == GENERIC_PARAM_CONTEXT_METHODTABLE)
{
MethodTable *pMTReal = dac_cast(pCF->GetParamTypeArg());
- _ASSERTE((pMTReal != NULL) || !pCF->IsFrameless());
+ _ASSERTE((pMTReal != NULL) || !pCF->IsFrameless() || pMD->IsAsync2Method());
if (pMTReal != NULL)
{
GcReportLoaderAllocator(gcctx->f, gcctx->sc, pMTReal->GetLoaderAllocator());
diff --git a/src/coreclr/vm/genericdict.cpp b/src/coreclr/vm/genericdict.cpp
index eea8def4f4ed9c..90d52a4f20016a 100644
--- a/src/coreclr/vm/genericdict.cpp
+++ b/src/coreclr/vm/genericdict.cpp
@@ -889,6 +889,7 @@ Dictionary::PopulateEntry(
uint32_t methodSlot = -1;
BOOL fRequiresDispatchStub = 0;
+ BOOL isAsync2Variant = 0;
if (isReadyToRunModule)
{
@@ -900,6 +901,7 @@ Dictionary::PopulateEntry(
isInstantiatingStub = ((methodFlags & ENCODE_METHOD_SIG_InstantiatingStub) != 0) || (kind == MethodEntrySlot);
isUnboxingStub = ((methodFlags & ENCODE_METHOD_SIG_UnboxingStub) != 0);
fMethodNeedsInstantiation = ((methodFlags & ENCODE_METHOD_SIG_MethodInstantiation) != 0);
+ isAsync2Variant = ((methodFlags & ENCODE_METHOD_SIG_Async2Variant) != 0);
if (methodFlags & ENCODE_METHOD_SIG_OwnerType)
{
@@ -950,6 +952,10 @@ Dictionary::PopulateEntry(
_ASSERTE(pZapSigContext->pInfoModule->IsFullModule());
pMethod = MemberLoader::GetMethodDescFromMethodDef(static_cast(pZapSigContext->pInfoModule), TokenFromRid(rid, mdtMethodDef), FALSE);
}
+ if (isAsync2Variant)
+ {
+ pMethod = pMethod->GetAsyncOtherVariant();
+ }
}
if (ownerType.IsNull())
@@ -993,6 +999,7 @@ Dictionary::PopulateEntry(
isInstantiatingStub = ((methodFlags & ENCODE_METHOD_SIG_InstantiatingStub) != 0);
isUnboxingStub = ((methodFlags & ENCODE_METHOD_SIG_UnboxingStub) != 0);
fMethodNeedsInstantiation = ((methodFlags & ENCODE_METHOD_SIG_MethodInstantiation) != 0);
+ isAsync2Variant = ((methodFlags & ENCODE_METHOD_SIG_Async2Variant) != 0);
if ((methodFlags & ENCODE_METHOD_SIG_SlotInsteadOfToken) != 0)
{
@@ -1034,11 +1041,18 @@ Dictionary::PopulateEntry(
// The RID map should have been filled out if we fully loaded the class
pMethod = pMethodDefMT->GetModule()->LookupMethodDef(token);
+
+ if (isAsync2Variant)
+ {
+ pMethod = pMethod->GetAsyncOtherVariant();
+ }
+
_ASSERTE(pMethod != NULL);
pMethod->CheckRestore();
}
}
+
if (fRequiresDispatchStub)
{
LoaderAllocator * pDictLoaderAllocator = (pMT != NULL) ? pMT->GetLoaderAllocator() : pMD->GetLoaderAllocator();
@@ -1063,6 +1077,8 @@ Dictionary::PopulateEntry(
break;
}
+ _ASSERTE((!!isAsync2Variant) == pMethod->IsAsync2VariantMethod());
+
Instantiation inst;
// Instantiate the method if needed, or create a stub to a static method in a generic class.
@@ -1107,6 +1123,8 @@ Dictionary::PopulateEntry(
inst,
(!isInstantiatingStub && !isUnboxingStub));
+ _ASSERTE((!!isAsync2Variant) == pMethod->IsAsync2VariantMethod());
+
if (kind == ConstrainedMethodEntrySlot)
{
if (isReadyToRunModule)
diff --git a/src/coreclr/vm/genmeth.cpp b/src/coreclr/vm/genmeth.cpp
index af6c53ee9d9bc7..55bf8be02942b2 100644
--- a/src/coreclr/vm/genmeth.cpp
+++ b/src/coreclr/vm/genmeth.cpp
@@ -91,6 +91,7 @@ static MethodDesc* CreateMethodDesc(LoaderAllocator *pAllocator,
classification,
TRUE /* fNonVtableSlot*/,
fNativeCodeSlot,
+ pTemplateMD->HasAsyncMethodData(),
pMT,
pamTracker,
pLoaderModule);
@@ -118,6 +119,10 @@ static MethodDesc* CreateMethodDesc(LoaderAllocator *pAllocator,
{
pMD->SetIsIntrinsic();
}
+ if (pTemplateMD->HasAsyncMethodData())
+ {
+ pMD->SetHasAsyncMethodData();
+ }
#ifdef FEATURE_METADATA_UPDATER
if (pTemplateMD->IsEnCAddedMethod())
@@ -129,6 +134,11 @@ static MethodDesc* CreateMethodDesc(LoaderAllocator *pAllocator,
pMD->SetMemberDef(token);
pMD->SetSlot(pTemplateMD->GetSlot());
+ if (pTemplateMD->HasAsyncMethodData())
+ {
+ *pMD->GetAddrOfAsyncMethodData() = pTemplateMD->GetAsyncMethodData();
+ }
+
#ifdef _DEBUG
// more info here
pMD->m_pszDebugMethodSignature = "";
@@ -236,6 +246,7 @@ static MethodDesc * FindTightlyBoundWrappedMethodDesc_DEBUG(MethodDesc * pMD)
mdMethodDef methodDef = pMD->GetMemberDef();
Module *pModule = pMD->GetModule();
+ bool isAsync2VariantMethod = pMD->IsAsync2VariantMethod();
MethodTable::MethodIterator it(pMD->GetCanonicalMethodTable());
it.MoveToEnd();
@@ -246,7 +257,8 @@ static MethodDesc * FindTightlyBoundWrappedMethodDesc_DEBUG(MethodDesc * pMD)
if (pCurMethod && !pCurMethod->IsUnboxingStub()) {
if ((pCurMethod->GetMemberDef() == methodDef) &&
- (pCurMethod->GetModule() == pModule))
+ (pCurMethod->GetModule() == pModule) &&
+ (pCurMethod->IsAsync2VariantMethod() == isAsync2VariantMethod))
{
return pCurMethod;
}
@@ -274,6 +286,7 @@ static MethodDesc * FindTightlyBoundUnboxingStub_DEBUG(MethodDesc * pMD)
mdMethodDef methodDef = pMD->GetMemberDef();
Module *pModule = pMD->GetModule();
+ bool isAsync2VariantMethod = pMD->IsAsync2VariantMethod();
MethodTable::MethodIterator it(pMD->GetCanonicalMethodTable());
it.MoveToEnd();
@@ -282,7 +295,8 @@ static MethodDesc * FindTightlyBoundUnboxingStub_DEBUG(MethodDesc * pMD)
MethodDesc* pCurMethod = it.GetMethodDesc();
if (pCurMethod && pCurMethod->IsUnboxingStub()) {
if ((pCurMethod->GetMemberDef() == methodDef) &&
- (pCurMethod->GetModule() == pModule)) {
+ (pCurMethod->GetModule() == pModule) &&
+ (pCurMethod->IsAsync2VariantMethod() == isAsync2VariantMethod)) {
return pCurMethod;
}
}
@@ -348,7 +362,8 @@ InstantiatedMethodDesc::NewInstantiatedMethodDesc(MethodTable *pExactMT,
pNewMD = FindLoadedInstantiatedMethodDesc(pExactMT,
pGenericMDescInRepMT->GetMemberDef(),
methodInst,
- getWrappedCode);
+ getWrappedCode,
+ pGenericMDescInRepMT->IsAsync2VariantMethod());
// Crst goes out of scope here
// We don't need to hold the crst while we build the MethodDesc, but we reacquire it later
@@ -468,7 +483,8 @@ InstantiatedMethodDesc::NewInstantiatedMethodDesc(MethodTable *pExactMT,
InstantiatedMethodDesc *pOldMD = FindLoadedInstantiatedMethodDesc(pExactMT,
pGenericMDescInRepMT->GetMemberDef(),
methodInst,
- getWrappedCode);
+ getWrappedCode,
+ pGenericMDescInRepMT->IsAsync2VariantMethod());
if (pOldMD == NULL)
{
@@ -542,7 +558,8 @@ InstantiatedMethodDesc::FindOrCreateExactClassMethod(MethodTable *pExactMT,
InstantiatedMethodDesc *pInstMD = FindLoadedInstantiatedMethodDesc(pExactMT,
pCanonicalMD->GetMemberDef(),
Instantiation(),
- FALSE);
+ FALSE,
+ pCanonicalMD->IsAsync2VariantMethod());
if (pInstMD == NULL)
{
@@ -564,7 +581,8 @@ InstantiatedMethodDesc*
InstantiatedMethodDesc::FindLoadedInstantiatedMethodDesc(MethodTable *pExactOrRepMT,
mdMethodDef methodDef,
Instantiation methodInst,
- BOOL getWrappedCode)
+ BOOL getWrappedCode,
+ BOOL asyncThunk)
{
CONTRACT(InstantiatedMethodDesc *)
{
@@ -598,7 +616,8 @@ InstantiatedMethodDesc::FindLoadedInstantiatedMethodDesc(MethodTable *pExactOrRe
methodDef,
FALSE /* not forceBoxedEntryPoint */,
methodInst,
- getWrappedCode);
+ getWrappedCode,
+ asyncThunk);
if (resultMD != NULL)
RETURN((InstantiatedMethodDesc*) resultMD);
@@ -723,6 +742,7 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
BOOL allowInstParam,
BOOL forceRemotableMethod,
BOOL allowCreate,
+ AsyncVariantLookup asyncVariantLookup,
ClassLoadLevel level)
{
CONTRACT(MethodDesc*)
@@ -758,7 +778,8 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
if (!pDefMD->HasClassOrMethodInstantiation() &&
methodInst.IsEmpty() &&
!forceBoxedEntryPoint &&
- !pDefMD->IsUnboxingStub())
+ !pDefMD->IsUnboxingStub() &&
+ asyncVariantLookup == AsyncVariantLookup::MatchingAsyncVariant)
{
// Make sure that pDefMD->GetMethodTable() and pExactMT are related types even
// if we took the fast path.
@@ -787,7 +808,7 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
COMPlusThrowHR(COR_E_TYPELOAD);
}
- if (pDefMD->HasClassOrMethodInstantiation() || !methodInst.IsEmpty())
+ if (pDefMD->HasClassOrMethodInstantiation() || !methodInst.IsEmpty() || asyncVariantLookup == AsyncVariantLookup::AsyncOtherVariant)
{
// General checks related to generics: arity (if any) must match and generic method
// instantiation (if any) must be well-formed.
@@ -797,7 +818,7 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
COMPlusThrowHR(COR_E_BADIMAGEFORMAT);
}
- pMDescInCanonMT = pExactMT->GetCanonicalMethodTable()->GetParallelMethodDesc(pDefMD);
+ pMDescInCanonMT = pExactMT->GetCanonicalMethodTable()->GetParallelMethodDesc(pDefMD, asyncVariantLookup);
if (!allowCreate && !pMDescInCanonMT->GetMethodTable()->IsFullyLoaded())
{
@@ -873,7 +894,8 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
methodDef,
TRUE /* forceBoxedEntryPoint */,
Instantiation(),
- FALSE /* no inst param */);
+ FALSE /* no inst param */,
+ pMDescInCanonMT->IsAsync2VariantMethod());
// If we didn't find it then create it...
if (!pResultMD)
@@ -891,7 +913,8 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
methodDef,
TRUE,
Instantiation(),
- FALSE);
+ FALSE,
+ pMDescInCanonMT->IsAsync2VariantMethod());
if (pResultMD == NULL)
{
AllocMemTracker amt;
@@ -937,7 +960,8 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
methodDef,
TRUE, /* forceBoxedEntryPoint */
methodInst,
- FALSE /* no inst param */);
+ FALSE /* no inst param */,
+ pMDescInCanonMT->IsAsync2VariantMethod());
if (!pResultMD)
{
@@ -954,11 +978,12 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
pExactMT,
FALSE /* not Unboxing */,
methodInst,
- FALSE);
+ FALSE, FALSE, TRUE, asyncVariantLookup);
_ASSERTE(pNonUnboxingStub->GetClassification() == mcInstantiated);
_ASSERTE(!pNonUnboxingStub->RequiresInstArg());
_ASSERTE(!pNonUnboxingStub->IsUnboxingStub());
+ _ASSERTE(pNonUnboxingStub->IsAsync2VariantMethod() == pMDescInCanonMT->IsAsync2VariantMethod());
// Enter the critical section *after* we've found or created the non-unboxing instantiating stub (else we'd have a race,
// and its possible that the non-unboxing instantiating stub may be in a different loader module than pLoaderModule
@@ -970,7 +995,8 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
methodDef,
TRUE, /* forceBoxedEntryPoint */
methodInst,
- FALSE /* no inst param */);
+ FALSE /* no inst param */,
+ pNonUnboxingStub->IsAsync2VariantMethod());
if (pResultMD == NULL)
{
@@ -1111,7 +1137,8 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
InstantiatedMethodDesc::FindLoadedInstantiatedMethodDesc(pExactMT->GetCanonicalMethodTable(),
methodDef,
Instantiation(repInst, methodInst.GetNumArgs()),
- TRUE);
+ TRUE,
+ pMDescInCanonMT->IsAsync2VariantMethod());
// No - so create one.
if (pInstMD == NULL)
@@ -1135,7 +1162,8 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
InstantiatedMethodDesc::FindLoadedInstantiatedMethodDesc(pExactMT,
methodDef,
methodInst,
- FALSE);
+ FALSE,
+ pMDescInCanonMT->IsAsync2VariantMethod());
// No - so create one. Go fetch the shared one first
if (pInstMD == NULL)
@@ -1154,6 +1182,7 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
/* allowInstParam */ TRUE,
/* forceRemotableMethod */ FALSE,
/* allowCreate */ TRUE,
+ asyncVariantLookup,
/* level */ level);
_ASSERTE(pWrappedMD->IsSharedByGenericInstantiations());
@@ -1174,7 +1203,8 @@ MethodDesc::FindOrCreateAssociatedMethodDesc(MethodDesc* pDefMD,
InstantiatedMethodDesc::FindLoadedInstantiatedMethodDesc(pExactMT,
methodDef,
methodInst,
- FALSE);
+ FALSE,
+ pMDescInCanonMT->IsAsync2VariantMethod());
// No - so create one.
if (pInstMD == NULL)
diff --git a/src/coreclr/vm/i386/cgencpu.h b/src/coreclr/vm/i386/cgencpu.h
index 50ae84f4bed81d..86fda62df1550f 100644
--- a/src/coreclr/vm/i386/cgencpu.h
+++ b/src/coreclr/vm/i386/cgencpu.h
@@ -405,7 +405,11 @@ struct HijackArgs
DWORD Esi;
DWORD Ebx;
DWORD Edx;
- DWORD Ecx;
+ union
+ {
+ DWORD Ecx;
+ size_t AsyncRet;
+ };
union
{
DWORD Eax;
diff --git a/src/coreclr/vm/ilstubcache.cpp b/src/coreclr/vm/ilstubcache.cpp
index 655a6b2d169f3f..dd968b92abe64c 100644
--- a/src/coreclr/vm/ilstubcache.cpp
+++ b/src/coreclr/vm/ilstubcache.cpp
@@ -158,6 +158,7 @@ namespace
case DynamicMethodDesc::StubTailCallCallTarget: return "IL_STUB_CallTailCallTarget";
case DynamicMethodDesc::StubVirtualStaticMethodDispatch: return "IL_STUB_VirtualStaticMethodDispatch";
case DynamicMethodDesc::StubDelegateShuffleThunk: return "IL_STUB_DelegateShuffleThunk";
+ case DynamicMethodDesc::StubAsyncResume: return "IL_STUB_AsyncResume";
default:
UNREACHABLE_MSG("Unknown stub type");
}
@@ -183,6 +184,7 @@ MethodDesc* ILStubCache::CreateNewMethodDesc(LoaderHeap* pCreationHeap, MethodTa
mcDynamic,
TRUE /* fNonVtableSlot */,
TRUE /* fNativeCodeSlot */,
+ FALSE /* HasAsyncMethodData */,
pMT,
pamTracker);
@@ -276,6 +278,11 @@ MethodDesc* ILStubCache::CreateNewMethodDesc(LoaderHeap* pCreationHeap, MethodTa
pMD->SetILStubType(DynamicMethodDesc::StubTailCallCallTarget);
}
else
+ if (SF_IsAsyncResumeStub(dwStubFlags))
+ {
+ pMD->SetILStubType(DynamicMethodDesc::StubAsyncResume);
+ }
+ else
#ifdef FEATURE_COMINTEROP
if (SF_IsCOMStub(dwStubFlags))
{
diff --git a/src/coreclr/vm/instmethhash.cpp b/src/coreclr/vm/instmethhash.cpp
index cdb8b8617dec88..4076925daff089 100644
--- a/src/coreclr/vm/instmethhash.cpp
+++ b/src/coreclr/vm/instmethhash.cpp
@@ -118,7 +118,8 @@ MethodDesc* InstMethodHashTable::FindMethodDesc(TypeHandle declaringType,
mdMethodDef token,
BOOL unboxingStub,
Instantiation inst,
- BOOL getSharedNotStub)
+ BOOL getSharedNotStub,
+ bool isAsync2Variant)
{
CONTRACTL
{
@@ -160,6 +161,11 @@ MethodDesc* InstMethodHashTable::FindMethodDesc(TypeHandle declaringType,
continue; // Next iteration of the for loop
}
+ if (pMD->IsAsync2VariantMethod() != isAsync2Variant)
+ {
+ continue;
+ }
+
if (!inst.IsEmpty())
{
Instantiation candidateInst = pMD->GetMethodInstantiation();
@@ -202,7 +208,7 @@ BOOL InstMethodHashTable::ContainsMethodDesc(MethodDesc* pMD)
return FindMethodDesc(
pMD->GetMethodTable(), pMD->GetMemberDef(), pMD->IsUnboxingStub(),
- pMD->GetMethodInstantiation(), pMD->RequiresInstArg()) != NULL;
+ pMD->GetMethodInstantiation(), pMD->RequiresInstArg(), pMD->IsAsync2VariantMethod()) != NULL;
}
#endif // #ifndef DACCESS_COMPILE
diff --git a/src/coreclr/vm/instmethhash.h b/src/coreclr/vm/instmethhash.h
index e7844088637047..d678a4232a150d 100644
--- a/src/coreclr/vm/instmethhash.h
+++ b/src/coreclr/vm/instmethhash.h
@@ -109,7 +109,8 @@ class InstMethodHashTable : public DacEnumerableHashTable &rDef
{
pDeclaringMT = pProps->pMeth->GetMethodTable();
tkMb = pProps->pMeth->GetMemberDef();
+ if (pProps->pMeth->IsAsync2Method())
+ {
+ ThrowHR(COR_E_NOTSUPPORTED);
+ }
cbCur = GetStringizedMethodDef(pDeclaringMT, tkMb, rDef, cbCur);
}
else
@@ -2490,6 +2494,9 @@ BOOL IsMethodVisibleFromCom(MethodDesc *pMD)
mdProperty pd;
LPCUTF8 pPropName;
ULONG uSemantic;
+ if (pMD->IsAsync2Method())
+ return false;
+
mdMethodDef md = pMD->GetMemberDef();
// See if there is property information for this member.
diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp
index 608d55dd8460d6..18312e040f1e38 100644
--- a/src/coreclr/vm/jithelpers.cpp
+++ b/src/coreclr/vm/jithelpers.cpp
@@ -52,6 +52,7 @@
#include "onstackreplacement.h"
#include "pgo.h"
#include "pgo_formatprocessing.h"
+#include "patchpointinfo.h"
#ifndef FEATURE_EH_FUNCLETS
#include "excep.h"
@@ -1483,6 +1484,37 @@ HCIMPL0(void, IL_Rethrow)
}
HCIMPLEND
+HCIMPL1(void, IL_ThrowExact, Object* obj)
+{
+ FCALL_CONTRACT;
+
+ /* Make no assumptions about the current machine state */
+ ResetCurrentContext();
+
+ FC_GC_POLL_NOT_NEEDED(); // throws always open up for GC
+
+ HELPER_METHOD_FRAME_BEGIN_ATTRIB_NOPOLL(Frame::FRAME_ATTR_EXCEPTION); // Set up a frame
+
+ OBJECTREF oref = ObjectToOBJECTREF(obj);
+
+#if defined(_DEBUG) && defined(TARGET_X86)
+ __helperframe.EnsureInit(NULL);
+ g_ExceptionEIP = (LPVOID)__helperframe.GetReturnAddress();
+#endif // defined(_DEBUG) && defined(TARGET_X86)
+
+ GetThread()->GetExceptionState()->SetRaisingForeignException();
+
+#ifdef FEATURE_EH_FUNCLETS
+ DispatchManagedException(oref);
+ UNREACHABLE();
+#endif
+
+ RaiseTheExceptionInternalOnly(oref, FALSE);
+
+ HELPER_METHOD_FRAME_END();
+}
+HCIMPLEND
+
#ifndef STATUS_STACK_BUFFER_OVERRUN // Not defined yet in CESDK includes
# define STATUS_STACK_BUFFER_OVERRUN ((NTSTATUS)0xC0000409L)
#endif
@@ -2346,6 +2378,179 @@ extern "C" void JIT_PatchpointWorkerWorkerWithPolicy(TransitionBlock * pTransiti
::SetLastError(dwLastError);
}
+extern "C" void JIT_ResumeOSRWorker(TransitionBlock * pTransitionBlock)
+{
+ // BEGIN_PRESERVE_LAST_ERROR;
+ DWORD dwLastError = ::GetLastError();
+
+ // This method may not return normally
+ STATIC_CONTRACT_NOTHROW;
+ STATIC_CONTRACT_GC_TRIGGERS;
+ STATIC_CONTRACT_MODE_COOPERATIVE;
+
+ PTR_PCODE pReturnAddress = (PTR_PCODE)(((BYTE*)pTransitionBlock) + TransitionBlock::GetOffsetOfReturnAddress());
+ PCODE ip = *pReturnAddress;
+ int ilOffset = *(int*)GetFirstArgumentRegisterValuePtr(pTransitionBlock);
+
+ // Find OSR method code for this IL offset.
+
+ PCODE osrMethodCode = 0;
+
+ {
+ EECodeInfo codeInfo(ip);
+ MethodDesc* pMD = codeInfo.GetMethodDesc();
+ PCODE tier0EntryAddr = (PCODE)codeInfo.GetStartAddress();
+ CodeVersionManager* codeVersionManager = pMD->GetCodeVersionManager();
+ CodeVersionManager::LockHolder lock;
+ NativeCodeVersionCollection nativeCodeVersions = codeVersionManager->GetNativeCodeVersions(pMD);
+ for (NativeCodeVersionIterator itr = nativeCodeVersions.Begin(), end = nativeCodeVersions.End(); itr != end; itr++)
+ {
+ if (itr->GetOptimizationTier() == NativeCodeVersion::OptimizationTier1OSR)
+ {
+ unsigned versionILOffset;
+ PatchpointInfo* ppi = itr->GetOSRInfo(&versionILOffset);
+ if (ppi->GetTier0EntryPoint() == tier0EntryAddr && versionILOffset == (unsigned)ilOffset)
+ {
+ osrMethodCode = itr->GetNativeCode();
+ break;
+ }
+ }
+ }
+ }
+
+ if (osrMethodCode == (PCODE)NULL)
+ {
+ STRESS_LOG2(LF_TIEREDCOMPILATION, LL_FATALERROR, "JIT_ResumeOSRWorker: patchpoint (0x%p:%d) TRANSITION", ip, ilOffset);
+ _ASSERTE(!"Did not find OSR method code");
+ EEPOLICY_HANDLE_FATAL_ERROR(COR_E_EXECUTIONENGINE);
+ }
+
+ // If we get here, we have code to transition to...
+
+ {
+ Thread *pThread = GetThread();
+
+#ifdef FEATURE_HIJACK
+ // We can't crawl the stack of a thread that currently has a hijack pending
+ // (since the hijack routine won't be recognized by any code manager). So we
+ // Undo any hijack, the EE will re-attempt it later.
+ pThread->UnhijackThread();
+#endif
+
+ // Find context for the original method
+ CONTEXT *pFrameContext = NULL;
+#if defined(TARGET_WINDOWS) && defined(TARGET_AMD64)
+ DWORD contextSize = 0;
+ ULONG64 xStateCompactionMask = 0;
+ DWORD contextFlags = CONTEXT_FULL;
+ if (Thread::AreShadowStacksEnabled())
+ {
+ xStateCompactionMask = XSTATE_MASK_CET_U;
+ contextFlags |= CONTEXT_XSTATE;
+ }
+
+ // The initialize call should fail but return contextSize
+ BOOL success = g_pfnInitializeContext2 ?
+ g_pfnInitializeContext2(NULL, contextFlags, NULL, &contextSize, xStateCompactionMask) :
+ InitializeContext(NULL, contextFlags, NULL, &contextSize);
+
+ _ASSERTE(!success && (GetLastError() == ERROR_INSUFFICIENT_BUFFER));
+
+ PVOID pBuffer = _alloca(contextSize);
+ success = g_pfnInitializeContext2 ?
+ g_pfnInitializeContext2(pBuffer, contextFlags, &pFrameContext, &contextSize, xStateCompactionMask) :
+ InitializeContext(pBuffer, contextFlags, &pFrameContext, &contextSize);
+ _ASSERTE(success);
+#else // TARGET_WINDOWS && TARGET_AMD64
+ CONTEXT frameContext;
+ frameContext.ContextFlags = CONTEXT_FULL;
+ pFrameContext = &frameContext;
+#endif // TARGET_WINDOWS && TARGET_AMD64
+
+ // Find context for the original method
+ RtlCaptureContext(pFrameContext);
+
+#if defined(TARGET_WINDOWS) && defined(TARGET_AMD64)
+ if (Thread::AreShadowStacksEnabled())
+ {
+ pFrameContext->ContextFlags |= CONTEXT_XSTATE;
+ SetXStateFeaturesMask(pFrameContext, xStateCompactionMask);
+ SetSSP(pFrameContext, _rdsspq());
+ }
+#endif // TARGET_WINDOWS && TARGET_AMD64
+
+ // Walk back to the original method frame
+ pThread->VirtualUnwindToFirstManagedCallFrame(pFrameContext);
+
+ // Remember original method FP and SP because new method will inherit them.
+ UINT_PTR currentSP = GetSP(pFrameContext);
+ UINT_PTR currentFP = GetFP(pFrameContext);
+
+ // We expect to be back at the right IP
+ if ((UINT_PTR)ip != GetIP(pFrameContext))
+ {
+ // Should be fatal
+ STRESS_LOG2(LF_TIEREDCOMPILATION, LL_FATALERROR, "Jit_Patchpoint: patchpoint (0x%p) TRANSITION"
+ " unexpected context IP 0x%p\n", ip, GetIP(pFrameContext));
+ EEPOLICY_HANDLE_FATAL_ERROR(COR_E_EXECUTIONENGINE);
+ }
+
+ // Now unwind back to the original method caller frame.
+ EECodeInfo callerCodeInfo(GetIP(pFrameContext));
+ ULONG_PTR establisherFrame = 0;
+ PVOID handlerData = NULL;
+ RtlVirtualUnwind(UNW_FLAG_NHANDLER, callerCodeInfo.GetModuleBase(), GetIP(pFrameContext), callerCodeInfo.GetFunctionEntry(),
+ pFrameContext, &handlerData, &establisherFrame, NULL);
+
+ // Now, set FP and SP back to the values they had just before this helper was called,
+ // since the new method must have access to the original method frame.
+ //
+ // TODO: if we access the patchpointInfo here, we can read out the FP-SP delta from there and
+ // use that to adjust the stack, likely saving some stack space.
+
+#if defined(TARGET_AMD64)
+ // If calls push the return address, we need to simulate that here, so the OSR
+ // method sees the "expected" SP misalgnment on entry.
+ _ASSERTE(currentSP % 16 == 0);
+ currentSP -= 8;
+
+#if defined(TARGET_WINDOWS)
+ DWORD64 ssp = GetSSP(pFrameContext);
+ if (ssp != 0)
+ {
+ SetSSP(pFrameContext, ssp - 8);
+ }
+#endif // TARGET_WINDOWS
+
+ pFrameContext->Rbp = currentFP;
+#endif // TARGET_AMD64
+
+ SetSP(pFrameContext, currentSP);
+
+ // Note we can get here w/o triggering, if there is an existing OSR method and
+ // we hit the patchpoint.
+ LOG((LF_TIEREDCOMPILATION, LL_INFO1000, "Jit_ResumeOSRWorker: patchpoint (0x%p:%d) TRANSITION to ip 0x%p\n", ip, ilOffset, osrMethodCode));
+
+ // Install new entry point as IP
+ SetIP(pFrameContext, osrMethodCode);
+
+#ifdef _DEBUG
+ // Keep this context around to aid in debugging OSR transition problems
+ static CONTEXT s_lastOSRTransitionContext;
+ s_lastOSRTransitionContext = *pFrameContext;
+#endif
+
+ // Restore last error (since call below does not return)
+ // END_PRESERVE_LAST_ERROR;
+ ::SetLastError(dwLastError);
+
+ // Transition!
+ ClrRestoreNonvolatileContext(pFrameContext);
+ }
+
+ // END_PRESERVE_LAST_ERROR;
+ ::SetLastError(dwLastError);
+}
#else
@@ -2369,6 +2574,16 @@ HCIMPL1(VOID, JIT_PartialCompilationPatchpoint, int ilOffset)
}
HCIMPLEND
+HCIMPL1(VOID, JIT_ResumeOSR, int ilOffset)
+{
+ // Stub version if OSR feature is disabled
+ //
+ // Should not be called.
+
+ UNREACHABLE();
+}
+HCIMPLEND
+
#endif // FEATURE_ON_STACK_REPLACEMENT
static unsigned HandleHistogramProfileRand()
diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp
index bb0f807ab00500..5e98fbbeccb43d 100644
--- a/src/coreclr/vm/jitinterface.cpp
+++ b/src/coreclr/vm/jitinterface.cpp
@@ -59,6 +59,7 @@
#endif
#include "tailcallhelp.h"
+#include "patchpointinfo.h"
// The Stack Overflow probe takes place in the COOPERATIVE_TRANSITION_BEGIN() macro
//
@@ -417,6 +418,8 @@ enum ConvToJitSigFlags : int
CONV_TO_JITSIG_FLAGS_LOCALSIG = 0x1,
};
+AsyncMethodSignatureKind ClassifyAsyncMethodSignatureCore(SigPointer sig, Module* pModule, PCCOR_SIGNATURE initialSig, ULONG* offsetOfAsyncDetails, bool *isValueTask);
+
//---------------------------------------------------------------------------------------
//
//@GENERICS:
@@ -510,6 +513,12 @@ static void ConvToJitSig(
sigRet->retType = CEEInfo::asCorInfoType(type, typeHnd, &sigRet->retTypeClass);
sigRet->retTypeSigClass = CORINFO_CLASS_HANDLE(typeHnd.AsPtr());
+ auto asyncMethodClassification = ClassifyAsyncMethodSignatureCore(sig, module, NULL, NULL, NULL);
+ if (IsAsyncSigAsync2(asyncMethodClassification))
+ {
+ sigRet->callConv = (CorInfoCallConv)(sigRet->callConv | CORINFO_CALLCONV_ASYNCCALL);
+ }
+
IfFailThrow(sig.SkipExactlyOne()); // must to a skip so we skip any class tokens associated with the return type
_ASSERTE(sigRet->retType < CORINFO_TYPE_COUNT);
@@ -1087,6 +1096,16 @@ void CEEInfo::resolveToken(/* IN, OUT */ CORINFO_RESOLVED_TOKEN * pResolvedToken
th = ClassLoader::LoadArrayTypeThrowing(th);
break;
+ case CORINFO_TOKENKIND_Await:
+ // in rare cases a method that returns Task is not actually TaskReturning (i.e. returns T).
+ // we cannot resolve to an async2 variant in such case.
+ // return NULL, so that caller would re-resolve as a regular method call
+ pMD = pMD->IsTaskReturningMethod() ?
+ pMD->GetAsyncOtherVariant(/*allowInstParam*/FALSE):
+ NULL;
+
+ break;
+
default:
// Disallow ELEMENT_TYPE_BYREF and ELEMENT_TYPE_VOID
if (et == ELEMENT_TYPE_BYREF || et == ELEMENT_TYPE_VOID)
@@ -3214,6 +3233,10 @@ void CEEInfo::ComputeRuntimeLookupForSharedGenericToken(DictionaryEntryKind entr
methodFlags |= ENCODE_METHOD_SIG_SlotInsteadOfToken;
}
+ if (pTemplateMD->IsAsync2VariantMethod())
+ {
+ methodFlags |= ENCODE_METHOD_SIG_Async2Variant;
+ }
sigBuilder.AppendData(methodFlags);
@@ -7580,7 +7603,7 @@ static void getMethodInfoHelper(
methInfo->EHcount = (unsigned short)EHCount;
localSig = pResolver->GetLocalSig();
}
- else if (ftn->TryGenerateUnsafeAccessor(&cxt.TransientResolver, &cxt.Header))
+ else if (ftn->TryGenerateTransientILImplementation(&cxt.TransientResolver, &cxt.Header))
{
scopeHnd = cxt.CreateScopeHandle();
@@ -7599,7 +7622,8 @@ static void getMethodInfoHelper(
methInfo->options = (CorInfoOptions)(((UINT32)methInfo->options) |
((ftn->AcquiresInstMethodTableFromThis() ? CORINFO_GENERICS_CTXT_FROM_THIS : 0) |
(ftn->RequiresInstMethodTableArg() ? CORINFO_GENERICS_CTXT_FROM_METHODTABLE : 0) |
- (ftn->RequiresInstMethodDescArg() ? CORINFO_GENERICS_CTXT_FROM_METHODDESC : 0)));
+ (ftn->RequiresInstMethodDescArg() ? CORINFO_GENERICS_CTXT_FROM_METHODDESC : 0) |
+ (ftn->IsStructMethodOperatingOnCopy() ? CORINFO_OPT_COPY_STRUCT_INSTANCE : 0)));
if (methInfo->options & CORINFO_GENERICS_CTXT_MASK)
{
@@ -10201,6 +10225,28 @@ void CEEInfo::getEEInfo(CORINFO_EE_INFO *pEEInfoOut)
EE_TO_JIT_TRANSITION();
}
+void CEEInfo::getAsync2Info(CORINFO_ASYNC2_INFO* pAsync2InfoOut)
+{
+ CONTRACTL {
+ THROWS;
+ GC_TRIGGERS;
+ MODE_PREEMPTIVE;
+ } CONTRACTL_END;
+
+ JIT_TO_EE_TRANSITION();
+
+ pAsync2InfoOut->continuationClsHnd = CORINFO_CLASS_HANDLE(CoreLibBinder::GetClass(CLASS__CONTINUATION));
+ pAsync2InfoOut->continuationNextFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__NEXT));
+ pAsync2InfoOut->continuationResumeFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__RESUME));
+ pAsync2InfoOut->continuationStateFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__STATE));
+ pAsync2InfoOut->continuationFlagsFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__FLAGS));
+ pAsync2InfoOut->continuationDataFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__DATA));
+ pAsync2InfoOut->continuationGCDataFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__GCDATA));
+ pAsync2InfoOut->continuationsNeedMethodHandle = m_pMethodBeingCompiled->GetLoaderAllocator()->CanUnload();
+
+ EE_TO_JIT_TRANSITION();
+}
+
// Return details about EE internal data structures
uint32_t CEEInfo::getThreadTLSIndex(void **ppIndirection)
{
@@ -10919,6 +10965,16 @@ void CEEJitInfo::WriteCodeBytes()
}
}
+void CEEJitInfo::PublishFinalCodeAddress(PCODE addr)
+{
+ LIMITED_METHOD_CONTRACT;
+
+ if (m_finalCodeAddressSlot != NULL)
+ {
+ *m_finalCodeAddressSlot = addr;
+ }
+}
+
/*********************************************************************/
void CEEJitInfo::BackoutJitData(EECodeGenManager * jitMgr)
{
@@ -11274,7 +11330,7 @@ void CInterpreterJitInfo::SetDebugInfo(PTR_BYTE pDebugInfo)
}
#endif // FEATURE_INTERPRETER
-void CEECodeGenInfo::CompressDebugInfo()
+void CEECodeGenInfo::CompressDebugInfo(PCODE nativeEntry)
{
CONTRACTL {
THROWS;
@@ -11294,6 +11350,9 @@ void CEECodeGenInfo::CompressDebugInfo()
if ((m_iOffsetMapping == 0) && (m_iNativeVarInfo == 0) && (patchpointInfo == NULL) && (m_numInlineTreeNodes == 0) && (m_numRichOffsetMappings == 0))
return;
+ if (patchpointInfo != NULL)
+ patchpointInfo->SetTier0EntryPoint(nativeEntry);
+
JIT_TO_EE_TRANSITION();
EX_TRY
@@ -12768,7 +12827,7 @@ CorJitResult invokeCompileMethodHelper(EECodeGenManager *jitMgr,
//
if (SUCCEEDED(ret) && !comp->JitAgain())
{
- comp->CompressDebugInfo();
+ comp->CompressDebugInfo((PCODE)*nativeEntry);
comp->MethodCompileComplete(info->ftn);
}
@@ -12918,6 +12977,9 @@ static CORJIT_FLAGS GetCompileFlags(PrepareCodeConfig* prepareConfig, MethodDesc
//
flags.Add(CEEInfo::GetBaseCompileFlags(ftn));
+ if (ftn->IsAsync2Method())
+ flags.Add(CORJIT_FLAGS::CORJIT_FLAG_RUNTIMEASYNCFUNCTION);
+
//
// Get CPU specific flags
//
@@ -13300,6 +13362,9 @@ PCODE UnsafeJitFunction(PrepareCodeConfig* config,
CORINFO_METHOD_INFO methodInfo;
getMethodInfoHelper(cxt, &methodInfo);
+ if (ILHeader == nullptr)
+ ILHeader = cxt.Header;
+
// If it's generic then we can only enter through an instantiated MethodDesc
_ASSERTE(!ftn->IsGenericMethodDefinition());
@@ -13398,6 +13463,8 @@ PCODE UnsafeJitFunction(PrepareCodeConfig* config,
ret |= THUMB_CODE;
#endif
+ jitInfo.PublishFinalCodeAddress(ret);
+
// We are done
break;
}
@@ -14476,6 +14543,373 @@ bool CEEInfo::getTailCallHelpers(CORINFO_RESOLVED_TOKEN* callToken,
return success;
}
+static Signature AllocateSignature(LoaderAllocator* alloc, SigBuilder& sigBuilder)
+{
+ DWORD sigLen;
+ PCCOR_SIGNATURE builderSig = (PCCOR_SIGNATURE)sigBuilder.GetSignature(&sigLen);
+ AllocMemTracker pamTracker;
+ PVOID newBlob = pamTracker.Track(alloc->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(sigLen)));
+ memcpy(newBlob, builderSig, sigLen);
+
+ pamTracker.SuppressRelease();
+ return Signature((PCCOR_SIGNATURE)newBlob, sigLen);
+}
+
+static Signature BuildResumptionStubSignature(LoaderAllocator* alloc)
+{
+ SigBuilder sigBuilder;
+ sigBuilder.AppendByte(IMAGE_CEE_CS_CALLCONV_DEFAULT);
+ sigBuilder.AppendData(1); // 1 argument
+ sigBuilder.AppendElementType(ELEMENT_TYPE_OBJECT); // return type
+ sigBuilder.AppendElementType(ELEMENT_TYPE_OBJECT); // continuation
+
+ return AllocateSignature(alloc, sigBuilder);
+}
+
+static Signature BuildResumptionStubCalliSignature(MetaSig& msig, MethodTable* mt, LoaderAllocator* alloc)
+{
+ unsigned numArgs = 0;
+ if (msig.HasThis())
+ {
+ numArgs++;
+ }
+
+ if (msig.HasGenericContextArg())
+ {
+ numArgs++;
+ }
+
+ numArgs++; // Continuation
+
+ numArgs += msig.NumFixedArgs();
+
+ SigBuilder sigBuilder;
+ sigBuilder.AppendByte(IMAGE_CEE_CS_CALLCONV_DEFAULT | IMAGE_CEE_CS_CALLCONV_HASTHIS | IMAGE_CEE_CS_CALLCONV_EXPLICITTHIS);
+ sigBuilder.AppendData(numArgs);
+
+ auto appendTypeHandle = [&](TypeHandle th) {
+ _ASSERTE(!th.IsByRef());
+ CorElementType ty = th.GetSignatureCorElementType();
+ if (CorTypeInfo::IsObjRef(ty))
+ {
+ // Especially to normalize System.__Canon.
+ sigBuilder.AppendElementType(ELEMENT_TYPE_OBJECT);
+ }
+ else if (CorTypeInfo::IsPrimitiveType(ty))
+ {
+ sigBuilder.AppendElementType(ty);
+ }
+ else
+ {
+ sigBuilder.AppendElementType(ELEMENT_TYPE_INTERNAL);
+ sigBuilder.AppendPointer(th.AsPtr());
+ }
+ };
+
+ appendTypeHandle(msig.GetRetTypeHandleThrowing()); // return type
+ if (msig.HasThis())
+ {
+ if (mt->IsValueType())
+ {
+ sigBuilder.AppendElementType(ELEMENT_TYPE_BYREF);
+ appendTypeHandle(TypeHandle(mt));
+ }
+ else
+ {
+ sigBuilder.AppendElementType(ELEMENT_TYPE_OBJECT);
+ }
+ }
+#ifndef TARGET_X86
+ if (msig.HasGenericContextArg())
+ {
+ sigBuilder.AppendElementType(ELEMENT_TYPE_I);
+ }
+
+ sigBuilder.AppendElementType(ELEMENT_TYPE_OBJECT); // continuation
+#endif
+
+ msig.Reset();
+ CorElementType ty;
+ while ((ty = msig.NextArg()) != ELEMENT_TYPE_END)
+ {
+ TypeHandle tyHnd = msig.GetLastTypeHandleThrowing();
+ appendTypeHandle(tyHnd);
+ }
+
+#ifdef TARGET_X86
+ if (msig.HasGenericContextArg())
+ {
+ sigBuilder.AppendElementType(ELEMENT_TYPE_I);
+ }
+
+ sigBuilder.AppendElementType(ELEMENT_TYPE_OBJECT); // continuation
+#endif
+
+ return AllocateSignature(alloc, sigBuilder);
+}
+
+CORINFO_METHOD_HANDLE CEEJitInfo::getAsyncResumptionStub()
+{
+ CONTRACTL{
+ THROWS;
+ GC_TRIGGERS;
+ MODE_PREEMPTIVE;
+ } CONTRACTL_END;
+
+ MethodDesc* md = m_pMethodBeingCompiled;
+
+ LoaderAllocator* loaderAlloc = md->GetLoaderAllocator();
+
+ Signature stubSig = BuildResumptionStubSignature(md->GetLoaderAllocator());
+
+ MetaSig msig(md);
+ Signature calliSig = BuildResumptionStubCalliSignature(msig, md->GetMethodTable(), md->GetLoaderAllocator());
+
+ SigTypeContext emptyCtx;
+ ILStubLinker sl(md->GetModule(), stubSig, &emptyCtx, NULL, ILSTUB_LINKER_FLAG_NONE);
+
+ ILCodeStream* pCode = sl.NewCodeStream(ILStubLinker::kDispatch);
+
+ int numArgs = 0;
+
+ if (msig.HasThis())
+ {
+ if (md->GetMethodTable()->IsValueType())
+ {
+ pCode->EmitLDC(0);
+ pCode->EmitCONV_U();
+ }
+ else
+ {
+ pCode->EmitLDNULL();
+ }
+
+ numArgs++;
+ }
+
+#ifndef TARGET_X86
+ if (msig.HasGenericContextArg())
+ {
+ pCode->EmitLDC(0);
+ numArgs++;
+ }
+
+ // Continuation
+ pCode->EmitLDARG(0);
+ numArgs++;
+#endif
+
+ msig.Reset();
+ CorElementType ty;
+ while ((ty = msig.NextArg()) != ELEMENT_TYPE_END)
+ {
+ TypeHandle tyHnd = msig.GetLastTypeHandleThrowing();
+ DWORD loc = pCode->NewLocal(LocalDesc(tyHnd));
+ pCode->EmitLDLOCA(loc);
+ pCode->EmitINITOBJ(pCode->GetToken(tyHnd));
+ pCode->EmitLDLOC(loc);
+ numArgs++;
+ }
+
+#ifdef TARGET_X86
+ if (msig.HasGenericContextArg())
+ {
+ pCode->EmitLDC(0);
+ numArgs++;
+ }
+
+ // Continuation
+ pCode->EmitLDARG(0);
+ numArgs++;
+#endif
+
+ // Resumption stubs are uniquely coupled to the code version (since the
+ // continuation is), so we need to make sure we always keep calling the
+ // same version here.
+ PrepareCodeConfig* config = GetThread()->GetCurrentPrepareCodeConfig();
+ NativeCodeVersion ncv = config->GetCodeVersion();
+ if (ncv.GetOptimizationTier() == NativeCodeVersion::OptimizationTier1OSR)
+ {
+#ifdef FEATURE_ON_STACK_REPLACEMENT
+ // The OSR version needs to resume in the tier0 version. The tier0
+ // version will handle setting up the frame that the OSR version
+ // expects and then delegating back into the OSR version (knowing to do
+ // so through information stored in the continuation).
+ _ASSERTE(m_pPatchpointInfoFromRuntime != NULL);
+ pCode->EmitLDC((DWORD_PTR)m_pPatchpointInfoFromRuntime->GetTier0EntryPoint());
+#else
+ _ASSERTE(!"Unexpected optimization tier with OSR disabled");
+#endif
+ }
+ else
+ {
+ {
+ AllocMemTracker pamTracker;
+ m_finalCodeAddressSlot = (PCODE*)pamTracker.Track(m_pMethodBeingCompiled->GetLoaderAllocator()->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(sizeof(PCODE))));
+ pamTracker.SuppressRelease();
+ }
+
+ pCode->EmitLDC((DWORD_PTR)m_finalCodeAddressSlot);
+ pCode->EmitLDIND_I();
+ }
+
+ pCode->EmitCALLI(pCode->GetSigToken(calliSig.GetRawSig(), calliSig.GetRawSigLen()), numArgs, msig.IsReturnTypeVoid() ? 0 : 1);
+
+ DWORD resultLoc = UINT_MAX;
+ TypeHandle resultTypeHnd;
+ if (!msig.IsReturnTypeVoid())
+ {
+ resultTypeHnd = msig.GetRetTypeHandleThrowing();
+ resultLoc = pCode->NewLocal(LocalDesc(resultTypeHnd));
+ pCode->EmitSTLOC(resultLoc);
+ }
+
+ TypeHandle continuationTypeHnd = CoreLibBinder::GetClass(CLASS__CONTINUATION);
+ DWORD newContinuationLoc = pCode->NewLocal(LocalDesc(continuationTypeHnd));
+ pCode->EmitCALL(METHOD__STUBHELPERS__ASYNC2_CALL_CONTINUATION, 0, 1);
+ pCode->EmitSTLOC(newContinuationLoc);
+
+ if (!msig.IsReturnTypeVoid())
+ {
+ ILCodeLabel* doneResult = pCode->NewCodeLabel();
+ pCode->EmitLDLOC(newContinuationLoc);
+ pCode->EmitBRTRUE(doneResult);
+
+ // Load 'next' of current continuation
+ pCode->EmitLDARG(0);
+ pCode->EmitLDFLD(FIELD__CONTINUATION__NEXT);
+
+ // Result is placed in GCData[0] if it has GC references (potentially boxing it).
+ bool isOrContainsGCPointers = false;
+ if (CorTypeInfo::IsObjRef(resultTypeHnd.GetInternalCorElementType()) || (resultTypeHnd.IsValueType() && resultTypeHnd.AsMethodTable()->ContainsGCPointers()))
+ {
+ // Load 'gcdata' of next continuation
+ pCode->EmitLDFLD(FIELD__CONTINUATION__GCDATA);
+
+ // Now we have the GC array. At the first index is the result.
+ pCode->EmitLDC(0);
+
+ // NOTE: that we are not using regular boxing (in EmitBOX sense) and allocate our own box instances via a helper.
+ // There are two reasons:
+ // - resultTypeHnd may be a nullable type and have different layout in boxed/unboxed forms.
+ // We do not want to deal with that.
+ // - resultTypeHnd may contain __Canon fields. Regular boxing would not allow that, but this box is used for a very
+ // specific internal purpose where we only require that the GC layout of the box matches the data
+ // that we store in it, thus we want to allow __Canon.
+ if (resultTypeHnd.IsValueType())
+ {
+ // make a box and dup the ref
+ MethodDesc* md = CoreLibBinder::GetMethod(METHOD__RUNTIME_HELPERS__ALLOC_CONTINUATION_RESULT_BOX);
+ pCode->EmitLDC((DWORD_PTR)resultTypeHnd.AsMethodTable());
+ pCode->EmitCALL(pCode->GetToken(md), 1, 1);
+ pCode->EmitDUP();
+ // dst is the offset of the first field in the box
+ pCode->EmitLDFLDA(FIELD__RAW_DATA__DATA);
+ // load the result
+ pCode->EmitLDLOC(resultLoc);
+ // store into the box
+ pCode->EmitSTOBJ(pCode->GetToken(resultTypeHnd));
+ }
+ else
+ {
+ // load the result
+ pCode->EmitLDLOC(resultLoc);
+ }
+
+ // Store the result.
+ pCode->EmitSTELEM_REF();
+ }
+ else
+ {
+ // Otherwise it goes into Data, either at offset 0 or 4 depending
+ // on CORINFO_CONTINUATION_OSR_IL_OFFSET_IN_DATA.
+ ILCodeLabel* hasOsrILOffset = pCode->NewCodeLabel();
+
+ unsigned nextContinuationLcl = pCode->NewLocal(LocalDesc(continuationTypeHnd));
+ pCode->EmitSTLOC(nextContinuationLcl);
+
+ // Load 'flags' of next continuation
+ pCode->EmitLDLOC(nextContinuationLcl);
+ pCode->EmitLDFLD(FIELD__CONTINUATION__FLAGS);
+ pCode->EmitLDC(CORINFO_CONTINUATION_OSR_IL_OFFSET_IN_DATA);
+ pCode->EmitAND();
+ pCode->EmitBRTRUE(hasOsrILOffset);
+
+ // Load 'data' of next continuation
+ pCode->EmitLDLOC(nextContinuationLcl);
+ pCode->EmitLDFLD(FIELD__CONTINUATION__DATA);
+ // Load address of array at index 0
+ pCode->EmitLDC(0);
+ pCode->EmitLDELEMA(pCode->GetToken(CoreLibBinder::GetClass(CLASS__BYTE)));
+
+ // Store at index 0.
+ pCode->EmitLDLOC(resultLoc);
+ pCode->EmitSTOBJ(pCode->GetToken(resultTypeHnd));
+
+ pCode->EmitBR(doneResult);
+
+ pCode->EmitLabel(hasOsrILOffset);
+
+ // Load 'data' of next continuation
+ pCode->EmitLDLOC(nextContinuationLcl);
+ pCode->EmitLDFLD(FIELD__CONTINUATION__DATA);
+ // Load address of array at index 4
+ pCode->EmitLDC(4);
+ pCode->EmitLDELEMA(pCode->GetToken(CoreLibBinder::GetClass(CLASS__BYTE)));
+
+ // Store at index 4.
+ pCode->EmitLDLOC(resultLoc);
+ pCode->EmitUNALIGNED(1);
+ pCode->EmitSTOBJ(pCode->GetToken(resultTypeHnd));
+ }
+
+ pCode->EmitLabel(doneResult);
+ }
+
+ pCode->EmitLDLOC(newContinuationLoc);
+ pCode->EmitRET();
+
+ MethodDesc* result =
+ ILStubCache::CreateAndLinkNewILStubMethodDesc(
+ md->GetLoaderAllocator(),
+ md->GetLoaderModule()->GetILStubCache()->GetOrCreateStubMethodTable(md->GetLoaderModule()),
+ ILSTUB_ASYNC_RESUME,
+ md->GetModule(),
+ stubSig.GetRawSig(), stubSig.GetRawSigLen(),
+ &emptyCtx,
+ &sl);
+
+ const char* optimizationTierName = nullptr;
+ switch (ncv.GetOptimizationTier())
+ {
+ case NativeCodeVersion::OptimizationTier0: optimizationTierName = "Tier0"; break;
+ case NativeCodeVersion::OptimizationTier1: optimizationTierName = "Tier1"; break;
+ case NativeCodeVersion::OptimizationTier1OSR: optimizationTierName = "Tier1OSR"; break;
+ case NativeCodeVersion::OptimizationTierOptimized: optimizationTierName = "Optimized"; break;
+ case NativeCodeVersion::OptimizationTier0Instrumented: optimizationTierName = "Tier0Instrumented"; break;
+ case NativeCodeVersion::OptimizationTier1Instrumented: optimizationTierName = "Tier1Instrumented"; break;
+ default: optimizationTierName = "UnknownTier"; break;
+ }
+
+ char name[256];
+ int numWritten = sprintf_s(name, ARRAY_SIZE(name), "IL_STUB_AsyncResume_%s_%s", m_pMethodBeingCompiled->GetName(), optimizationTierName);
+ if (numWritten != -1)
+ {
+ AllocMemTracker pamTracker;
+ void* allocedMem = pamTracker.Track(m_pMethodBeingCompiled->GetLoaderAllocator()->GetLowFrequencyHeap()->AllocMem(S_SIZE_T(numWritten + 1)));
+ memcpy(allocedMem, name, (size_t)(numWritten + 1));
+ result->AsDynamicMethodDesc()->SetMethodName((LPCUTF8)allocedMem);
+ pamTracker.SuppressRelease();
+ }
+
+#ifdef _DEBUG
+ LOG((LF_STUBS, LL_INFO1000, "ASYNC: Resumption stub %s created\n", name));
+ sl.LogILStub(CORJIT_FLAGS());
+#endif
+
+ return CORINFO_METHOD_HANDLE(result);
+}
+
bool CEEInfo::convertPInvokeCalliToCall(CORINFO_RESOLVED_TOKEN * pResolvedToken, bool fMustConvert)
{
return false;
@@ -14669,6 +15103,12 @@ PatchpointInfo* CEEInfo::getOSRInfo(unsigned* ilOffset)
UNREACHABLE(); // only called on derived class.
}
+CORINFO_METHOD_HANDLE CEEInfo::getAsyncResumptionStub()
+{
+ LIMITED_METHOD_CONTRACT;
+ UNREACHABLE(); // only called on derived class.
+}
+
void* CEEInfo::getHelperFtn(CorInfoHelpFunc ftnNum, /* IN */
void ** ppIndirection) /* OUT */
{
diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h
index 705a1df5395afa..82cf79f4ce0a34 100644
--- a/src/coreclr/vm/jitinterface.h
+++ b/src/coreclr/vm/jitinterface.h
@@ -86,6 +86,7 @@ BOOL LoadDynamicInfoEntry(Module *currentModule,
// These must be implemented in assembly and generate a TransitionBlock then calling JIT_PatchpointWorkerWithPolicy in order to actually be used.
EXTERN_C FCDECL2(void, JIT_Patchpoint, int* counter, int ilOffset);
EXTERN_C FCDECL1(void, JIT_PartialCompilationPatchpoint, int ilOffset);
+EXTERN_C FCDECL1(void, JIT_ResumeOSR, int ilOffset);
//
// JIT HELPER ALIASING FOR PORTABILITY.
@@ -608,7 +609,7 @@ class CEECodeGenInfo : public CEEInfo
ULONG32 cMap, ICorDebugInfo::OffsetMapping *pMap) override final;
void setVars(CORINFO_METHOD_HANDLE ftn, ULONG32 cVars,
ICorDebugInfo::NativeVarInfo *vars) override final;
- void CompressDebugInfo();
+ void CompressDebugInfo(PCODE nativeEntry);
virtual void SetDebugInfo(PTR_BYTE pDebugInfo) = 0;
virtual PatchpointInfo* GetPatchpointInfo()
@@ -897,6 +898,7 @@ class CEEJitInfo final : public CEECodeGenInfo
m_pPatchpointInfoFromRuntime(NULL),
m_ilOffset(0)
#endif
+ , m_finalCodeAddressSlot(NULL)
{
CONTRACTL
{
@@ -947,6 +949,9 @@ class CEEJitInfo final : public CEECodeGenInfo
void setPatchpointInfo(PatchpointInfo* patchpointInfo) override;
PatchpointInfo* getOSRInfo(unsigned* ilOffset) override;
+ void PublishFinalCodeAddress(PCODE addr);
+ virtual CORINFO_METHOD_HANDLE getAsyncResumptionStub() override final;
+
protected :
#ifdef FEATURE_PGO
@@ -991,6 +996,7 @@ protected :
PatchpointInfo * m_pPatchpointInfoFromRuntime;
unsigned m_ilOffset;
#endif
+ PCODE* m_finalCodeAddressSlot;
};
diff --git a/src/coreclr/vm/loongarch64/asmhelpers.S b/src/coreclr/vm/loongarch64/asmhelpers.S
index 6f2a89383852c4..0ed172699ee902 100644
--- a/src/coreclr/vm/loongarch64/asmhelpers.S
+++ b/src/coreclr/vm/loongarch64/asmhelpers.S
@@ -1101,6 +1101,15 @@ LEAF_ENTRY JIT_PartialCompilationPatchpoint, _TEXT
b C_FUNC(JIT_Patchpoint)
LEAF_END JIT_PartialCompilationPatchpoint, _TEXT
+NESTED_ENTRY JIT_ResumeOSR, _TEXT, NoHandler
+ PROLOG_WITH_TRANSITION_BLOCK
+
+ addi.d $a0, $sp, __PWTB_TransitionBlock // TransitionBlock *
+ bl C_FUNC(JIT_ResumeOSR)
+
+ EPILOG_WITH_TRANSITION_BLOCK_RETURN
+NESTED_END JIT_ResumeOSR, _TEXT
+
#endif // FEATURE_TIERED_COMPILATION
// ------------------------------------------------------------------
diff --git a/src/coreclr/vm/memberload.cpp b/src/coreclr/vm/memberload.cpp
index 472847f041790b..91eb170612255c 100644
--- a/src/coreclr/vm/memberload.cpp
+++ b/src/coreclr/vm/memberload.cpp
@@ -781,6 +781,7 @@ MemberLoader::GetMethodDescFromMemberDefOrRefOrSpec(
allowInstParam,
/* forceRemotableMethod */ FALSE,
/* allowCreate */ TRUE,
+ AsyncVariantLookup::MatchingAsyncVariant,
/* level */ owningTypeLoadLevel);
} // MemberLoader::GetMethodDescFromMemberDefOrRefOrSpec
diff --git a/src/coreclr/vm/metasig.h b/src/coreclr/vm/metasig.h
index 2626a3237bed57..37938c35a9cfea 100644
--- a/src/coreclr/vm/metasig.h
+++ b/src/coreclr/vm/metasig.h
@@ -74,7 +74,7 @@
#endif
#define SM(varname, args, retval) METASIG_BODY( SM_ ## varname, args retval )
#define IM(varname, args, retval) METASIG_BODY( IM_ ## varname, args retval )
-#define GM(varname, n, conv, args, retval) METASIG_BODY( GM_ ## varname, args retval )
+#define GM(varname, conv, n, args, retval) METASIG_BODY( GM_ ## varname, args retval )
#define Fld(varname, val) METASIG_BODY( Fld_ ## varname, val )
#endif
@@ -564,6 +564,28 @@ DEFINE_METASIG_T(IM(Dec_RetVoid, g(DECIMAL), v))
DEFINE_METASIG_T(IM(Currency_RetVoid, g(CURRENCY), v))
DEFINE_METASIG_T(SM(RefDec_RetVoid, r(g(DECIMAL)), v))
+DEFINE_METASIG_T(IM(Exception_RetTaskOfT, C(EXCEPTION), GI(C(TASK_1), 1, G(0))))
+DEFINE_METASIG_T(IM(T_RetTaskOfT, G(0), GI(C(TASK_1), 1, G(0))))
+
+DEFINE_METASIG_T(IM(Exception_RetTask, C(EXCEPTION), C(TASK)))
+DEFINE_METASIG_T(IM(RetTask, _, C(TASK)))
+
+DEFINE_METASIG_T(IM(Exception_RetValueTaskOfT, C(EXCEPTION), GI(g(VALUETASK_1), 1, G(0))))
+DEFINE_METASIG_T(IM(T_RetValueTaskOfT, G(0), GI(g(VALUETASK_1), 1, G(0))))
+
+DEFINE_METASIG_T(IM(Exception_RetValueTask, C(EXCEPTION), g(VALUETASK)))
+DEFINE_METASIG_T(IM(RetValueTask, _, g(VALUETASK)))
+
+DEFINE_METASIG_T(GM(Exception_RetTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, C(EXCEPTION), GI(C(TASK_1), 1, M(0))))
+DEFINE_METASIG_T(GM(T_RetTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, M(0), GI(C(TASK_1), 1, M(0))))
+DEFINE_METASIG_T(GM(Exception_RetValueTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, C(EXCEPTION), GI(g(VALUETASK_1), 1, M(0))))
+DEFINE_METASIG_T(GM(T_RetValueTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, M(0), GI(g(VALUETASK_1), 1, M(0))))
+
+DEFINE_METASIG_T(SM(RetTask, , C(TASK)))
+DEFINE_METASIG_T(SM(RetValueTask, , g(VALUETASK)))
+DEFINE_METASIG_T(SM(Exception_RetTask, C(EXCEPTION), C(TASK)))
+DEFINE_METASIG_T(SM(Exception_RetValueTask, C(EXCEPTION), g(VALUETASK)))
+
DEFINE_METASIG(GM(RefT_T_T_RetT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, r(M(0)) M(0) M(0), M(0)))
DEFINE_METASIG(SM(RefObject_Object_Object_RetObject, r(j) j j, j))
DEFINE_METASIG(SM(RefByte_Byte_Byte_RetByte, r(b) b b, b))
@@ -606,6 +628,13 @@ DEFINE_METASIG(SM(PtrByte_RetStr, P(b), s))
DEFINE_METASIG(SM(Str_RetPtrByte, s, P(b)))
DEFINE_METASIG(SM(PtrByte_RetVoid, P(b), v))
+DEFINE_METASIG_T(SM(RetContinuation, , C(CONTINUATION)))
+DEFINE_METASIG(GM(T_RetVoid, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, M(0), v))
+DEFINE_METASIG_T(SM(Continuation_RetTask, C(CONTINUATION), C(TASK)))
+DEFINE_METASIG_T(GM(Continuation_RetTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, C(CONTINUATION), GI(C(TASK_1), 1, M(0))))
+DEFINE_METASIG_T(SM(Continuation_RetValueTask, C(CONTINUATION), g(VALUETASK)))
+DEFINE_METASIG_T(GM(Continuation_RetValueTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, C(CONTINUATION), GI(g(VALUETASK_1), 1, M(0))))
+
// Undefine macros in case we include the file again in the compilation unit
#undef DEFINE_METASIG
diff --git a/src/coreclr/vm/method.cpp b/src/coreclr/vm/method.cpp
index bafea6ab4e7d75..7d623403581e3d 100644
--- a/src/coreclr/vm/method.cpp
+++ b/src/coreclr/vm/method.cpp
@@ -82,6 +82,7 @@ const BYTE MethodDesc::s_ClassificationSizeTable[] = {
// This extended part of the table is used for faster MethodDesc size lookup.
// We index using optional slot flags into it
METHOD_DESC_SIZES(sizeof(NonVtableSlot)),
+
METHOD_DESC_SIZES(sizeof(MethodImpl)),
METHOD_DESC_SIZES(sizeof(NonVtableSlot) + sizeof(MethodImpl)),
@@ -89,6 +90,15 @@ const BYTE MethodDesc::s_ClassificationSizeTable[] = {
METHOD_DESC_SIZES(sizeof(NonVtableSlot) + sizeof(NativeCodeSlot)),
METHOD_DESC_SIZES(sizeof(MethodImpl) + sizeof(NativeCodeSlot)),
METHOD_DESC_SIZES(sizeof(NonVtableSlot) + sizeof(MethodImpl) + sizeof(NativeCodeSlot)),
+
+ METHOD_DESC_SIZES(sizeof(AsyncMethodData)),
+ METHOD_DESC_SIZES(sizeof(NonVtableSlot) + sizeof(AsyncMethodData)),
+ METHOD_DESC_SIZES(sizeof(MethodImpl) + sizeof(AsyncMethodData)),
+ METHOD_DESC_SIZES(sizeof(NonVtableSlot) + sizeof(MethodImpl) + sizeof(AsyncMethodData)),
+ METHOD_DESC_SIZES(sizeof(NativeCodeSlot) + sizeof(AsyncMethodData)),
+ METHOD_DESC_SIZES(sizeof(NonVtableSlot) + sizeof(NativeCodeSlot) + sizeof(AsyncMethodData)),
+ METHOD_DESC_SIZES(sizeof(MethodImpl) + sizeof(NativeCodeSlot) + sizeof(AsyncMethodData)),
+ METHOD_DESC_SIZES(sizeof(NonVtableSlot) + sizeof(MethodImpl) + sizeof(NativeCodeSlot) + sizeof(AsyncMethodData)),
};
#ifndef FEATURE_COMINTEROP
@@ -123,7 +133,8 @@ SIZE_T MethodDesc::SizeOf()
(mdfClassification
| mdfHasNonVtableSlot
| mdfMethodImpl
- | mdfHasNativeCodeSlot)];
+ | mdfHasNativeCodeSlot
+ | mdfHasAsyncMethodData)];
return size;
}
@@ -422,6 +433,13 @@ void MethodDesc::GetSig(PCCOR_SIGNATURE *ppSig, DWORD *pcSig)
return;
}
}
+ if (IsAsync2VariantMethod())
+ {
+ Signature sig = GetAddrOfAsyncMethodData()->sig;
+ *ppSig = sig.GetRawSig();
+ *pcSig = sig.GetRawSigLen();
+ return;
+ }
GetSigFromMetadata(GetMDImport(), ppSig, pcSig);
PREFIX_ASSUME(*ppSig != NULL);
@@ -450,6 +468,7 @@ void MethodDesc::GetSigFromMetadata(IMDInternalImport * importer,
}
CONTRACTL_END
+ _ASSERTE(!IsAsync2VariantMethod());
if (FAILED(importer->GetSigOfMethodDef(GetMemberDef(), pcSig, ppSig)))
{ // Class loader already asked for signature, so this should always succeed (unless there's a
// bug or a new code path)
@@ -722,7 +741,7 @@ BOOL MethodDesc::HasSameMethodDefAs(MethodDesc * pMD)
if (this == pMD)
return TRUE;
- return (GetMemberDef() == pMD->GetMemberDef()) && (GetModule() == pMD->GetModule());
+ return (GetMemberDef() == pMD->GetMemberDef()) && (GetModule() == pMD->GetModule() && pMD->IsAsync2VariantMethod() == IsAsync2VariantMethod());
}
//*******************************************************************************
@@ -1078,6 +1097,18 @@ PTR_PCODE MethodDesc::GetAddrOfNativeCodeSlot()
return (PTR_PCODE)(dac_cast(this) + size);
}
+//*******************************************************************************
+PTR_AsyncMethodData MethodDesc::GetAddrOfAsyncMethodData() const
+{
+ WRAPPER_NO_CONTRACT;
+
+ _ASSERTE(HasAsyncMethodData());
+
+ SIZE_T size = s_ClassificationSizeTable[m_wFlags & (mdfClassification | mdfHasNonVtableSlot | mdfMethodImpl | mdfHasNativeCodeSlot)];
+
+ return dac_cast(dac_cast(this) + size);
+}
+
//*******************************************************************************
BOOL MethodDesc::IsVoid()
{
@@ -1171,6 +1202,9 @@ COR_ILMETHOD* MethodDesc::GetILHeader()
}
CONTRACTL_END
+ if (IsRuntimeSupplied())
+ return NULL;
+
Module *pModule = GetModule();
// Always pickup overrides like reflection emit, EnC, etc.
@@ -1809,7 +1843,7 @@ MethodDesc* MethodDesc::StripMethodInstantiation()
//*******************************************************************************
MethodDescChunk *MethodDescChunk::CreateChunk(LoaderHeap *pHeap, DWORD methodDescCount,
- DWORD classification, BOOL fNonVtableSlot, BOOL fNativeCodeSlot, MethodTable *pInitialMT, AllocMemTracker *pamTracker, Module *pLoaderModule)
+ DWORD classification, BOOL fNonVtableSlot, BOOL fNativeCodeSlot, BOOL fAsyncMethodData, MethodTable *pInitialMT, AllocMemTracker *pamTracker, Module *pLoaderModule)
{
CONTRACT(MethodDescChunk *)
{
@@ -1833,6 +1867,9 @@ MethodDescChunk *MethodDescChunk::CreateChunk(LoaderHeap *pHeap, DWORD methodDes
if (fNativeCodeSlot)
oneSize += sizeof(MethodDesc::NativeCodeSlot);
+ if (fAsyncMethodData)
+ oneSize += sizeof(AsyncMethodData);
+
_ASSERTE((oneSize & MethodDesc::ALIGNMENT_MASK) == 0);
DWORD maxMethodDescsPerChunk = (DWORD)(MethodDescChunk::MaxSizeOfMethodDescs / oneSize);
@@ -1875,6 +1912,8 @@ MethodDescChunk *MethodDescChunk::CreateChunk(LoaderHeap *pHeap, DWORD methodDes
pMD->SetHasNonVtableSlot();
if (fNativeCodeSlot)
pMD->SetHasNativeCodeSlot();
+ if (fAsyncMethodData)
+ pMD->SetHasAsyncMethodData();
_ASSERTE(pMD->SizeOf() == oneSize);
@@ -2845,7 +2884,11 @@ bool MethodDesc::DetermineAndSetIsEligibleForTieredCompilation()
!IsWrapperStub() &&
// Functions with NoOptimization or AggressiveOptimization don't participate in tiering
- !IsJitOptimizationLevelRequested())
+ !IsJitOptimizationLevelRequested() &&
+
+ // Tiering the async thunk methods doesn't make sense
+ !IsAsyncThunkMethod()
+ )
{
InterlockedUpdateFlags3(enum_flag3_IsEligibleForTieredCompilation, TRUE);
return true;
diff --git a/src/coreclr/vm/method.hpp b/src/coreclr/vm/method.hpp
index 73db7083022bd9..d02dda8b256ae4 100644
--- a/src/coreclr/vm/method.hpp
+++ b/src/coreclr/vm/method.hpp
@@ -50,11 +50,72 @@ GVAL_DECL(TADDR, g_MiniMetaDataBuffAddress);
EXTERN_C VOID STDCALL NDirectImportThunk();
-#define METHOD_TOKEN_REMAINDER_BIT_COUNT 12
+#define METHOD_TOKEN_REMAINDER_BIT_COUNT 11
#define METHOD_TOKEN_REMAINDER_MASK ((1 << METHOD_TOKEN_REMAINDER_BIT_COUNT) - 1)
#define METHOD_TOKEN_RANGE_BIT_COUNT (24 - METHOD_TOKEN_REMAINDER_BIT_COUNT)
#define METHOD_TOKEN_RANGE_MASK ((1 << METHOD_TOKEN_RANGE_BIT_COUNT) - 1)
+enum class AsyncMethodKind
+{
+ // Regular methods not returning tasks
+ // These are "normal" methods that do not get other variants.
+ // NOTE: Generic T-returning methods are NotAsync, even if T could be a Task.
+ NotAsync,
+
+ // Regular methods that return Task/ValueTask
+ // These methods have a synthetic variant that is an Async2-callable helper.
+ TaskReturning,
+
+ // Task-returning methods marked as MethodImpl::Async in metadata.
+ // These methods have bodies that are actually thunks to Async2 implementation variants (Async2VariantImpl)
+ RuntimeAsync,
+
+ //=============================================================
+ // On {TaskReturning, Async2VariantThunk} and {RuntimeAsync, Async2VariantImpl} pairs:
+ //
+ // When we see a Task-returning method we create 2 method varaints that logically match the same method definition.
+ // One variant has the same signature/callconv as the defining method and another is a matching Async2 variant.
+ // Depending on whether the definition was a runtime async method or an ordinary Task-returning method,
+ // one variant is the actual implementation and another is a thunk.
+ //
+ // The signature of the Async2 variant is formed from the original signature by replacing Task return type with
+ // modreq'd element type:
+ // Example: "Task Foo();" ===> "modreq(Task`) int Foo();"
+ // Example: "ValueTask Bar();" ===> "modreq(ValueTask) void Bar();"
+ //
+ // It is possible to get from one variant to another unambiguously via GetAsyncOtherVariant.
+ //
+ // Async2 methods are called with CORINFO_CALLCONV_ASYNCCALL call convention.
+ //
+ // NOTE: not all Async2 methods are "variants" from a pair, see Async2ExplicitImpl below.
+ //=============================================================
+
+ // The following methods use special calling convention (CORINFO_CALLCONV_ASYNCCALL)
+ // These methods are emitted by the JIT as resumable state machines and also take an extra
+ // parameter and extra return - the continuation object.
+
+ // Async2 methods with actual IL implementation of a MethodImpl::Async method.
+ Async2VariantImpl,
+
+ // Async2 methods with synthetic bodies that forward to a TaskReturning method.
+ Async2VariantThunk,
+
+ // Methods that are explicitly declared as Async2 in metadata while not Task returning.
+ // This is a special case used in a few infrastructure methods like `Await`.
+ // Such methods do not get non-Async2 variants/thunks and can only be called from another Async2 method.
+ // NOTE: These methods have the original signature and it is not possible to tell if the method is Async2
+ // from the signature alone, thus all these methods are also JIT intrinsics.
+ Async2ExplicitImpl,
+};
+
+struct AsyncMethodData
+{
+ AsyncMethodKind kind;
+ Signature sig;
+};
+
+typedef DPTR(struct AsyncMethodData) PTR_AsyncMethodData;
+
//=============================================================
// Splits methoddef token into two pieces for
// storage inside a methoddesc.
@@ -128,8 +189,8 @@ enum MethodDescFlags
// Has slot for native code
mdfHasNativeCodeSlot = 0x0020,
- // Method was added via Edit And Continue
- mdfEnCAddedMethod = 0x0040,
+ // HasAsyncMethodData
+ mdfHasAsyncMethodData = 0x0040,
// Method is static
mdfStatic = 0x0080,
@@ -166,6 +227,36 @@ struct MethodDescCodeData final
};
using PTR_MethodDescCodeData = DPTR(MethodDescCodeData);
+enum class AsyncVariantLookup
+{
+ MatchingAsyncVariant = 0,
+ AsyncOtherVariant
+};
+
+enum class AsyncMethodSignatureKind
+{
+ TaskReturningMethod,
+ TaskNonGenericReturningMethod,
+ Async2Method,
+ Async2MethodNonGeneric,
+ NormalMethod
+};
+
+inline bool IsAsyncSigNormal(AsyncMethodSignatureKind input)
+{
+ return input == AsyncMethodSignatureKind::NormalMethod;
+}
+
+inline bool IsAsyncSigAsync2(AsyncMethodSignatureKind input)
+{
+ return (input == AsyncMethodSignatureKind::Async2Method) || (input == AsyncMethodSignatureKind::Async2MethodNonGeneric);
+}
+
+inline bool IsAsyncSigTaskReturning(AsyncMethodSignatureKind input)
+{
+ return (input == AsyncMethodSignatureKind::TaskReturningMethod) || (input == AsyncMethodSignatureKind::TaskNonGenericReturningMethod);
+}
+
// The size of this structure needs to be a multiple of MethodDesc::ALIGNMENT
//
// @GENERICS:
@@ -602,7 +693,8 @@ class MethodDesc
{
LIMITED_METHOD_DAC_CONTRACT;
return mcFCall == GetClassification()
- || mcArray == GetClassification();
+ || mcArray == GetClassification()
+ || IsAsyncThunkMethod();
}
inline DWORD IsArray() const
@@ -803,7 +895,7 @@ class MethodDesc
MODE_ANY;
}
CONTRACTL_END;
- return IsIL() && !IsUnboxingStub() && GetRVA();
+ return IsIL() && !IsUnboxingStub() && GetRVA() && !IsRuntimeSupplied();
}
COR_ILMETHOD* GetILHeader();
@@ -1399,6 +1491,10 @@ class MethodDesc
BOOL SetNativeCodeInterlocked(PCODE addr, PCODE pExpected = 0);
PTR_PCODE GetAddrOfNativeCodeSlot();
+ PTR_AsyncMethodData GetAddrOfAsyncMethodData() const;
+#ifndef DACCESS_COMPILE
+ const AsyncMethodData& GetAsyncMethodData() { _ASSERTE(HasAsyncMethodData()); return *GetAddrOfAsyncMethodData(); }
+#endif
BOOL MayHaveNativeCode();
@@ -1545,6 +1641,7 @@ class MethodDesc
BOOL allowInstParam,
BOOL forceRemotableMethod = FALSE,
BOOL allowCreate = TRUE,
+ AsyncVariantLookup variantLookup = AsyncVariantLookup::MatchingAsyncVariant,
ClassLoadLevel level = CLASS_LOADED);
// Normalize methoddesc for reflection
@@ -1552,6 +1649,11 @@ class MethodDesc
TypeHandle instType,
Instantiation methodInst);
+ MethodDesc* GetAsyncOtherVariant(BOOL allowInstParam = TRUE)
+ {
+ return FindOrCreateAssociatedMethodDesc(this, GetMethodTable(), FALSE, GetMethodInstantiation(), allowInstParam, FALSE, TRUE, AsyncVariantLookup::AsyncOtherVariant);
+ }
+
// True if a MD is an funny BoxedEntryPointStub (not from the method table) or
// an MD for a generic instantiation...In other words the MethodDescs and the
// MethodTable are guaranteed to be "tightly-knit", i.e. if one is present in
@@ -1662,8 +1764,11 @@ class MethodDesc
enum {
// There are flags available for use here (currently 4 flags bits are available); however, new bits are hard to come by, so any new flags bits should
// have a fairly strong justification for existence.
- enum_flag3_TokenRemainderMask = 0x0FFF, // This must equal METHOD_TOKEN_REMAINDER_MASK calculated higher in this file.
+ enum_flag3_TokenRemainderMask = 0x07FF, // This must equal METHOD_TOKEN_REMAINDER_MASK calculated higher in this file.
// for this method.
+ // Method was added via Edit And Continue
+ enum_flag3_EnCAddedMethod = 0x0800,
+
// enum_flag3_HasPrecode implies that enum_flag3_HasStableEntryPoint is set.
enum_flag3_HasStableEntryPoint = 0x1000, // The method entrypoint is stable (either precode or actual code)
enum_flag3_HasPrecode = 0x2000, // Precode has been allocated for this method
@@ -1744,17 +1849,91 @@ class MethodDesc
m_wFlags |= mdfHasNativeCodeSlot;
}
+ // Historically we use "Async2" to mean methods that can be called via CORINFO_CALLCONV_ASYNCCALL
+ // CONSIDER: We could have a better name for the concept, but it is hard to beat shortness of "Async2"
+ inline bool IsAsync2Method() const
+ {
+ LIMITED_METHOD_DAC_CONTRACT;
+ if (!HasAsyncMethodData())
+ return false;
+ auto asyncKind = GetAddrOfAsyncMethodData()->kind;
+ return asyncKind == AsyncMethodKind::Async2VariantThunk ||
+ asyncKind == AsyncMethodKind::Async2VariantImpl ||
+ asyncKind == AsyncMethodKind::Async2ExplicitImpl;
+ }
+
+ // Is this an Async2-callable variant method?
+ // If yes, the method has another non-Async2 variant.
+ inline bool IsAsync2VariantMethod() const
+ {
+ LIMITED_METHOD_DAC_CONTRACT;
+ if (!HasAsyncMethodData())
+ return false;
+ auto asyncKind = GetAddrOfAsyncMethodData()->kind;
+ return asyncKind == AsyncMethodKind::Async2VariantThunk ||
+ asyncKind == AsyncMethodKind::Async2VariantImpl;
+ }
+
+ // Is this a small(ish) synthetic Task/async2 adapter to an async2/Task implementation?
+ // If yes, the method has another variant, which has the actual user-defined method body.
+ // Depending on whether user defined method was runtime async or not, the corresponding thunk
+ // will be an ordinary or async2 variant.
+ inline bool IsAsyncThunkMethod() const
+ {
+ LIMITED_METHOD_DAC_CONTRACT;
+ if (!HasAsyncMethodData())
+ return false;
+
+ auto asyncType = GetAddrOfAsyncMethodData()->kind;
+ return asyncType == AsyncMethodKind::Async2VariantThunk ||
+ asyncType == AsyncMethodKind::RuntimeAsync;
+ }
+
+ inline bool IsTaskReturningMethod() const
+ {
+ LIMITED_METHOD_DAC_CONTRACT;
+ if (!HasAsyncMethodData())
+ return false;
+ auto asyncKind = GetAddrOfAsyncMethodData()->kind;
+ return asyncKind == AsyncMethodKind::RuntimeAsync ||
+ asyncKind == AsyncMethodKind::TaskReturning;
+ }
+
+ inline bool IsStructMethodOperatingOnCopy()
+ {
+ if (!GetMethodTable()->IsValueType() || IsStatic())
+ return false;
+
+ if (!HasAsyncMethodData())
+ return false;
+
+ // Only async2 methods backed by actual user code operate on copies.
+ // Thunks with runtime-supplied implementation do not.
+ return GetAddrOfAsyncMethodData()->kind == AsyncMethodKind::Async2VariantImpl;
+ }
+
+ inline bool HasAsyncMethodData() const
+ {
+ return (m_wFlags & mdfHasAsyncMethodData) != 0;
+ }
+
+ inline void SetHasAsyncMethodData()
+ {
+ LIMITED_METHOD_CONTRACT;
+ m_wFlags |= mdfHasAsyncMethodData;
+ }
+
#ifdef FEATURE_METADATA_UPDATER
inline BOOL IsEnCAddedMethod()
{
LIMITED_METHOD_DAC_CONTRACT;
- return (m_wFlags & mdfEnCAddedMethod) != 0;
+ return (m_wFlags3AndTokenRemainder & enum_flag3_EnCAddedMethod) != 0;
}
inline void SetIsEnCAddedMethod()
{
LIMITED_METHOD_CONTRACT;
- m_wFlags |= mdfEnCAddedMethod;
+ m_wFlags3AndTokenRemainder |= enum_flag3_EnCAddedMethod;
}
#else
inline BOOL IsEnCAddedMethod()
@@ -1885,8 +2064,17 @@ class MethodDesc
PCODE JitCompileCodeLockedEventWrapper(PrepareCodeConfig* pConfig, JitListLockEntry* pEntry);
PCODE JitCompileCodeLocked(PrepareCodeConfig* pConfig, COR_ILMETHOD_DECODER* pilHeader, JitListLockEntry* pLockEntry, ULONG* pSizeOfCode);
-public:
+ bool TryGenerateAsyncThunk(DynamicResolver** resolver, COR_ILMETHOD_DECODER** methodILDecoder);
bool TryGenerateUnsafeAccessor(DynamicResolver** resolver, COR_ILMETHOD_DECODER** methodILDecoder);
+ void EmitJitStateMachineBasedRuntimeAsyncThunk(MethodDesc* pAsyncOtherVariant, MetaSig& thunkMsig, ILStubLinker* pSL);
+ void EmitAsync2MethodThunk(MethodDesc* pAsyncOtherVariant, MetaSig& msig, ILStubLinker* pSL);
+ SigPointer GetAsync2ThunkResultTypeSig();
+ int GetTokenForGenericMethodCallWithAsyncReturnType(ILCodeStream* pCode, MethodDesc* md);
+ int GetTokenForGenericTypeMethodCallWithAsyncReturnType(ILCodeStream* pCode, MethodDesc* md);
+ int GetTokenForAwaitAwaiterInstantiatedOverTaskAwaiterType(ILCodeStream* pCode, TypeHandle taskAwaiterType);
+public:
+ static void CreateDerivedTargetSigWithExtraParams(MetaSig& msig, SigBuilder* stubSigBuilder);
+ bool TryGenerateTransientILImplementation(DynamicResolver** resolver, COR_ILMETHOD_DECODER** methodILDecoder);
#endif // DACCESS_COMPILE
#ifdef HAVE_GCCOVER
@@ -2217,7 +2405,7 @@ class MethodDescChunk
friend class MethodDesc;
enum {
- enum_flag_TokenRangeMask = 0x0FFF, // This must equal METHOD_TOKEN_RANGE_MASK calculated higher in this file
+ enum_flag_TokenRangeMask = 0x1FFF, // This must equal METHOD_TOKEN_RANGE_MASK calculated higher in this file
// These are separate to allow the flags space available and used to be obvious here
// and for the logic that splits the token to be algorithmically generated based on the
// #define
@@ -2239,6 +2427,7 @@ class MethodDescChunk
DWORD classification,
BOOL fNonVtableSlot,
BOOL fNativeCodeSlot,
+ BOOL fAsyncMethodData,
MethodTable *initialMT,
class AllocMemTracker *pamTracker,
Module* pLoaderModule = NULL);
@@ -2522,6 +2711,8 @@ class DynamicMethodDesc : public StoredSigMethodDesc
StubDelegateInvokeMethod,
+ StubAsyncResume,
+
StubLast
};
@@ -2592,6 +2783,12 @@ class DynamicMethodDesc : public StoredSigMethodDesc
return m_pszMethodName;
}
+ void SetMethodName(PTR_CUTF8 name)
+ {
+ LIMITED_METHOD_DAC_CONTRACT;
+ m_pszMethodName = name;
+ }
+
// Based on the current flags, compute the equivalent as COR metadata.
WORD GetAttrs() const
{
@@ -3457,7 +3654,8 @@ class InstantiatedMethodDesc final : public MethodDesc
static InstantiatedMethodDesc* FindLoadedInstantiatedMethodDesc(MethodTable *pMT,
mdMethodDef methodDef,
Instantiation methodInst,
- BOOL getSharedNotStub);
+ BOOL getSharedNotStub,
+ BOOL asyncThunk);
private:
@@ -3652,6 +3850,8 @@ ReadyToRunStandaloneMethodMetadata* GetReadyToRunStandaloneMethodMetadata(Method
void InitReadyToRunStandaloneMethodMetadata();
#endif // FEATURE_READYTORUN
+AsyncMethodSignatureKind ClassifyAsyncMethodSignature(SigPointer sig, Module* pModule, ULONG* offsetOfAsyncDetails, bool *pIsValueType);
+
#include "method.inl"
#endif // !_METHOD_H
diff --git a/src/coreclr/vm/methodtable.cpp b/src/coreclr/vm/methodtable.cpp
index 0b00bf1647b3fd..efb2ad647f2993 100644
--- a/src/coreclr/vm/methodtable.cpp
+++ b/src/coreclr/vm/methodtable.cpp
@@ -5534,6 +5534,7 @@ namespace
FALSE, // allowInstParam
TRUE, // forceRemoteableMethod
TRUE, // allowCreate
+ AsyncVariantLookup::MatchingAsyncVariant,
level // level
);
}
@@ -7777,7 +7778,8 @@ namespace
{
MethodDesc* pMD = it.GetMethodDesc();
if (pMD->GetMemberDef() == tkMethod
- && pMD->GetModule() == mod)
+ && pMD->GetModule() == mod
+ && pMD->IsAsync2VariantMethod() == pDefMD->IsAsync2VariantMethod())
{
return pMD;
}
@@ -7787,7 +7789,7 @@ namespace
}
}
-MethodDesc* MethodTable::GetParallelMethodDesc(MethodDesc* pDefMD)
+MethodDesc* MethodTable::GetParallelMethodDesc(MethodDesc* pDefMD, AsyncVariantLookup asyncVariantLookup)
{
CONTRACTL
{
@@ -7797,12 +7799,36 @@ MethodDesc* MethodTable::GetParallelMethodDesc(MethodDesc* pDefMD)
}
CONTRACTL_END;
+ if (asyncVariantLookup == AsyncVariantLookup::MatchingAsyncVariant)
+ {
#ifdef FEATURE_METADATA_UPDATER
- if (pDefMD->IsEnCAddedMethod())
- return GetParallelMethodDescForEnC(this, pDefMD);
+ if (pDefMD->IsEnCAddedMethod())
+ return GetParallelMethodDescForEnC(this, pDefMD);
#endif // FEATURE_METADATA_UPDATER
- return GetMethodDescForSlot_NoThrow(pDefMD->GetSlot()); // TODO! We should probably use the throwing variant where possible
+ return GetMethodDescForSlot_NoThrow(pDefMD->GetSlot()); // TODO! We should probably use the throwing variant where possible
+ }
+ else
+ {
+ // Slow path for finding the Async2 variant (or not-Async2 variant, if we start from Async2 one)
+ // This could be optimized with some trickery around slot numbers, but doing so is ... confusing, so I'm not implementing this yet
+ mdMethodDef tkMethod = pDefMD->GetMemberDef();
+ Module* mod = pDefMD->GetModule();
+ bool isAsync2VariantMethod = pDefMD->IsAsync2VariantMethod();
+
+ MethodTable::IntroducedMethodIterator it(this);
+ for (; it.IsValid(); it.Next())
+ {
+ MethodDesc* pMD = it.GetMethodDesc();
+ if (pMD->GetMemberDef() == tkMethod
+ && pMD->GetModule() == mod
+ && pMD->IsAsync2VariantMethod() != isAsync2VariantMethod)
+ {
+ return pMD;
+ }
+ }
+ return NULL;
+ }
}
#ifndef DACCESS_COMPILE
@@ -8179,9 +8205,23 @@ MethodTable::TryResolveVirtualStaticMethodOnThisType(MethodTable* pInterfaceType
{
COMPlusThrow(kTypeLoadException, E_FAIL);
}
+
+ bool differsByAsyncVariant = false;
if (!pMethodDecl->HasSameMethodDefAs(pInterfaceMD))
{
- continue;
+ if (pMethodDecl->GetMemberDef() == pInterfaceMD->GetMemberDef() &&
+ pMethodDecl->GetModule() == pInterfaceMD->GetModule() &&
+ pMethodDecl->IsAsync2VariantMethod() != pInterfaceMD->IsAsync2VariantMethod())
+ {
+ differsByAsyncVariant = true;
+ pMethodDecl = pMethodDecl->GetAsyncOtherVariant();
+ if (verifyImplemented)
+ return pMethodDecl;
+ }
+ else
+ {
+ continue;
+ }
}
// Spec requires that all body token for MethodImpls that refer to static virtual implementation methods must be MethodDef tokens.
@@ -8207,6 +8247,11 @@ MethodTable::TryResolveVirtualStaticMethodOnThisType(MethodTable* pInterfaceType
COMPlusThrow(kTypeLoadException, E_FAIL);
}
+ if (differsByAsyncVariant)
+ {
+ pMethodImpl = pMethodImpl->GetAsyncOtherVariant();
+ }
+
if (!verifyImplemented && instantiateMethodParameters)
{
pMethodImpl = pMethodImpl->FindOrCreateAssociatedMethodDesc(
@@ -8217,6 +8262,7 @@ MethodTable::TryResolveVirtualStaticMethodOnThisType(MethodTable* pInterfaceType
/* allowInstParam */ FALSE,
/* forceRemotableMethod */ FALSE,
/* allowCreate */ TRUE,
+ AsyncVariantLookup::MatchingAsyncVariant,
/* level */ level);
}
if (pMethodImpl != nullptr)
diff --git a/src/coreclr/vm/methodtable.h b/src/coreclr/vm/methodtable.h
index 19dddae79d2350..78009bcf95dd23 100644
--- a/src/coreclr/vm/methodtable.h
+++ b/src/coreclr/vm/methodtable.h
@@ -64,6 +64,7 @@ class ClassFactoryBase;
#endif // FEATURE_COMINTEROP_UNMANAGED_ACTIVATION
class ArgDestination;
enum class WellKnownAttribute : DWORD;
+enum class AsyncVariantLookup;
struct MethodTableAuxiliaryData;
typedef DPTR(MethodTableAuxiliaryData) PTR_MethodTableAuxiliaryData;
@@ -1753,7 +1754,7 @@ class MethodTable
MethodTable * GetRestoredSlotMT(DWORD slot);
// Used to map methods on the same slot between instantiations.
- MethodDesc * GetParallelMethodDesc(MethodDesc * pDefMD);
+ MethodDesc * GetParallelMethodDesc(MethodDesc * pDefMD, AsyncVariantLookup asyncVariantLookup = (AsyncVariantLookup)0);
//-------------------------------------------------------------------
// BoxedEntryPoint MethodDescs.
diff --git a/src/coreclr/vm/methodtablebuilder.cpp b/src/coreclr/vm/methodtablebuilder.cpp
index 16186cb5b08f1a..9635ca0ec1ef78 100644
--- a/src/coreclr/vm/methodtablebuilder.cpp
+++ b/src/coreclr/vm/methodtablebuilder.cpp
@@ -910,22 +910,26 @@ MethodTableBuilder::MethodSignature::GetMethodAttributes() const
STANDARD_VM_CONTRACT;
IMDInternalImport * pIMD = GetModule()->GetMDImport();
+ DWORD cSig;
+ PCCOR_SIGNATURE pSig;
if (TypeFromToken(GetToken()) == mdtMethodDef)
{
- DWORD cSig;
- if (FAILED(pIMD->GetNameAndSigOfMethodDef(GetToken(), &m_pSig, &cSig, &m_szName)))
+ if (FAILED(pIMD->GetNameAndSigOfMethodDef(GetToken(), &pSig, &cSig, &m_szName)))
{ // We have empty name or signature on error, do nothing
}
- m_cSig = static_cast(cSig);
}
else
{
CONSISTENCY_CHECK(TypeFromToken(m_tok) == mdtMemberRef);
- DWORD cSig;
- if (FAILED(pIMD->GetNameAndSigOfMemberRef(GetToken(), &m_pSig, &cSig, &m_szName)))
+ if (FAILED(pIMD->GetNameAndSigOfMemberRef(GetToken(), &pSig, &cSig, &m_szName)))
{ // We have empty name or signature on error, do nothing
}
+ }
+ // Don't overwrite signature that may have already been provided for AsyncThunk method
+ if (m_cSig == 0)
+ {
m_cSig = static_cast(cSig);
+ m_pSig = pSig;
}
}
@@ -988,9 +992,14 @@ MethodTableBuilder::bmtRTMethod::bmtRTMethod(
MethodDesc * pMD)
: m_pOwningType(pOwningType),
m_pMD(pMD),
- m_methodSig(pMD->GetModule(),
- pMD->GetMemberDef(),
- &pOwningType->GetSubstitution())
+ m_methodSig(pMD->IsAsync2VariantMethod()
+ ? MethodSignature(pMD->GetModule(),
+ pMD->GetMemberDef(),
+ pMD->GetSignature(),
+ &pOwningType->GetSubstitution())
+ : MethodSignature(pMD->GetModule(),
+ pMD->GetMemberDef(),
+ &pOwningType->GetSubstitution()))
{
CONTRACTL
{
@@ -1015,6 +1024,7 @@ MethodTableBuilder::bmtMDMethod::bmtMDMethod(
m_dwImplAttrs(dwImplAttrs),
m_dwRVA(dwRVA),
m_type(type),
+ m_asyncMethodKind(AsyncMethodKind::NotAsync),
m_implType(implType),
m_methodSig(pOwningType->GetModule(),
tok,
@@ -1032,6 +1042,41 @@ MethodTableBuilder::bmtMDMethod::bmtMDMethod(
}
CONTRACTL_END;
}
+
+ MethodTableBuilder::bmtMDMethod::bmtMDMethod(
+ bmtMDType * pOwningType,
+ mdMethodDef tok,
+ DWORD dwDeclAttrs,
+ DWORD dwImplAttrs,
+ DWORD dwRVA,
+ Signature sig,
+ AsyncMethodKind asyncMethodKind,
+ MethodClassification type,
+ METHOD_IMPL_TYPE implType)
+ : m_pOwningType(pOwningType),
+ m_dwDeclAttrs(dwDeclAttrs),
+ m_dwImplAttrs(dwImplAttrs),
+ m_dwRVA(dwRVA),
+ m_type(type),
+ m_asyncMethodKind(asyncMethodKind),
+ m_implType(implType),
+ m_methodSig(pOwningType->GetModule(),
+ tok,
+ sig,
+ &pOwningType->GetSubstitution()),
+ m_pMD(NULL),
+ m_pUnboxedMD(NULL),
+ m_slotIndex(INVALID_SLOT_INDEX),
+ m_unboxedSlotIndex(INVALID_SLOT_INDEX)
+ {
+ CONTRACTL
+ {
+ THROWS;
+ GC_TRIGGERS;
+ MODE_ANY;
+ }
+ CONTRACTL_END;
+ }
//*******************************************************************************
void
MethodTableBuilder::ImportParentMethods()
@@ -2645,6 +2690,224 @@ HRESULT MethodTableBuilder::FindMethodDeclarationForMethodImpl(
#pragma warning(push)
#pragma warning(disable:21000) // Suppress PREFast warning about overly large function
#endif // _PREFAST_
+
+void GetNameOfTypeDefOrRef(Module* pModule, mdToken tk, LPCSTR* pName, LPCSTR* pNamespace)
+{
+ *pName = "";
+ *pNamespace = "";
+ if (TypeFromToken(tk) == mdtTypeDef)
+ {
+ IfFailThrow(pModule->GetMDImport()->GetNameOfTypeDef(tk, pName, pNamespace));
+ }
+ else if (TypeFromToken(tk) == mdtTypeRef)
+ {
+ IfFailThrow(pModule->GetMDImport()->GetNameOfTypeRef(tk, pNamespace, pName));
+ }
+}
+
+bool IsTypeDefOrRefImplementedInSystemModule(Module* pModule, mdToken tk)
+{
+ if (TypeFromToken(tk) == mdtTypeDef)
+ {
+ if (pModule->IsSystem())
+ {
+ return true;
+ }
+ }
+ else if (TypeFromToken(tk) == mdtTypeRef)
+ {
+ mdToken tkTypeDef;
+ Module* pModuleOfTypeDef;
+
+ ClassLoader::ResolveTokenToTypeDefThrowing(pModule, tk, &pModuleOfTypeDef, &tkTypeDef);
+ if (pModuleOfTypeDef->IsSystem())
+ {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+bool IsTypeDefOrRefAByRefStruct(Module* pModule, mdToken tk)
+{
+ if (TypeFromToken(tk) == mdtTypeRef)
+ {
+ if(!ClassLoader::ResolveTokenToTypeDefThrowing(pModule, tk, &pModule, &tk))
+ {
+ return false;
+ }
+ }
+
+ HRESULT hr = pModule->GetCustomAttribute(tk,
+ WellKnownAttribute::IsByRefLike,
+ NULL, NULL);
+
+ if (hr == S_OK)
+ return true;
+
+ return false;
+}
+
+AsyncMethodSignatureKind ClassifyAsyncMethodSignatureCore(SigPointer sig, Module* pModule, PCCOR_SIGNATURE initialSig, ULONG* offsetOfAsyncDetails, bool *isValueTask);
+
+AsyncMethodSignatureKind ClassifyAsyncMethodSignature(SigPointer sig, Module* pModule, ULONG* offsetOfAsyncDetails, bool *isValueTask)
+{
+ PCCOR_SIGNATURE initialSig = sig.GetPtr();
+ uint32_t data;
+ IfFailThrow(sig.GetCallingConvInfo(&data));
+ if (data & IMAGE_CEE_CS_CALLCONV_GENERIC)
+ {
+ // Skip over generic argument count
+ IfFailThrow(sig.GetData(&data));
+ }
+
+ // Return argument count
+ IfFailThrow(sig.GetData(&data));
+ return ClassifyAsyncMethodSignatureCore(sig, pModule, initialSig, offsetOfAsyncDetails, isValueTask);
+}
+
+AsyncMethodSignatureKind ClassifyAsyncMethodSignatureCore(SigPointer sig, Module* pModule, PCCOR_SIGNATURE initialSig, ULONG* offsetOfAsyncDetails, bool *pIsValueTask)
+{
+ uint32_t data;
+ bool dummy = false;
+ if (pIsValueTask == NULL)
+ {
+ pIsValueTask = &dummy;
+ }
+
+ *pIsValueTask = false;
+
+ // Now we should be parsing the return type
+
+ // If the first custommodifier is a MOD_REQ to CallConvAsync2Call
+ // Then this is a async2 function
+ CorElementType elemType;
+ if (offsetOfAsyncDetails != NULL)
+ *offsetOfAsyncDetails = (ULONG)(sig.GetPtr() - initialSig);
+ BYTE elemTypeByte;
+ IfFailThrow(sig.GetByte(&elemTypeByte)); // Don't use GetElemType as it skips custom modifiers
+ elemType = (CorElementType)elemTypeByte;
+
+ LPCSTR name, _namespace;
+ mdToken tk;
+ while ((elemType == ELEMENT_TYPE_CMOD_OPT) || (elemType == ELEMENT_TYPE_CMOD_REQD))
+ {
+ CorElementType elemTypeCmod = elemType;
+
+ IfFailThrow(sig.GetToken(&tk));
+ GetNameOfTypeDefOrRef(pModule, tk, &name, &_namespace);
+ IfFailThrow(sig.GetByte(&elemTypeByte)); // Don't use GetElemType as it skips custom modifiers
+ elemType = (CorElementType)elemTypeByte;
+ if (strcmp(_namespace, "System.Threading.Tasks") == 0 && IsTypeDefOrRefImplementedInSystemModule(pModule, tk) && (elemTypeCmod == ELEMENT_TYPE_CMOD_REQD))
+ {
+ if ((strcmp(name, "Task`1") == 0) || (strcmp(name, "ValueTask`1") == 0))
+ {
+ *pIsValueTask = name[0] == 'V';
+ // This must have been the last CMOD before the element type, and the element type MUST be one which can be expressed structurally as a generic parameter
+ switch (elemType)
+ {
+ case ELEMENT_TYPE_CLASS:
+ case ELEMENT_TYPE_VALUETYPE:
+ case ELEMENT_TYPE_STRING:
+ case ELEMENT_TYPE_OBJECT:
+ case ELEMENT_TYPE_I:
+ case ELEMENT_TYPE_U:
+ case ELEMENT_TYPE_I1:
+ case ELEMENT_TYPE_U1:
+ case ELEMENT_TYPE_I2:
+ case ELEMENT_TYPE_U2:
+ case ELEMENT_TYPE_I4:
+ case ELEMENT_TYPE_U4:
+ case ELEMENT_TYPE_I8:
+ case ELEMENT_TYPE_U8:
+ case ELEMENT_TYPE_R4:
+ case ELEMENT_TYPE_R8:
+ case ELEMENT_TYPE_BOOLEAN:
+ case ELEMENT_TYPE_CHAR:
+ case ELEMENT_TYPE_ARRAY:
+ case ELEMENT_TYPE_SZARRAY:
+ case ELEMENT_TYPE_VAR:
+ case ELEMENT_TYPE_MVAR:
+ return AsyncMethodSignatureKind::Async2Method;
+ case ELEMENT_TYPE_TYPEDBYREF:
+ ThrowHR(COR_E_BADIMAGEFORMAT);
+ case ELEMENT_TYPE_GENERICINST:
+ IfFailThrow(sig.GetElemType(&elemType));
+ if (elemType == ELEMENT_TYPE_CLASS)
+ {
+ return AsyncMethodSignatureKind::Async2Method;
+ }
+ else
+ {
+ IfFailThrow(sig.GetToken(&tk));
+ if (IsTypeDefOrRefAByRefStruct(pModule, tk))
+ {
+ ThrowHR(COR_E_BADIMAGEFORMAT);
+ }
+ else
+ {
+ return AsyncMethodSignatureKind::Async2Method;
+ }
+ }
+ default:
+ ThrowHR(COR_E_BADIMAGEFORMAT);
+ }
+ }
+ else if ((strcmp(name, "Task") == 0) || (strcmp(name, "ValueTask") == 0))
+ {
+ *pIsValueTask = name[0] == 'V';
+ if (elemType == ELEMENT_TYPE_VOID)
+ {
+ return AsyncMethodSignatureKind::Async2MethodNonGeneric;
+ }
+ else
+ {
+ ThrowHR(COR_E_BADIMAGEFORMAT);
+ }
+ }
+ }
+
+ if (offsetOfAsyncDetails != NULL)
+ *offsetOfAsyncDetails = (ULONG)(sig.GetPtr() - initialSig) - 1;
+ }
+
+ if (elemType == ELEMENT_TYPE_GENERICINST)
+ {
+ IfFailThrow(sig.GetElemType(&elemType));
+ if (elemType == ELEMENT_TYPE_INTERNAL)
+ return AsyncMethodSignatureKind::NormalMethod;
+
+ *pIsValueTask = (elemType == ELEMENT_TYPE_VALUETYPE);
+ IfFailThrow(sig.GetToken(&tk));
+ IfFailThrow(sig.GetData(&data));
+ if (data == 1)
+ {
+ // This might be System.Threading.Tasks.Task`1
+ GetNameOfTypeDefOrRef(pModule, tk, &name, &_namespace);
+ if ((strcmp(name, *pIsValueTask ? "ValueTask`1" : "Task`1") == 0) && strcmp(_namespace, "System.Threading.Tasks") == 0)
+ {
+ if (IsTypeDefOrRefImplementedInSystemModule(pModule, tk))
+ return AsyncMethodSignatureKind::TaskReturningMethod;
+ }
+ }
+ }
+ else if ((elemType == ELEMENT_TYPE_CLASS) || (elemType == ELEMENT_TYPE_VALUETYPE))
+ {
+ IfFailThrow(sig.GetToken(&tk));
+ *pIsValueTask = (elemType == ELEMENT_TYPE_VALUETYPE);
+ // This might be System.Threading.Tasks.Task or ValueTask
+ GetNameOfTypeDefOrRef(pModule, tk, &name, &_namespace);
+ if ((strcmp(name, *pIsValueTask ? "ValueTask" : "Task") == 0) && strcmp(_namespace, "System.Threading.Tasks") == 0)
+ {
+ if (IsTypeDefOrRefImplementedInSystemModule(pModule, tk))
+ return AsyncMethodSignatureKind::TaskNonGenericReturningMethod;
+ }
+ }
+
+ return AsyncMethodSignatureKind::NormalMethod;
+}
+
//---------------------------------------------------------------------------------------
//
// Used by BuildMethodTable
@@ -2700,7 +2963,7 @@ MethodTableBuilder::EnumerateClassMethods()
if ((DWORD)MAX_SLOT_INDEX <= cMethAndGaps)
BuildMethodTableThrowException(IDS_CLASSLOAD_TOO_MANY_METHODS);
- bmtMethod->m_cMaxDeclaredMethods = (SLOT_INDEX)cMethAndGaps;
+ bmtMethod->m_cMaxDeclaredMethods = (SLOT_INDEX)cMethAndGaps * 2;
bmtMethod->m_cDeclaredMethods = 0;
bmtMethod->m_rgDeclaredMethods = new (GetStackingAllocator())
bmtMDMethod *[bmtMethod->m_cMaxDeclaredMethods];
@@ -2729,7 +2992,7 @@ MethodTableBuilder::EnumerateClassMethods()
{
BuildMethodTableThrowException(BFA_METHOD_TOKEN_OUT_OF_RANGE);
}
- if (!bmtProp->fNoSanityChecks && FAILED(pMDInternalImport->GetSigOfMethodDef(tok, &cMemberSignature, &pMemberSignature)))
+ if (FAILED(pMDInternalImport->GetSigOfMethodDef(tok, &cMemberSignature, &pMemberSignature)))
{
BuildMethodTableThrowException(hr, BFA_BAD_SIGNATURE, mdMethodDefNil);
}
@@ -2775,6 +3038,19 @@ MethodTableBuilder::EnumerateClassMethods()
}
}
+ SigParser sig(pMemberSignature, cMemberSignature);
+ ULONG offsetOfAsyncDetails = 0;
+ bool isAsyncValueType = false;
+ AsyncMethodSignatureKind asyncMethodType;
+
+ asyncMethodType = IsDelegate() ? AsyncMethodSignatureKind::NormalMethod : ClassifyAsyncMethodSignature(sig, GetModule(), &offsetOfAsyncDetails, &isAsyncValueType);
+
+ if (IsAsyncSigAsync2(asyncMethodType))
+ {
+ // not expected in actual metadata methods
+ BuildMethodTableThrowException(IDS_CLASSLOAD_BADFORMAT);
+ }
+
bool hasGenericMethodArgsComputed = false;
bool hasGenericMethodArgs = this->GetModule()->m_pMethodIsGenericMap->IsGeneric(tok, &hasGenericMethodArgsComputed);
if (!hasGenericMethodArgsComputed)
@@ -3310,28 +3586,164 @@ MethodTableBuilder::EnumerateClassMethods()
// Create a new bmtMDMethod representing this method and add it to the
// declared method list.
//
+ bmtMDMethod *pDeclaredMethod = NULL;
+ for (int insertCount = 0; insertCount < 2; insertCount++)
+ {
+ bmtMDMethod * pNewMethod;
+ if (insertCount == 0)
+ {
+ pNewMethod = new (GetStackingAllocator()) bmtMDMethod(
+ bmtInternal->pType,
+ tok,
+ dwMemberAttrs,
+ dwImplFlags,
+ dwMethodRVA,
+ type,
+ implType);
+
+ if (IsAsyncSigTaskReturning(asyncMethodType))
+ {
+ // ordinary Task-returning method:
+ // declare a TaskReturning method and add a helper thunk with Async2 signature
+ //
+ // IsMiAsync Task-returning method:
+ // declare a RuntimeAsync method and add a helper method with the actual implementation
+ // the RuntimeAsync method becomes a thunk to the implementation helper.
+ pNewMethod->SetAsyncMethodKind(IsMiAsync(dwImplFlags) ? AsyncMethodKind::RuntimeAsync : AsyncMethodKind::TaskReturning);
+ }
+ else
+ {
+ _ASSERTE(IsAsyncSigNormal(asyncMethodType));
- bmtMDMethod * pNewMethod = new (GetStackingAllocator()) bmtMDMethod(
- bmtInternal->pType,
- tok,
- dwMemberAttrs,
- dwImplFlags,
- dwMethodRVA,
- type,
- implType);
+ if (IsMiAsync(dwImplFlags))
+ {
+ // Explicitly-async methods have special semantics that is useful in the implementation of runtime async itself.
+ // It should not be valid to declare such methods outside of runtime infrastructure methods.
+ if (!IsTypeDefOrRefImplementedInSystemModule(GetModule(), this->GetCl()))
+ {
+ BuildMethodTableThrowException(IDS_CLASSLOAD_BADFORMAT);
+ }
- bmtMethod->AddDeclaredMethod(pNewMethod);
+ pNewMethod->SetAsyncMethodKind(AsyncMethodKind::Async2ExplicitImpl);
+ }
+ else
+ {
+ pNewMethod->SetAsyncMethodKind(AsyncMethodKind::NotAsync);
+ }
+ }
- //
- // Update the count of the various types of methods.
- //
+ pDeclaredMethod = pNewMethod;
+ }
+ else
+ {
+ ULONG cAsyncThunkMemberSignature = cMemberSignature;
+ AsyncMethodKind asyncKind;
+ ULONG originalTokenOffsetFromAsyncDetailsOffset;
+ ULONG newTokenOffsetFromAsyncDetailsOffset;
+ ULONG originalPrefixSize;
+ ULONG originalSuffixSize;
+ ULONG newSuffixSize;
+ ULONG newPrefixSize;
+
+ if (asyncMethodType == AsyncMethodSignatureKind::TaskNonGenericReturningMethod)
+ {
+ cAsyncThunkMemberSignature += 1;
+ originalTokenOffsetFromAsyncDetailsOffset = 1;
+ newTokenOffsetFromAsyncDetailsOffset = 1;
+ asyncKind = IsMiAsync(dwImplFlags) ? AsyncMethodKind::Async2VariantImpl : AsyncMethodKind::Async2VariantThunk;
+ originalPrefixSize = 1;
+ newPrefixSize = 1;
+ originalSuffixSize = 0;
+ newSuffixSize = 1;
+ }
+ else if (asyncMethodType == AsyncMethodSignatureKind::TaskReturningMethod)
+ {
+ cAsyncThunkMemberSignature -= 2;
+ originalTokenOffsetFromAsyncDetailsOffset = 2;
+ newTokenOffsetFromAsyncDetailsOffset = 1;
+ asyncKind = IsMiAsync(dwImplFlags)? AsyncMethodKind::Async2VariantImpl : AsyncMethodKind::Async2VariantThunk;
+ originalPrefixSize = 2;
+ newPrefixSize = 1;
+ originalSuffixSize = 1;
+ newSuffixSize = 0;
+ }
+ else
+ {
+ UNREACHABLE();
+ }
- bmtVT->dwMaxVtableSize++;
+ BYTE* pNewMemberSignature = AllocateFromHighFrequencyHeap(S_SIZE_T(cAsyncThunkMemberSignature));
+ ULONG tokenLen = CorSigUncompressedDataSize(&pMemberSignature[offsetOfAsyncDetails + originalTokenOffsetFromAsyncDetailsOffset]);
+ ULONG originalTokenOffset = offsetOfAsyncDetails + originalTokenOffsetFromAsyncDetailsOffset;
+ ULONG newTokenOffset = offsetOfAsyncDetails + newTokenOffsetFromAsyncDetailsOffset;
+ ULONG originalRemainingSigOffset = offsetOfAsyncDetails + originalPrefixSize + tokenLen + originalSuffixSize;
+ ULONG newRemainingSigOffset = offsetOfAsyncDetails + newPrefixSize + tokenLen + newSuffixSize;
- // Increment the number of non-abstract declared methods
- if (!IsMdAbstract(dwMemberAttrs))
- {
- bmtMethod->dwNumDeclaredNonAbstractMethods++;
+ ULONG initialCopyLen = offsetOfAsyncDetails;
+ memcpy(pNewMemberSignature, pMemberSignature, initialCopyLen);
+ memcpy(pNewMemberSignature + newTokenOffset, pMemberSignature + originalTokenOffset, tokenLen);
+
+ _ASSERTE((cMemberSignature - originalRemainingSigOffset) == (cAsyncThunkMemberSignature - newRemainingSigOffset));
+ memcpy(pNewMemberSignature + newRemainingSigOffset, pMemberSignature + originalRemainingSigOffset, cMemberSignature - originalRemainingSigOffset);
+
+ BYTE elemTypeClassOrValuetype = isAsyncValueType ? (BYTE)ELEMENT_TYPE_VALUETYPE : (BYTE)ELEMENT_TYPE_CLASS;
+
+ if (asyncMethodType == AsyncMethodSignatureKind::TaskNonGenericReturningMethod)
+ {
+ // Incoming sig will look like ... E_T_CLASS/E_T_VALUETYPE
+ // and needs to be translated to ELEMENT_TYPE_CMOD_REQD E_T_VOID
+
+ // Replace the E_T_CLASS/E_T_VALUETYPE with ELEMENT_TYPE_CMOD_REQD, and then add the E_T_VOID
+ pNewMemberSignature[offsetOfAsyncDetails] = ELEMENT_TYPE_CMOD_REQD;
+ pNewMemberSignature[newRemainingSigOffset - 1] = ELEMENT_TYPE_VOID;
+ }
+ else
+ {
+ _ASSERTE(asyncMethodType == AsyncMethodSignatureKind::TaskReturningMethod);
+ // Incoming sig will look something like ... E_T_GENERICINST E_T_CLASS/E_T_VALUETYPE 1 E_T_I4 ....
+ // And needs to be translated to ELEMENT_TYPE_CMOD_REQD E_T_I4
+
+ // Replace the ELEMENT_TYPE_GENERICINST with ELEMENT_TYPE_CMOD_REQD, and then remove the 1 which specifies the generic arg count for Task
+ pNewMemberSignature[offsetOfAsyncDetails] = ELEMENT_TYPE_CMOD_REQD;
+ }
+
+ Signature newMemberSig(pNewMemberSignature, cAsyncThunkMemberSignature);
+ pNewMethod = new (GetStackingAllocator()) bmtMDMethod(
+ bmtInternal->pType,
+ tok,
+ dwMemberAttrs,
+ dwImplFlags,
+ dwMethodRVA,
+ newMemberSig,
+ asyncKind,
+ type,
+ implType);
+
+ _ASSERTE(pNewMethod->IsAsync2Variant());
+
+ pNewMethod->SetAsyncOtherVariant(pDeclaredMethod);
+ pDeclaredMethod->SetAsyncOtherVariant(pNewMethod);
+ }
+
+ bmtMethod->AddDeclaredMethod(pNewMethod);
+
+ //
+ // Update the count of the various types of methods.
+ //
+
+ bmtVT->dwMaxVtableSize++;
+
+ // Increment the number of non-abstract declared methods
+ if (!IsMdAbstract(dwMemberAttrs))
+ {
+ bmtMethod->dwNumDeclaredNonAbstractMethods++;
+ }
+
+ // Normal methods only insert a single method
+ if (IsAsyncSigNormal(asyncMethodType))
+ {
+ break;
+ }
}
}
@@ -5162,6 +5574,9 @@ MethodTableBuilder::InitNewMethodDesc(
if (NeedsNativeCodeSlot(pMethod))
pNewMD->SetHasNativeCodeSlot();
+ if (pMethod->GetAsyncMethodKind() != AsyncMethodKind::NotAsync)
+ pNewMD->SetHasAsyncMethodData();
+
// Now we know the classification we can allocate the correct type of
// method desc and perform any classification specific initialization.
@@ -5189,6 +5604,12 @@ MethodTableBuilder::InitNewMethodDesc(
strcpy_s((char *) pszDebugMethodNameCopy, len, pszDebugMethodName);
#endif // _DEBUG
+ Signature sig;
+ if (pMethod->IsAsync2Variant())
+ {
+ sig = pMethod->GetMethodSignature().GetSignatureClass();
+ }
+
// Do the init specific to each classification of MethodDesc & assign some common fields
InitMethodDesc(pNewMD,
pMethod->GetMethodType(),
@@ -5198,7 +5619,9 @@ MethodTableBuilder::InitNewMethodDesc(
FALSE, // fEnC
pMethod->GetRVA(),
GetMDImport(),
- pName
+ pName,
+ sig,
+ pMethod->GetAsyncMethodKind()
COMMA_INDEBUG(pszDebugMethodNameCopy)
COMMA_INDEBUG(GetDebugClassName())
COMMA_INDEBUG("") // FIX this happens on global methods, give better info
@@ -5479,7 +5902,7 @@ MethodTableBuilder::PlaceVirtualMethods()
// that the name+signature corresponds to. Used by ProcessMethodImpls and ProcessInexactMethodImpls
// Always returns the first match that it finds. Affects the ambiguities in code:#ProcessInexactMethodImpls_Ambiguities
MethodTableBuilder::bmtMethodHandle
-MethodTableBuilder::FindDeclMethodOnInterfaceEntry(bmtInterfaceEntry *pItfEntry, MethodSignature &declSig, bool searchForStaticMethods)
+MethodTableBuilder::FindDeclMethodOnInterfaceEntry(bmtInterfaceEntry *pItfEntry, MethodSignature &declSig, AsyncVariantLookup variantLookup, bool searchForStaticMethods)
{
STANDARD_VM_CONTRACT;
@@ -5516,6 +5939,25 @@ MethodTableBuilder::FindDeclMethodOnInterfaceEntry(bmtInterfaceEntry *pItfEntry,
}
}
+ if (variantLookup == AsyncVariantLookup::AsyncOtherVariant && !declMethod.IsNull())
+ {
+ bmtRTMethod* declRTMethod = declMethod.AsRTMethod();
+ declMethod = {};
+ for (; !slotIt.AtEnd(); slotIt.Next())
+ {
+ bmtRTMethod* slotDeclMethod = slotIt->Decl().AsRTMethod();
+
+ if ((slotDeclMethod->GetOwningType() == declRTMethod->GetOwningType()) &&
+ (slotDeclMethod->GetMethodDesc()->GetMethodTable() == declRTMethod->GetMethodDesc()->GetMethodTable()) &&
+ (slotDeclMethod->GetMethodDesc()->GetMemberDef() == declRTMethod->GetMethodDesc()->GetMemberDef()) &&
+ (slotDeclMethod->GetMethodDesc()->IsAsync2VariantMethod() != declRTMethod->GetMethodDesc()->IsAsync2VariantMethod()))
+ {
+ declMethod = slotIt->Decl();
+ break;
+ }
+ }
+ }
+
return declMethod;
}
@@ -5584,6 +6026,10 @@ MethodTableBuilder::ProcessInexactMethodImpls()
continue;
}
+ AsyncVariantLookup asyncVariantOfDeclToFind = !it->IsAsync2Variant() ?
+ AsyncVariantLookup::MatchingAsyncVariant :
+ AsyncVariantLookup::AsyncOtherVariant;
+
// If this method serves as the BODY of a MethodImpl specification, then
// we should iterate all the MethodImpl's for this class and see just how many
// of them this method participates in as the BODY.
@@ -5638,7 +6084,7 @@ MethodTableBuilder::ProcessInexactMethodImpls()
pItfEntry = &bmtInterface->pInterfaceMap[i];
// Search for declmethod on this interface
- declMethod = FindDeclMethodOnInterfaceEntry(pItfEntry, declSig);
+ declMethod = FindDeclMethodOnInterfaceEntry(pItfEntry, declSig, asyncVariantOfDeclToFind);
// If we didn't find a match, continue on to next interface in the equivalence set
if (declMethod.IsNull())
@@ -5725,6 +6171,10 @@ MethodTableBuilder::ProcessMethodImpls()
continue;
}
+ AsyncVariantLookup asyncVariantOfDeclToFind = !it->IsAsync2Variant() ?
+ AsyncVariantLookup::MatchingAsyncVariant :
+ AsyncVariantLookup::AsyncOtherVariant;
+
// If this method serves as the BODY of a MethodImpl specification, then
// we should iterate all the MethodImpl's for this class and see just how many
// of them this method participates in as the BODY.
@@ -5766,7 +6216,7 @@ MethodTableBuilder::ProcessMethodImpls()
}
CONSISTENCY_CHECK(TypeFromToken(mdDecl) == mdtMethodDef);
- declMethod = bmtMethod->FindDeclaredMethodByToken(mdDecl);
+ declMethod = bmtMethod->FindDeclaredMethodByToken(mdDecl, asyncVariantOfDeclToFind);
}
else
{ // We can't call GetDescFromMemberDefOrRef here because this
@@ -5900,12 +6350,20 @@ MethodTableBuilder::ProcessMethodImpls()
}
// 3. Find the matching method.
- declMethod = FindDeclMethodOnInterfaceEntry(pItfEntry, declSig, isVirtualStaticOverride); // Search for statics when the impl is non-virtual
+ declMethod = FindDeclMethodOnInterfaceEntry(pItfEntry, declSig, asyncVariantOfDeclToFind, isVirtualStaticOverride); // Search for statics when the impl is non-virtual
}
else
{
GetHalfBakedClass()->SetHasVTableMethodImpl();
- declMethod = FindDeclMethodOnClassInHierarchy(it, pDeclMT, declSig);
+ declMethod = FindDeclMethodOnClassInHierarchy(it, pDeclMT, declSig, asyncVariantOfDeclToFind);
+ }
+
+ if (declMethod.IsNull() && asyncVariantOfDeclToFind == AsyncVariantLookup::AsyncOtherVariant)
+ {
+ // when implementing/overriding, we may see a Task-returning method
+ // which matches a T-returning method in the interface/base, which would not have variants.
+ // in such case the async2 variant of the Task-returning method does not implement/override anything.
+ continue;
}
if (declMethod.IsNull())
@@ -5956,7 +6414,7 @@ MethodTableBuilder::ProcessMethodImpls()
}
-MethodTableBuilder::bmtMethodHandle MethodTableBuilder::FindDeclMethodOnClassInHierarchy(const DeclaredMethodIterator& it, MethodTable * pDeclMT, MethodSignature &declSig)
+MethodTableBuilder::bmtMethodHandle MethodTableBuilder::FindDeclMethodOnClassInHierarchy(const DeclaredMethodIterator& it, MethodTable * pDeclMT, MethodSignature &declSig, AsyncVariantLookup variantLookup)
{
bmtRTType * pDeclType = NULL;
bmtMethodHandle declMethod;
@@ -6042,6 +6500,19 @@ MethodTableBuilder::bmtMethodHandle MethodTableBuilder::FindDeclMethodOnClassInH
FALSE,
iPass == 0 ? &newVisited : NULL))
{
+ if (variantLookup == AsyncVariantLookup::AsyncOtherVariant)
+ {
+ if (pCurMD->IsTaskReturningMethod() || pCurMD->IsAsync2VariantMethod())
+ {
+ pCurMD = pCurMD->GetAsyncOtherVariant();
+ }
+ else
+ {
+ declMethod = {};
+ break;
+ }
+ }
+
declMethod = (*bmtParent->pSlotTable)[pCurMD->GetSlot()].Decl();
break;
}
@@ -6069,7 +6540,9 @@ MethodTableBuilder::InitMethodDesc(
BOOL fEnC,
DWORD RVA, // Only needed for NDirect case
IMDInternalImport * pIMDII, // Needed for NDirect, EEImpl(Delegate) cases
- LPCSTR pMethodName // Only needed for mcEEImpl (Delegate) case
+ LPCSTR pMethodName, // Only needed for mcEEImpl (Delegate) case
+ Signature sig, // Only needed for the Async2 Thunk case
+ AsyncMethodKind asyncKind
COMMA_INDEBUG(LPCUTF8 pszDebugMethodName)
COMMA_INDEBUG(LPCUTF8 pszDebugClassName)
COMMA_INDEBUG(LPCUTF8 pszDebugMethodSignature)
@@ -6228,6 +6701,20 @@ MethodTableBuilder::InitMethodDesc(
#endif // !_DEBUG
pNewMD->SetSynchronized();
+ if (asyncKind != AsyncMethodKind::NotAsync)
+ {
+ AsyncMethodData* pAsyncMethodData = pNewMD->GetAddrOfAsyncMethodData();
+ pAsyncMethodData->kind = asyncKind;
+ if (asyncKind == AsyncMethodKind::Async2VariantThunk || asyncKind == AsyncMethodKind::Async2VariantImpl)
+ {
+ pAsyncMethodData->sig = sig;
+ }
+ else
+ {
+ _ASSERTE(sig.GetRawSig() == NULL);
+ }
+ }
+
#ifdef _DEBUG
pNewMD->m_pszDebugMethodName = (LPUTF8)pszDebugMethodName;
pNewMD->m_pszDebugClassName = (LPUTF8)pszDebugClassName;
@@ -6959,6 +7446,9 @@ VOID MethodTableBuilder::AllocAndInitMethodDescs()
if (NeedsNativeCodeSlot(*it))
size += sizeof(MethodDesc::NativeCodeSlot);
+
+ if (it->GetAsyncMethodKind() != AsyncMethodKind::NotAsync)
+ size += sizeof(AsyncMethodData);
// See comment in AllocAndInitMethodDescChunk
if (NeedsTightlyBoundUnboxingStub(*it))
@@ -7086,6 +7576,21 @@ VOID MethodTableBuilder::AllocAndInitMethodDescChunk(COUNT_T startIndex, COUNT_T
if (bmtGenerics->GetNumGenericArgs() == 0) {
pUnboxedMD->SetHasNonVtableSlot();
+
+ // By settings HasNonVTableSlot, the following chunks of data have been shifted around.
+ // This is an example of the fragility noted in the memcpy comment above
+ if (pUnboxedMD->HasNativeCodeSlot())
+ {
+ *pUnboxedMD->GetAddrOfNativeCodeSlot() = *pMD->GetAddrOfNativeCodeSlot();
+ }
+ if (pUnboxedMD->HasMethodImplSlot())
+ {
+ *pUnboxedMD->GetMethodImpl() = *pMD->GetMethodImpl();
+ }
+ if (pUnboxedMD->HasAsyncMethodData())
+ {
+ *pUnboxedMD->GetAddrOfAsyncMethodData() = *pMD->GetAddrOfAsyncMethodData();
+ }
}
//////////////////////////////////////////////////////////
@@ -7132,7 +7637,6 @@ MethodTableBuilder::NeedsNativeCodeSlot(bmtMDMethod * pMDMethod)
{
LIMITED_METHOD_CONTRACT;
-
#ifdef FEATURE_TIERED_COMPILATION
// Keep in-sync with MethodDesc::DetermineAndSetIsEligibleForTieredCompilation()
if ((g_pConfig->TieredCompilation() &&
diff --git a/src/coreclr/vm/methodtablebuilder.h b/src/coreclr/vm/methodtablebuilder.h
index a361608bd66f5d..26e08a5d01bcf9 100644
--- a/src/coreclr/vm/methodtablebuilder.h
+++ b/src/coreclr/vm/methodtablebuilder.h
@@ -671,6 +671,31 @@ class MethodTableBuilder
INDEBUG(CheckGetMethodAttributes();)
}
+ //-----------------------------------------------------------------------------------------
+ // This constructor can be used with hard-coded signatures that are used for
+ // representing async2 variant methods
+ MethodSignature(
+ Module * pModule,
+ mdToken tok,
+ Signature sig,
+ const Substitution * pSubst)
+ : m_pModule(pModule),
+ m_tok(tok),
+ m_szName(NULL),
+ m_pSig(sig.GetRawSig()),
+ m_cSig(sig.GetRawSigLen()),
+ m_pSubst(pSubst),
+ m_nameHash(INVALID_NAME_HASH)
+ {
+ CONTRACTL {
+ PRECONDITION(CheckPointer(pModule));
+ PRECONDITION(TypeFromToken(tok) == mdtMethodDef ||
+ TypeFromToken(tok) == mdtMemberRef);
+ PRECONDITION(CheckPointer(m_pSig));
+ PRECONDITION(m_cSig != 0);
+ } CONTRACTL_END;
+ }
+
//-----------------------------------------------------------------------------------------
// This constructor can be used with hard-coded signatures that are used for
// locating .ctor and .cctor methods.
@@ -742,6 +767,11 @@ class MethodTableBuilder
GetSignature() const
{ WRAPPER_NO_CONTRACT; CheckGetMethodAttributes(); return m_pSig; }
+ //-----------------------------------------------------------------------------------------
+ // Returns the metadata signature for the method.
+ inline Signature GetSignatureClass() const
+ { WRAPPER_NO_CONTRACT; CheckGetMethodAttributes(); return Signature(m_pSig, (ULONG)m_cSig); }
+
//-----------------------------------------------------------------------------------------
// Returns the signature length.
inline size_t
@@ -914,6 +944,22 @@ class MethodTableBuilder
MethodClassification type,
METHOD_IMPL_TYPE implType);
+ //-----------------------------------------------------------------------------------------
+ // Constructor. This takes all the information already extracted from metadata interface
+ // because the place that creates these types already has this data. Alternatively,
+ // a constructor could be written to take a token and metadata scope instead. Also,
+ // it might be interesting to move MethodClassification and METHOD_IMPL_TYPE to setter functions.
+ bmtMDMethod(
+ bmtMDType * pOwningType,
+ mdMethodDef tok,
+ DWORD dwDeclAttrs,
+ DWORD dwImplAttrs,
+ DWORD dwRVA,
+ Signature sig,
+ AsyncMethodKind thunkKind,
+ MethodClassification type,
+ METHOD_IMPL_TYPE implType);
+
//-----------------------------------------------------------------------------------------
// Returns the type that owns the *declaration* of this method. This makes sure that a
// method can be properly interpreted in the context of substitutions at any time.
@@ -1014,6 +1060,26 @@ class MethodTableBuilder
GetRVA() const
{ LIMITED_METHOD_CONTRACT; return m_dwRVA; }
+ bool IsAsync2Variant() const
+ {
+ return GetAsyncMethodKind() == AsyncMethodKind::Async2VariantThunk ||
+ GetAsyncMethodKind() == AsyncMethodKind::Async2VariantImpl;
+ }
+
+ void SetAsyncMethodKind(AsyncMethodKind kind)
+ {
+ m_asyncMethodKind = kind;
+ }
+
+ AsyncMethodKind GetAsyncMethodKind() const
+ {
+ LIMITED_METHOD_CONTRACT;
+ return m_asyncMethodKind;
+ }
+
+ bmtMDMethod * GetAsyncOtherVariant() const { return m_asyncOtherVariant; }
+ void SetAsyncOtherVariant(bmtMDMethod* pAsyncOtherVariant) { m_asyncOtherVariant = pAsyncOtherVariant; }
+
private:
//-----------------------------------------------------------------------------------------
bmtMDType * m_pOwningType;
@@ -1022,8 +1088,10 @@ class MethodTableBuilder
DWORD m_dwImplAttrs;
DWORD m_dwRVA;
MethodClassification m_type; // Specific MethodDesc flavour
+ AsyncMethodKind m_asyncMethodKind;
METHOD_IMPL_TYPE m_implType; // Whether or not the method is a methodImpl body
MethodSignature m_methodSig;
+ bmtMDMethod* m_asyncOtherVariant = NULL;
MethodDesc * m_pMD; // MethodDesc created and assigned to this method
MethodDesc * m_pUnboxedMD; // Unboxing MethodDesc if this is a virtual method on a valuetype
@@ -1916,14 +1984,22 @@ class MethodTableBuilder
// Searches the declared methods for a method with a token value equal to tok.
bmtMDMethod *
FindDeclaredMethodByToken(
- mdMethodDef tok)
+ mdMethodDef tok, AsyncVariantLookup variantLookup)
{
LIMITED_METHOD_CONTRACT;
for (SLOT_INDEX i = 0; i < m_cDeclaredMethods; ++i)
{
if ((*this)[i]->GetMethodSignature().GetToken() == tok)
{
- return (*this)[i];
+ auto result = (*this)[i];
+ if (variantLookup == AsyncVariantLookup::AsyncOtherVariant)
+ {
+ return result->GetAsyncOtherVariant();
+ }
+ else
+ {
+ return result;
+ }
}
}
return NULL;
@@ -2609,7 +2685,9 @@ class MethodTableBuilder
BOOL fEnC,
DWORD RVA, // Only needed for NDirect case
IMDInternalImport * pIMDII, // Needed for NDirect, EEImpl(Delegate) cases
- LPCSTR pMethodName // Only needed for mcEEImpl (Delegate) case
+ LPCSTR pMethodName, // Only needed for mcEEImpl (Delegate) case
+ Signature sig, // Only needed for the async thunk (Async2 Thunk) case
+ AsyncMethodKind asyncKind
COMMA_INDEBUG(LPCUTF8 pszDebugMethodName)
COMMA_INDEBUG(LPCUTF8 pszDebugClassName)
COMMA_INDEBUG(LPCUTF8 pszDebugMethodSignature));
@@ -2686,13 +2764,13 @@ class MethodTableBuilder
// Find the decl method on a given interface entry that matches the method name+signature specified
// If none is found, return a null method handle
bmtMethodHandle
- FindDeclMethodOnInterfaceEntry(bmtInterfaceEntry *pItfEntry, MethodSignature &declSig, bool searchForStaticMethods = false);
+ FindDeclMethodOnInterfaceEntry(bmtInterfaceEntry *pItfEntry, MethodSignature &declSig, AsyncVariantLookup variantLookup, bool searchForStaticMethods = false);
// --------------------------------------------------------------------------------------------
// Find the decl method within the class hierarchy method name+signature specified
// If none is found, return a null method handle
bmtMethodHandle
- FindDeclMethodOnClassInHierarchy(const DeclaredMethodIterator& it, MethodTable * pDeclMT, MethodSignature &declSig);
+ FindDeclMethodOnClassInHierarchy(const DeclaredMethodIterator& it, MethodTable * pDeclMT, MethodSignature &declSig, AsyncVariantLookup variantLookup);
// --------------------------------------------------------------------------------------------
// Throws if an entry already exists that has been MethodImpl'd. Adds the interface slot and
diff --git a/src/coreclr/vm/multicorejit.cpp b/src/coreclr/vm/multicorejit.cpp
index ab6d2e65fc74ea..15393e28f16b89 100644
--- a/src/coreclr/vm/multicorejit.cpp
+++ b/src/coreclr/vm/multicorejit.cpp
@@ -398,6 +398,12 @@ HRESULT MulticoreJitRecorder::WriteOutput(IStream * pStream)
}
MethodDesc * pMethod = m_JitInfoArray[i].GetMethodDescAndClean();
+ if (pMethod->IsAsync2VariantMethod())
+ {
+ // TODO consider adding support for async variants in the future
+ skipped++;
+ continue;
+ }
if (m_JitInfoArray[i].IsGenericMethodInfo())
{
diff --git a/src/coreclr/vm/namespace.h b/src/coreclr/vm/namespace.h
index 699049b3ea0847..f3ba695d07dc2d 100644
--- a/src/coreclr/vm/namespace.h
+++ b/src/coreclr/vm/namespace.h
@@ -16,6 +16,7 @@
#define g_RuntimeNS g_SystemNS ".Runtime"
#define g_IONS g_SystemNS ".IO"
#define g_ThreadingNS g_SystemNS ".Threading"
+#define g_TasksNS g_ThreadingNS ".Tasks"
#define g_CollectionsNS g_SystemNS ".Collections"
#define g_ResourcesNS g_SystemNS ".Resources"
#define g_GlobalizationNS g_SystemNS ".Globalization"
diff --git a/src/coreclr/vm/prestub.cpp b/src/coreclr/vm/prestub.cpp
index dc9dbeeace5d88..fc47da0587e59c 100644
--- a/src/coreclr/vm/prestub.cpp
+++ b/src/coreclr/vm/prestub.cpp
@@ -729,6 +729,15 @@ namespace
COR_ILMETHOD_DECODER* pHeader = NULL;
COR_ILMETHOD* ilHeader = pConfig->GetILHeader();
+
+ // For async method the methoddef represents a thunk with runtime-provided implementation,
+ // while the default IL logically belongs to the implementation method desc.
+ // If config returned no IL for an implementation method desc, then ask the method desc itself.
+ if (ilHeader == NULL && pMD->IsAsync2VariantMethod() && !pMD->IsAsyncThunkMethod())
+ {
+ ilHeader = pMD->GetILHeader();
+ }
+
if (ilHeader == NULL)
return NULL;
@@ -1032,6 +1041,630 @@ PCODE MethodDesc::JitCompileCodeLocked(PrepareCodeConfig* pConfig, COR_ILMETHOD_
return pCode;
}
+
+bool MethodDesc::TryGenerateTransientILImplementation(DynamicResolver** resolver, COR_ILMETHOD_DECODER** methodILDecoder)
+{
+ if (TryGenerateAsyncThunk(resolver, methodILDecoder))
+ {
+ return true;
+ }
+
+ if (TryGenerateUnsafeAccessor(resolver, methodILDecoder))
+ {
+ return true;
+ }
+
+ return false;
+}
+
+bool MethodDesc::TryGenerateAsyncThunk(DynamicResolver** resolver, COR_ILMETHOD_DECODER** methodILDecoder)
+{
+ STANDARD_VM_CONTRACT;
+ _ASSERTE(resolver != NULL);
+ _ASSERTE(methodILDecoder != NULL);
+ _ASSERTE(*resolver == NULL && *methodILDecoder == NULL);
+ _ASSERTE(IsIL());
+ _ASSERTE(GetRVA() == 0);
+
+ if (!IsAsyncThunkMethod())
+ {
+ return false;
+ }
+
+ MethodDesc* pAsyncOtherVariant = this->GetAsyncOtherVariant();
+ _ASSERTE(!IsWrapperStub() && !pAsyncOtherVariant->IsWrapperStub());
+
+ MetaSig msig(this);
+
+ SigTypeContext sigContext(pAsyncOtherVariant);
+ ILStubLinker sl(
+ GetModule(),
+ GetSignature(),
+ &sigContext,
+ pAsyncOtherVariant,
+ (ILStubLinkerFlags)ILSTUB_LINKER_FLAG_NONE);
+
+ if (IsAsync2Method())
+ {
+ EmitAsync2MethodThunk(pAsyncOtherVariant, msig, &sl);
+ }
+ else
+ {
+ EmitJitStateMachineBasedRuntimeAsyncThunk(pAsyncOtherVariant, msig, &sl);
+ }
+
+ NewHolder ilResolver = new ILStubResolver();
+ // Initialize the resolver target details.
+ ilResolver->SetStubMethodDesc(this);
+ ilResolver->SetStubTargetMethodDesc(pAsyncOtherVariant);
+
+ // Generate all IL associated data for JIT
+ {
+ UINT maxStack;
+ size_t cbCode = sl.Link(&maxStack);
+ DWORD cbSig = sl.GetLocalSigSize();
+
+ COR_ILMETHOD_DECODER* pILHeader = ilResolver->AllocGeneratedIL(cbCode, cbSig, maxStack);
+ BYTE* pbBuffer = (BYTE*)pILHeader->Code;
+ BYTE* pbLocalSig = (BYTE*)pILHeader->LocalVarSig;
+ _ASSERTE(cbSig == pILHeader->cbLocalVarSig);
+
+ size_t numEH = sl.GetNumEHClauses();
+ if (numEH > 0)
+ {
+ sl.WriteEHClauses(ilResolver->AllocEHSect(numEH));
+ }
+
+ sl.GenerateCode(pbBuffer, cbCode);
+ sl.GetLocalSig(pbLocalSig, cbSig);
+
+ // Store the token lookup map
+ ilResolver->SetTokenLookupMap(sl.GetTokenLookupMap());
+ ilResolver->SetJitFlags(CORJIT_FLAGS(CORJIT_FLAGS::CORJIT_FLAG_IL_STUB));
+
+ *resolver = (DynamicResolver*)ilResolver;
+ *methodILDecoder = pILHeader;
+ }
+
+ ilResolver.SuppressRelease();
+ return true;
+}
+
+void MethodDesc::EmitJitStateMachineBasedRuntimeAsyncThunk(MethodDesc* pAsyncOtherVariant, MetaSig& thunkMsig, ILStubLinker* pSL)
+{
+ _ASSERTE(!pAsyncOtherVariant->IsAsyncThunkMethod());
+
+ ILCodeStream* pCode = pSL->NewCodeStream(ILStubLinker::kDispatch);
+
+ unsigned continuationLocal = pCode->NewLocal(LocalDesc(CoreLibBinder::GetClass(CLASS__CONTINUATION)));
+
+ TypeHandle thTaskRet = thunkMsig.GetRetTypeHandleThrowing();
+
+ bool isValueTask = thTaskRet.GetMethodTable()->IsValueType();
+
+ LocalDesc returnLocalDesc(thTaskRet);
+ DWORD returnLocal = pCode->NewLocal(returnLocalDesc);
+
+ TypeHandle thLogicalRetType;
+ DWORD logicalResultLocal = UINT_MAX;
+ if (thTaskRet.GetNumGenericArgs() > 0)
+ {
+ thLogicalRetType = thTaskRet.GetMethodTable()->GetInstantiation()[0];
+ logicalResultLocal = pCode->NewLocal(LocalDesc(thLogicalRetType));
+ }
+
+ LocalDesc exceptionLocalDesc(CoreLibBinder::GetClass(CLASS__EXCEPTION));
+ DWORD exceptionLocal = pCode->NewLocal(exceptionLocalDesc);
+
+ LocalDesc executionAndSyncBlockStoreLocalDesc(CoreLibBinder::GetClass(CLASS__EXECUTIONANDSYNCBLOCKSTORE));
+ DWORD executionAndSyncBlockStoreLocal = pCode->NewLocal(executionAndSyncBlockStoreLocalDesc);
+
+ ILCodeLabel* pNoExceptionLabel = pCode->NewCodeLabel();
+ ILCodeLabel* pReturnResultLabel = pCode->NewCodeLabel();
+ ILCodeLabel* pSuspendedLabel = pCode->NewCodeLabel();
+
+ pCode->EmitLDLOCA(executionAndSyncBlockStoreLocal);
+ pCode->EmitCALL(pCode->GetToken(CoreLibBinder::GetMethod(METHOD__EXECUTIONANDSYNCBLOCKSTORE__PUSH)), 1, 0);
+
+ {
+ pCode->BeginTryBlock();
+ pCode->EmitNOP("Separate try blocks");
+ {
+ pCode->BeginTryBlock();
+
+ DWORD localArg = 0;
+ if (thunkMsig.HasThis())
+ {
+ pCode->EmitLDARG(localArg++);
+ }
+
+ for (UINT iArg = 0; iArg < thunkMsig.NumFixedArgs(); iArg++)
+ {
+ pCode->EmitLDARG(localArg++);
+ }
+
+ int token;
+ _ASSERTE(!pAsyncOtherVariant->IsWrapperStub());
+ if (pAsyncOtherVariant->HasClassOrMethodInstantiation())
+ {
+ // For generic code emit generic signatures.
+ int typeSigToken = mdTokenNil;
+ if (pAsyncOtherVariant->HasClassInstantiation())
+ {
+ SigBuilder typeSigBuilder;
+ typeSigBuilder.AppendElementType(ELEMENT_TYPE_GENERICINST);
+ typeSigBuilder.AppendElementType(ELEMENT_TYPE_INTERNAL);
+ // TODO: Encoding potentially shared method tables in
+ // signatures of tokens seems odd, but this hits assert
+ // with the typical method table.
+ typeSigBuilder.AppendPointer(pAsyncOtherVariant->GetMethodTable());
+ DWORD numClassTypeArgs = pAsyncOtherVariant->GetNumGenericClassArgs();
+ typeSigBuilder.AppendData(numClassTypeArgs);
+ for (DWORD i = 0; i < numClassTypeArgs; ++i)
+ {
+ typeSigBuilder.AppendElementType(ELEMENT_TYPE_VAR);
+ typeSigBuilder.AppendData(i);
+ }
+
+ DWORD typeSigLen;
+ PCCOR_SIGNATURE typeSig = (PCCOR_SIGNATURE)typeSigBuilder.GetSignature(&typeSigLen);
+ typeSigToken = pCode->GetSigToken(typeSig, typeSigLen);
+ }
+
+ if (pAsyncOtherVariant->HasMethodInstantiation())
+ {
+ SigBuilder methodSigBuilder;
+ DWORD numMethodTypeArgs = pAsyncOtherVariant->GetNumGenericMethodArgs();
+ methodSigBuilder.AppendByte(IMAGE_CEE_CS_CALLCONV_GENERICINST);
+ methodSigBuilder.AppendData(numMethodTypeArgs);
+ for (DWORD i = 0; i < numMethodTypeArgs; ++i)
+ {
+ methodSigBuilder.AppendElementType(ELEMENT_TYPE_MVAR);
+ methodSigBuilder.AppendData(i);
+ }
+
+ DWORD sigLen;
+ PCCOR_SIGNATURE sig = (PCCOR_SIGNATURE)methodSigBuilder.GetSignature(&sigLen);
+ int methodSigToken = pCode->GetSigToken(sig, sigLen);
+ token = pCode->GetToken(pAsyncOtherVariant, typeSigToken, methodSigToken);
+ }
+ else
+ {
+ token = pCode->GetToken(pAsyncOtherVariant, typeSigToken);
+ }
+ }
+ else
+ {
+ token = pCode->GetToken(pAsyncOtherVariant);
+ }
+
+ pCode->EmitCALL(token, localArg, logicalResultLocal != UINT_MAX ? 1 : 0);
+
+ if (logicalResultLocal != UINT_MAX)
+ pCode->EmitSTLOC(logicalResultLocal);
+ pCode->EmitCALL(METHOD__STUBHELPERS__ASYNC2_CALL_CONTINUATION, 0, 1);
+ pCode->EmitSTLOC(continuationLocal);
+ pCode->EmitLEAVE(pNoExceptionLabel);
+ pCode->EndTryBlock();
+ }
+ // Catch
+ {
+ pCode->BeginCatchBlock(pCode->GetToken(CoreLibBinder::GetClass(CLASS__EXCEPTION)));
+
+ int fromExceptionToken;
+ if (logicalResultLocal != UINT_MAX)
+ {
+ MethodDesc* fromExceptionMD;
+ if (isValueTask)
+ fromExceptionMD = CoreLibBinder::GetMethod(METHOD__VALUETASK__FROM_EXCEPTION_1);
+ else
+ fromExceptionMD = CoreLibBinder::GetMethod(METHOD__TASK__FROM_EXCEPTION_1);
+
+ fromExceptionMD = FindOrCreateAssociatedMethodDesc(fromExceptionMD, fromExceptionMD->GetMethodTable(), FALSE, Instantiation(&thLogicalRetType, 1), FALSE);
+
+ fromExceptionToken = GetTokenForGenericMethodCallWithAsyncReturnType(pCode, fromExceptionMD);
+ }
+ else
+ {
+ MethodDesc* fromExceptionMD;
+ if (isValueTask)
+ fromExceptionMD = CoreLibBinder::GetMethod(METHOD__VALUETASK__FROM_EXCEPTION);
+ else
+ fromExceptionMD = CoreLibBinder::GetMethod(METHOD__TASK__FROM_EXCEPTION);
+
+ fromExceptionToken = pCode->GetToken(fromExceptionMD);
+ }
+ pCode->EmitCALL(fromExceptionToken, 1, 1);
+ pCode->EmitSTLOC(returnLocal);
+ pCode->EmitLEAVE(pReturnResultLabel);
+ pCode->EndCatchBlock();
+ }
+ pCode->EndTryBlock();
+ }
+ //
+ {
+ pCode->BeginFinallyBlock();
+ pCode->EmitLDLOCA(executionAndSyncBlockStoreLocal);
+ pCode->EmitCALL(pCode->GetToken(CoreLibBinder::GetMethod(METHOD__EXECUTIONANDSYNCBLOCKSTORE__POP)), 1, 0);
+ pCode->EmitENDFINALLY();
+ pCode->EndFinallyBlock();
+ }
+
+ pCode->EmitLabel(pNoExceptionLabel);
+ pCode->EmitLDLOC(continuationLocal);
+ pCode->EmitBRTRUE(pSuspendedLabel);
+ if (logicalResultLocal != UINT_MAX)
+ {
+ pCode->EmitLDLOC(logicalResultLocal);
+ MethodDesc* md;
+ if (isValueTask)
+ md = CoreLibBinder::GetMethod(METHOD__VALUETASK__FROM_RESULT_T);
+ else
+ md = CoreLibBinder::GetMethod(METHOD__TASK__FROM_RESULT_T);
+ md = FindOrCreateAssociatedMethodDesc(md, md->GetMethodTable(), FALSE, Instantiation(&thLogicalRetType, 1), FALSE);
+
+ int fromResultToken = GetTokenForGenericMethodCallWithAsyncReturnType(pCode, md);
+ pCode->EmitCALL(fromResultToken, 1, 1);
+ }
+ else
+ {
+ if (isValueTask)
+ pCode->EmitCALL(METHOD__VALUETASK__GET_COMPLETED_TASK, 0, 1);
+ else
+ pCode->EmitCALL(METHOD__TASK__GET_COMPLETED_TASK, 0, 1);
+ }
+
+ pCode->EmitSTLOC(returnLocal);
+ pCode->EmitLabel(pReturnResultLabel);
+ pCode->EmitLDLOC(returnLocal);
+ pCode->EmitRET();
+
+ pCode->EmitLabel(pSuspendedLabel);
+
+ int finalizeTaskReturningThunkToken;
+ if (logicalResultLocal != UINT_MAX)
+ {
+ MethodDesc* md;
+ if (isValueTask)
+ md = CoreLibBinder::GetMethod(METHOD__RUNTIME_HELPERS__FINALIZE_VALUETASK_RETURNING_THUNK_1);
+ else
+ md = CoreLibBinder::GetMethod(METHOD__RUNTIME_HELPERS__FINALIZE_TASK_RETURNING_THUNK_1);
+
+ md = FindOrCreateAssociatedMethodDesc(md, md->GetMethodTable(), FALSE, Instantiation(&thLogicalRetType, 1), FALSE);
+ finalizeTaskReturningThunkToken = GetTokenForGenericMethodCallWithAsyncReturnType(pCode, md);
+ }
+ else
+ {
+ MethodDesc* md;
+ if (isValueTask)
+ md = CoreLibBinder::GetMethod(METHOD__RUNTIME_HELPERS__FINALIZE_VALUETASK_RETURNING_THUNK);
+ else
+ md = CoreLibBinder::GetMethod(METHOD__RUNTIME_HELPERS__FINALIZE_TASK_RETURNING_THUNK);
+ finalizeTaskReturningThunkToken = pCode->GetToken(md);
+ }
+ pCode->EmitLDLOC(continuationLocal);
+ pCode->EmitCALL(finalizeTaskReturningThunkToken, 1, 1);
+ pCode->EmitRET();
+}
+
+// Given an async method, return a SigPointer to the unwrapped result type. For
+// example, for async2 Task Foo() this returns the signature representing
+// (MVAR 0). For Task, it returns the signature representing (int).
+SigPointer MethodDesc::GetAsync2ThunkResultTypeSig()
+{
+ _ASSERTE(IsAsyncThunkMethod());
+ PCCOR_SIGNATURE pSigRaw;
+ DWORD cSig;
+ if (FAILED(GetMDImport()->GetSigOfMethodDef(GetMemberDef(), &cSig, &pSigRaw)))
+ {
+ _ASSERTE(!"Loaded MethodDesc should not fail to get signature");
+ pSigRaw = NULL;
+ cSig = 0;
+ }
+
+ SigPointer pSig(pSigRaw, cSig);
+ uint32_t callConvInfo;
+ IfFailThrow(pSig.GetCallingConvInfo(&callConvInfo));
+
+ if ((callConvInfo & IMAGE_CEE_CS_CALLCONV_GENERIC) != 0)
+ {
+ // GenParamCount
+ IfFailThrow(pSig.GetData(NULL));
+ }
+
+ // ParamCount
+ IfFailThrow(pSig.GetData(NULL));
+
+ // ReturnType comes now. Skip the modifiers (like async2 modifier).
+ IfFailThrow(pSig.SkipCustomModifiers());
+
+ CorElementType etype;
+ IfFailThrow(pSig.PeekElemType(&etype));
+
+ // here we should have something Task or ValueTask
+ _ASSERTE(etype == ELEMENT_TYPE_GENERICINST);
+
+ // GENERICINST