[TaskToThread] Add Task.runInline.

The new intrinsic, exposed via static functions on Task<T, Never> and
Task<T, Error> (rethrowing), begins an asynchronous context within a
synchronous caller's context.  This is only available for use under the
task-to-thread concurrency model, and even then only under SPI.

include/swift/ABI/MetadataValues.h

@@ -2360,6 +2360,8 @@ enum class TaskOptionRecordKind : uint8_t {
   AsyncLet  = 2,
   /// Request a child task for an 'async let'.
   AsyncLetWithBuffer = 3,
+  /// Request a child task for swift_task_run_inline.
+  RunInline = UINT8_MAX,
 };
 
 /// Flags for cancellation records.

include/swift/ABI/TaskOptions.h

@@ -141,6 +141,24 @@ public:
   }
 };
 
+class RunInlineTaskOptionRecord : public TaskOptionRecord {
+  void *allocation;
+  size_t allocationBytes;
+
+public:
+  RunInlineTaskOptionRecord(void *allocation, size_t allocationBytes)
+      : TaskOptionRecord(TaskOptionRecordKind::RunInline),
+        allocation(allocation), allocationBytes(allocationBytes) {}
+
+  void *getAllocation() const { return allocation; }
+
+  size_t getAllocationBytes() const { return allocationBytes; }
+
+  static bool classof(const TaskOptionRecord *record) {
+    return record->getKind() == TaskOptionRecordKind::RunInline;
+  }
+};
+
 } // end namespace swift
 
 #endif

include/swift/AST/Builtins.def

@@ -894,6 +894,16 @@ BUILTIN_MISC_OPERATION(StartAsyncLet, "startAsyncLet", "", Special)
 /// destruction.
 BUILTIN_MISC_OPERATION(StartAsyncLetWithLocalBuffer, "startAsyncLetWithLocalBuffer", "", Special)
 
+/// taskRunInline()<T>: (
+///     () async -> T
+/// ) -> T
+///
+/// Create an async context inline in the current synchronous context and run
+/// the specified closure.
+///
+/// This is only supported under the task-to-thread concurrency model.
+BUILTIN_MISC_OPERATION(TaskRunInline, "taskRunInline", "", Special)
+
 /// endAsyncLet(): (Builtin.RawPointer) -> Void
 ///
 /// DEPRECATED. The swift_asyncLet_finish intrinsic and endAsyncLetLifetime

include/swift/Basic/Features.def

@@ -82,6 +82,7 @@ LANGUAGE_FEATURE(BuiltinCreateAsyncTaskInGroup, 0, "MainActor executor building
 LANGUAGE_FEATURE(BuiltinMove, 0, "Builtin.move()", true)
 LANGUAGE_FEATURE(BuiltinCopy, 0, "Builtin.copy()", true)
 LANGUAGE_FEATURE(BuiltinStackAlloc, 0, "Builtin.stackAlloc", true)
+LANGUAGE_FEATURE(BuiltinTaskRunInline, 0, "Builtin.taskRunInline", true)
 SUPPRESSIBLE_LANGUAGE_FEATURE(SpecializeAttributeWithAvailability, 0, "@_specialize attribute with availability", true)
 LANGUAGE_FEATURE(BuiltinAssumeAlignment, 0, "Builtin.assumeAlignment", true)
 SUPPRESSIBLE_LANGUAGE_FEATURE(UnsafeInheritExecutor, 0, "@_unsafeInheritExecutor", true)

include/swift/Runtime/Concurrency.h

@@ -92,6 +92,20 @@ AsyncTaskAndContext swift_task_create_common(
     TaskContinuationFunction *function, void *closureContext,
     size_t initialContextSize);
 
+#if SWIFT_CONCURRENCY_TASK_TO_THREAD_MODEL
+#define SWIFT_TASK_RUN_INLINE_INITIAL_CONTEXT_BYTES 4096
+/// Begin an async context in the current sync context and run the indicated
+/// closure in it.
+///
+/// This is only supported under the task-to-thread concurrency model and
+/// relies on a synchronous implementation of task blocking in order to work.
+SWIFT_EXPORT_FROM(swift_Concurrency)
+SWIFT_CC(swift)
+void swift_task_run_inline(OpaqueValue *result, void *closureAFP,
+                           OpaqueValue *closureContext,
+                           const Metadata *futureResultType);
+#endif
+
 /// Allocate memory in a task.
 ///
 /// This must be called synchronously with the task.

include/swift/Runtime/RuntimeFunctions.def

@@ -2033,6 +2033,24 @@ FUNCTION(EndAsyncLet,
          ATTRS(NoUnwind),
          EFFECT(Concurrency))
 
+/// void swift_task_run_inline(
+///     OpaqueValue *result,
+///     void *closureAFP,
+///     OpaqueValue *closureContext,
+///     const Metadata *futureResultType
+/// )
+FUNCTION(TaskRunInline,
+         swift_task_run_inline, SwiftCC,
+         TaskRunInlineAvailability,
+         RETURNS(VoidTy),
+         ARGS(OpaquePtrTy,       // OpaqueValue *result
+              Int8PtrTy,         // void *closure
+              OpaquePtrTy,       // OpaqueValue *closureContext
+              TypeMetadataPtrTy, // const Metadata *futureResultType
+         ),
+         ATTRS(NoUnwind),
+         EFFECT(NoEffect))
+
 // void swift_taskGroup_initialize(TaskGroup *group);
 FUNCTION(TaskGroupInitialize,
          swift_taskGroup_initialize, SwiftCC,

lib/AST/ASTPrinter.cpp

@@ -2905,6 +2905,8 @@ static bool usesFeatureBuiltinMove(Decl *decl) {
 
 static bool usesFeatureBuiltinCopy(Decl *decl) { return false; }
 
+static bool usesFeatureBuiltinTaskRunInline(Decl *) { return false; }
+
 static bool usesFeatureSpecializeAttributeWithAvailability(Decl *decl) {
   if (auto func = dyn_cast<AbstractFunctionDecl>(decl)) {
     for (auto specialize : func->getAttrs().getAttributes<SpecializeAttr>()) {

lib/AST/Builtins.cpp

@@ -1488,6 +1488,14 @@ static ValueDecl *getCreateAsyncTaskInGroup(ASTContext &ctx, Identifier id) {
   return builder.build(id);
 }
 
+static ValueDecl *getTaskRunInline(ASTContext &ctx, Identifier id) {
+  return getBuiltinFunction(
+      ctx, id, _thin, _generics(_unrestricted),
+      _parameters(
+          _function(_async(_noescape(_thick)), _typeparam(0), _parameters())),
+      _typeparam(0));
+}
+
 static ValueDecl *getConvertTaskToJob(ASTContext &ctx, Identifier id) {
   return getBuiltinFunction(ctx, id,
                             _thin,
@@ -2838,6 +2846,9 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) {
   case BuiltinValueKind::CreateAsyncTaskInGroup:
     return getCreateAsyncTaskInGroup(Context, Id);
 
+  case BuiltinValueKind::TaskRunInline:
+    return getTaskRunInline(Context, Id);
+
   case BuiltinValueKind::TargetOSVersionAtLeast:
     return getTargetOSVersionAtLeast(Context, Id);
 

lib/IRGen/GenBuiltin.cpp

@@ -283,6 +283,15 @@ void irgen::emitBuiltinCall(IRGenFunction &IGF, const BuiltinInfo &Builtin,
     return;
   }
 
+  if (Builtin.ID == BuiltinValueKind::TaskRunInline) {
+    auto result = args.claimNext();
+    auto closure = args.claimNext();
+    auto closureContext = args.claimNext();
+
+    emitTaskRunInline(IGF, substitutions, result, closure, closureContext);
+    return;
+  }
+
   if (Builtin.ID == BuiltinValueKind::CreateTaskGroup) {
     // Claim metadata pointer.
     (void)args.claimAll();

lib/IRGen/GenConcurrency.cpp

@@ -333,3 +333,18 @@ llvm::Function *IRGenModule::getAwaitAsyncContinuationFn() {
   Builder.CreateRetVoid();
   return suspendFn;
 }
+
+void irgen::emitTaskRunInline(IRGenFunction &IGF, SubstitutionMap subs,
+                              llvm::Value *result, llvm::Value *closure,
+                              llvm::Value *closureContext) {
+  assert(subs.getReplacementTypes().size() == 1 &&
+         "taskRunInline should have a type substitution");
+  auto resultType = subs.getReplacementTypes()[0]->getCanonicalType();
+  auto resultTypeMetadata = IGF.emitAbstractTypeMetadataRef(resultType);
+
+  auto *call = IGF.Builder.CreateCall(
+      IGF.IGM.getTaskRunInlineFn(),
+      {result, closure, closureContext, resultTypeMetadata});
+  call->setDoesNotThrow();
+  call->setCallingConv(IGF.IGM.SwiftCC);
+}

lib/IRGen/GenConcurrency.h

@@ -74,6 +74,10 @@ llvm::Value *emitCreateTaskGroup(IRGenFunction &IGF, SubstitutionMap subs);
 /// Emit the destroyTaskGroup builtin.
 void emitDestroyTaskGroup(IRGenFunction &IGF, llvm::Value *group);
 
+void emitTaskRunInline(IRGenFunction &IGF, SubstitutionMap subs,
+                       llvm::Value *result, llvm::Value *closure,
+                       llvm::Value *closureContext);
+
 } // end namespace irgen
 } // end namespace swift
 

lib/IRGen/IRGenModule.cpp

@@ -841,6 +841,15 @@ namespace RuntimeConstants {
     return RuntimeAvailability::AlwaysAvailable;
   }
 
+  RuntimeAvailability TaskRunInlineAvailability(ASTContext &context) {
+    if (context.LangOpts.isConcurrencyModelTaskToThread()) {
+      return RuntimeAvailability::AlwaysAvailable;
+    }
+    // swift_task_run_inline is only available under task-to-thread execution
+    // model.
+    return RuntimeAvailability::ConditionallyAvailable;
+  }
+
 } // namespace RuntimeConstants
 
 // We don't use enough attributes to justify generalizing the

lib/SIL/IR/OperandOwnership.cpp

@@ -865,6 +865,8 @@ visitResumeThrowingContinuationThrowing(BuiltinInst *bi, StringRef attr) {
   return OperandOwnership::TrivialUse;
 }
 
+BUILTIN_OPERAND_OWNERSHIP(TrivialUse, TaskRunInline)
+
 BUILTIN_OPERAND_OWNERSHIP(InteriorPointer, CancelAsyncTask)
 BUILTIN_OPERAND_OWNERSHIP(InteriorPointer, InitializeDefaultActor)
 BUILTIN_OPERAND_OWNERSHIP(InteriorPointer, DestroyDefaultActor)

lib/SIL/IR/ValueOwnership.cpp

@@ -561,6 +561,7 @@ CONSTANT_OWNERSHIP_BUILTIN(None, StartAsyncLetWithLocalBuffer)
 CONSTANT_OWNERSHIP_BUILTIN(None, EndAsyncLetLifetime)
 CONSTANT_OWNERSHIP_BUILTIN(None, CreateTaskGroup)
 CONSTANT_OWNERSHIP_BUILTIN(None, DestroyTaskGroup)
+CONSTANT_OWNERSHIP_BUILTIN(None, TaskRunInline)
 CONSTANT_OWNERSHIP_BUILTIN(None, Move)
 CONSTANT_OWNERSHIP_BUILTIN(None, Copy)
 

lib/SIL/Utils/MemAccessUtils.cpp

@@ -2436,6 +2436,11 @@ static void visitBuiltinAddress(BuiltinInst *builtin,
       visitor(&builtin->getAllOperands()[2]);
       return;
 
+    // Writes back to the first operand.
+    case BuiltinValueKind::TaskRunInline:
+      visitor(&builtin->getAllOperands()[0]);
+      return;
+
     // These effect both operands.
     case BuiltinValueKind::Move:
     case BuiltinValueKind::Copy:

lib/SILOptimizer/Transforms/AccessEnforcementReleaseSinking.cpp

@@ -183,6 +183,7 @@ static bool isBarrier(SILInstruction *inst) {
     case BuiltinValueKind::StartAsyncLet:
     case BuiltinValueKind::CreateAsyncTask:
     case BuiltinValueKind::CreateAsyncTaskInGroup:
+    case BuiltinValueKind::TaskRunInline:
     case BuiltinValueKind::StartAsyncLetWithLocalBuffer:
     case BuiltinValueKind::ConvertTaskToJob:
     case BuiltinValueKind::InitializeDefaultActor:

stdlib/public/BackDeployConcurrency/Task.cpp

@@ -481,7 +481,7 @@ static AsyncTaskAndContext swift_task_create_commonImpl(
       jobFlags.task_setIsChildTask(true);
       break;
 
-    case TaskOptionRecordKind::AsyncLetWithBuffer:
+    case TaskOptionRecordKind::AsyncLetWithBuffer: {
       auto *aletRecord = cast<AsyncLetWithBufferTaskOptionRecord>(option);
       asyncLet = aletRecord->getAsyncLet();
       // TODO: Actually digest the result buffer into the async let task
@@ -494,6 +494,10 @@ static AsyncTaskAndContext swift_task_create_commonImpl(
       jobFlags.task_setIsChildTask(true);
       break;
     }
+    case TaskOptionRecordKind::RunInline:
+      assert(false && "used task option record kind which isn't back deployed");
+      break;
+    }
   }
 
   // Add to the task group, if requested.

stdlib/public/Concurrency/Task.cpp

@@ -472,6 +472,23 @@ static void completeTaskWithClosure(SWIFT_ASYNC_CONTEXT AsyncContext *context,
   return completeTaskAndRelease(context, error);
 }
 
+/// The function that we put in the context of an inline task to handle the
+/// final return.
+///
+/// Because inline tasks can't produce errors, this function doesn't have an
+/// error parameter.
+///
+/// Because inline tasks' closures are noescaping, their closure contexts are
+/// stack allocated; so this function doesn't release them.
+SWIFT_CC(swiftasync)
+static void completeInlineTask(SWIFT_ASYNC_CONTEXT AsyncContext *context) {
+  // Set that there's no longer a running task in the current thread.
+  auto task = _swift_task_clearCurrent();
+  assert(task && "completing task, but there is no active task registered");
+
+  completeTaskImpl(task, context, /*error=*/nullptr);
+}
+
 SWIFT_CC(swiftasync)
 static void non_future_adapter(SWIFT_ASYNC_CONTEXT AsyncContext *_context) {
   auto asyncContextPrefix = reinterpret_cast<AsyncContextPrefix *>(
@@ -619,6 +636,7 @@ static AsyncTaskAndContext swift_task_create_commonImpl(
   TaskGroup *group = nullptr;
   AsyncLet *asyncLet = nullptr;
   bool hasAsyncLetResultBuffer = false;
+  RunInlineTaskOptionRecord *runInlineOption = nullptr;
   for (auto option = options; option; option = option->getParent()) {
     switch (option->getKind()) {
     case TaskOptionRecordKind::Executor:
@@ -638,7 +656,7 @@ static AsyncTaskAndContext swift_task_create_commonImpl(
       jobFlags.task_setIsChildTask(true);
       break;
 
-    case TaskOptionRecordKind::AsyncLetWithBuffer:
+    case TaskOptionRecordKind::AsyncLetWithBuffer: {
       auto *aletRecord = cast<AsyncLetWithBufferTaskOptionRecord>(option);
       asyncLet = aletRecord->getAsyncLet();
       // TODO: Actually digest the result buffer into the async let task
@@ -651,6 +669,11 @@ static AsyncTaskAndContext swift_task_create_commonImpl(
       jobFlags.task_setIsChildTask(true);
       break;
     }
+    case TaskOptionRecordKind::RunInline: {
+      runInlineOption = cast<RunInlineTaskOptionRecord>(option);
+      break;
+    }
+    }
   }
 
   // Add to the task group, if requested.
@@ -765,6 +788,17 @@ static AsyncTaskAndContext swift_task_create_commonImpl(
       initialSlabSize = asyncLet->getSizeOfPreallocatedSpace()
                           - amountToAllocate;
     }
+  } else if (runInlineOption && runInlineOption->getAllocation()) {
+    // NOTE: If the space required for the task and initial context was
+    //       greater than SWIFT_TASK_RUN_INLINE_INITIAL_CONTEXT_BYTES,
+    //       getAllocation will return nullptr and we'll fall back to malloc to
+    //       allocate the buffer.
+    //
+    // This was already checked in swift_task_run_inline.
+    size_t runInlineBufferBytes = runInlineOption->getAllocationBytes();
+    assert(amountToAllocate <= runInlineBufferBytes);
+    allocation = runInlineOption->getAllocation();
+    initialSlabSize = runInlineBufferBytes - amountToAllocate;
   } else {
     allocation = malloc(amountToAllocate);
   }
@@ -852,12 +886,15 @@ static AsyncTaskAndContext swift_task_create_commonImpl(
                        task, task->getTaskId(), parent, basePriority);
 
   // Initialize the task-local allocator.
-  initialContext->ResumeParent = reinterpret_cast<TaskContinuationFunction *>(
-                            asyncLet       ? &completeTask
-                          : closureContext ? &completeTaskWithClosure
-                                           : &completeTaskAndRelease);
-  if (asyncLet && initialSlabSize > 0) {
-    assert(parent);
+  initialContext->ResumeParent =
+      runInlineOption ? &completeInlineTask
+                      : reinterpret_cast<TaskContinuationFunction *>(
+                            asyncLet         ? &completeTask
+                            : closureContext ? &completeTaskWithClosure
+                                             : &completeTaskAndRelease);
+  if ((asyncLet || (runInlineOption && runInlineOption->getAllocation())) &&
+      initialSlabSize > 0) {
+    assert(parent || (runInlineOption && runInlineOption->getAllocation()));
     void *initialSlab = (char*)allocation + amountToAllocate;
     task->Private.initializeWithSlab(basePriority, initialSlab,
                                      initialSlabSize);
@@ -934,6 +971,64 @@ getAsyncClosureEntryPointAndContextSize(void *function) {
           fnPtr->ExpectedContextSize};
 }
 
+#if SWIFT_CONCURRENCY_TASK_TO_THREAD_MODEL
+SWIFT_CC(swift)
+void swift::swift_task_run_inline(OpaqueValue *result, void *closureAFP,
+                                  OpaqueValue *closureContext,
+                                  const Metadata *futureResultType) {
+  // Ensure that we're currently in a synchronous context.
+  if (swift_task_getCurrent()) {
+    swift_Concurrency_fatalError(0, "called runInline within an async context");
+  }
+
+  // Unpack the asynchronous function pointer.
+  FutureAsyncSignature::FunctionType *closure;
+  size_t closureContextSize;
+  std::tie(closure, closureContextSize) =
+      getAsyncClosureEntryPointAndContextSize<
+          FutureAsyncSignature,
+          SpecialPointerAuthDiscriminators::AsyncFutureFunction>(closureAFP);
+
+  // If the initial task and initial async frame aren't too big, allocate enough
+  // stack space for them and for use as the initial task slab.
+  //
+  // If they are too big, swift_task_create_common will fall back to malloc.
+  size_t candidateAllocationBytes = SWIFT_TASK_RUN_INLINE_INITIAL_CONTEXT_BYTES;
+  size_t minimumAllocationSize =
+      amountToAllocateForHeaderAndTask(/*parent=*/nullptr, /*group=*/nullptr,
+                                       futureResultType, closureContextSize)
+          .second;
+  void *allocation = nullptr;
+  size_t allocationBytes = 0;
+  if (minimumAllocationSize <= candidateAllocationBytes) {
+    allocationBytes = candidateAllocationBytes;
+    allocation = alloca(allocationBytes);
+  }
+
+  // Create a task to run the closure.  Pass a RunInlineTaskOptionRecord
+  // containing a pointer to the allocation enabling us to provide our stack
+  // allocation rather than swift_task_create_common having to malloc it.
+  RunInlineTaskOptionRecord option(allocation, allocationBytes);
+  auto taskAndContext = swift_task_create_common(
+      /*rawTaskCreateFlags=*/0, &option, futureResultType,
+      reinterpret_cast<TaskContinuationFunction *>(closure), closureContext,
+      /*initialContextSize=*/closureContextSize);
+
+  // Run the task.
+  swift_job_run(taskAndContext.Task, ExecutorRef::generic());
+  // Under the task-to-thread concurrency model, the task should always have
+  // completed by this point.
+
+  // Copy the result out to our caller.
+  auto *futureResult = taskAndContext.Task->futureFragment()->getStoragePtr();
+  futureResultType->getValueWitnesses()->initializeWithCopy(
+      result, futureResult, futureResultType);
+
+  // Destroy the task.
+  taskAndContext.Task->~AsyncTask();
+}
+#endif
+
 SWIFT_CC(swift)
 AsyncTaskAndContext swift::swift_task_create(
     size_t taskCreateFlags,

stdlib/public/Concurrency/Task.swift

@@ -961,6 +961,67 @@ func _reportUnexpectedExecutor(_ _filenameStart: Builtin.RawPointer,
 @_silgen_name("swift_task_getCurrentThreadPriority")
 func _getCurrentThreadPriority() -> Int
 
+#if SWIFT_STDLIB_TASK_TO_THREAD_MODEL_CONCURRENCY
+@available(SwiftStdlib 5.8, *)
+@usableFromInline
+@_unavailableFromAsync(message: "Use _taskRunInline from a sync context to begin an async context.")
+internal func _taskRunInline<T>(_ body: () async -> T) -> T {
+#if compiler(>=5.5) && $BuiltinTaskRunInline
+  return Builtin.taskRunInline(body)
+#else
+  fatalError("Unsupported Swift compiler")
+#endif
+}
+
+@available(SwiftStdlib 5.8, *)
+extension Task where Failure == Never {
+  /// Start an async context within the current sync context and run the
+  /// provided async closure, returning the value it produces.
+  @available(SwiftStdlib 5.8, *)
+  @_spi(_TaskToThreadModel)
+  @_unavailableFromAsync(message: "Use Task.runInline from a sync context to begin an async context.")
+  public static func runInline(_ body: () async -> Success) -> Success {
+    return _taskRunInline(body)
+  }
+}
+
+@available(SwiftStdlib 5.8, *)
+extension Task where Failure == Error {
+  @available(SwiftStdlib 5.8, *)
+  @_alwaysEmitIntoClient
+  @usableFromInline
+  internal static func _runInlineHelper<T>(
+    body: () async -> Result<T, Error>,
+    rescue: (Result<T, Error>) throws -> T
+  ) rethrows -> T {
+    return try rescue(
+      _taskRunInline(body)
+    )
+  }
+
+  /// Start an async context within the current sync context and run the
+  /// provided async closure, returning or throwing the value or error it
+  /// produces.
+  @available(SwiftStdlib 5.8, *)
+  @_spi(_TaskToThreadModel)
+  @_unavailableFromAsync(message: "Use Task.runInline from a sync context to begin an async context.")
+  public static func runInline(_ body: () async throws -> Success) rethrows -> Success {
+    return try _runInlineHelper(
+      body: {
+        do { 
+          let value = try await body() 
+          return Result.success(value)
+        }
+        catch let error {
+          return Result.failure(error)
+        }
+    },
+      rescue: { try $0.get() }
+    )
+  }
+}
+#endif
+
 #if _runtime(_ObjC)
 
 /// Intrinsic used by SILGen to launch a task for bridging a Swift async method

test/Concurrency/builtin_silgen.swift

@@ -22,3 +22,12 @@ func runDetached() {
     await suspend()
   }
 }
+
+// CHECK-LABEL: sil{{.*}} @$s14builtin_silgen13testRunInlineyxxyYaXElF : {{.*}} {
+// CHECK:       {{bb[0-9]+}}([[RESULT:%[^,]+]] : $*T, [[CLOSURE:%[^,]+]] : $@noescape @async @callee_guaranteed @substituted <τ_0_0> () -> @out τ_0_0 for <T>):
+// CHECK:         builtin "taskRunInline"<T>([[RESULT]] : $*T, [[CLOSURE]] : $@noescape @async @callee_guaranteed @substituted <τ_0_0> () -> @out τ_0_0 for <T>) : $()
+// CHECK-LABEL: } // end sil function '$s14builtin_silgen13testRunInlineyxxyYaXElF'
+@available(SwiftStdlib 5.1, *)
+func testRunInline<T>(_ cl: () async -> T) -> T {
+  return Builtin.taskRunInline(cl)
+}

test/IRGen/async/builtins.sil

@@ -148,3 +148,22 @@ bb0:
   %21 = tuple ()
   return %21 : $()
 }
+
+
+// CHECK-LABEL: define{{.*}} swiftcc void @testRunInline(
+// CHECK-SAME:      %swift.opaque* noalias nocapture sret(%swift.opaque) [[RESULT:%[^,]+]],
+// CHECK-SAME:      i8* [[CLOSURE:%[^,]+]],
+// CHECK-SAME:      %swift.opaque* [[CLOSURE_CONTEXT:%[^,]+]],
+// CHECK-SAME:      %swift.type* [[FUTURE_RESULT_TYPE:%[^,]+]])
+// CHECK-SAME:  {
+// CHECK:         call swiftcc void @swift_task_run_inline(
+// CHECK-SAME:        %swift.opaque* [[RESULT]],
+// CHECK-SAME:        i8* [[CLOSURE]],
+// CHECK-SAME:        %swift.opaque* [[CLOSURE_CONTEXT]],
+// CHECK-SAME:        %swift.type* [[FUTURE_RESULT_TYPE]])
+// CHECK:       }
+sil hidden @testRunInline : $@convention(thin) <T> (@noescape @async @callee_guaranteed @substituted <τ_0_0> () -> @out τ_0_0 for <T>) -> @out T {
+entry(%result : $*T, %closure : $@noescape @async @callee_guaranteed @substituted <τ_0_0> () -> @out τ_0_0 for <T>):
+  %void = builtin "taskRunInline"<T>(%result : $*T, %closure : $@noescape @async @callee_guaranteed @substituted <τ_0_0> () -> @out τ_0_0 for <T>) : $()
+  return %void : $()
+}

test/stdlib/run_inline.swift

@@ -0,0 +1,441 @@
+// RUN: %target-run-simple-swift(-parse-as-library  -Xfrontend -disable-availability-checking -Xfrontend -concurrency-model=task-to-thread) | %FileCheck %s
+
+// REQUIRES: concurrency
+// REQUIRES: executable_test
+// REQUIRES: freestanding
+// REQUIRES: concurrency_runtime
+
+@_spi(_TaskToThreadModel) import _Concurrency
+
+// =============================================================================
+// Driver                                                                     {{
+// =============================================================================
+@main struct Main {
+  // Note that this isn't and can't be async.
+  static func main() {
+    // Test some of the following combinations
+    // - closure context: no, immutable, mutable
+    // - returns:         void, trivial, nontrivial, gigantic
+    // - throws:          can't, no, yes(trivial), yes(nontrivial)
+    // - callee:          no, sync, async
+
+    // 0000
+    NoClosureContext_VoidReturn_CantThrow_NoCallee()
+    // 0100
+    NoClosureContext_TrivialReturn_CantThrow_NoCallee()
+
+    // 1000
+    ImmutableClosureContext_VoidReturn_CantThrow_NoCallee()
+    // 1012
+    ImmutableClosureContext_VoidReturn_NoThrow_AsyncCallee()
+    // 1021
+    ImmutableClosureContext_VoidReturn_TrivialThrow_SyncCallee()
+    // 1101
+    ImmutableClosureContext_TrivialReturn_CantThrow_SyncCallee()
+    // 1301
+    ImmutableClosureContext_GiganticReturn_CantThrow_SyncCallee()
+
+    // 2000
+    MutableClosureContext_VoidReturn_CantThrow_NoCallee()
+    // 2212
+    MutableClosureContext_NontrivialReturn_NoThrow_AsyncCallee()
+    // 2232
+    MutableClosureContext_NontrivialReturn_NontrivialThrow_AsyncCallee()
+  }
+}
+// =============================================================================
+// Driver                                                                     }}
+// =============================================================================
+
+// =============================================================================
+// Tests                                                                      {{
+// =============================================================================
+func NoClosureContext_VoidReturn_CantThrow_NoCallee() {
+  // CHECK: NoClosureContext_VoidReturn_CantThrow_NoCallee() before
+  // CHECK: NoClosureContext_VoidReturn_CantThrow_NoCallee() during
+  // CHECK: NoClosureContext_VoidReturn_CantThrow_NoCallee() after
+  print("\(#function) before")
+  Task.runInline {
+    print("\(#function) during")
+  }
+  print("\(#function) after")
+}
+func NoClosureContext_TrivialReturn_CantThrow_NoCallee() {
+  // CHECK: NoClosureContext_TrivialReturn_CantThrow_NoCallee() before
+  // CHECK: NoClosureContext_TrivialReturn_CantThrow_NoCallee() during
+  // CHECK: NoClosureContext_TrivialReturn_CantThrow_NoCallee() result=Trivial(context: "NoClosureContext_TrivialReturn_CantThrow_NoCallee()")
+  // CHECK: NoClosureContext_TrivialReturn_CantThrow_NoCallee() after
+  print("\(#function) before")
+  let result = Task.runInline {
+    print("\(#function) during")
+    return Trivial(#function)
+  }
+  print("\(#function) result=\(result)")
+  print("\(#function) after")
+}
+
+func ImmutableClosureContext_VoidReturn_CantThrow_NoCallee() {
+  // CHECK: ImmutableClosureContext_VoidReturn_CantThrow_NoCallee() before: immutable
+  // CHECK: ImmutableClosureContext_VoidReturn_CantThrow_NoCallee() during: immutable
+  // CHECK: ImmutableClosureContext_VoidReturn_CantThrow_NoCallee() after: immutable
+  let immutableField = "immutable"
+  print("\(#function) before: \(immutableField)")
+  Task.runInline {
+    print("\(#function) during: \(immutableField)")
+  }
+  print("\(#function) after: \(immutableField)")
+}
+
+func ImmutableClosureContext_VoidReturn_NoThrow_AsyncCallee() {
+  // CHECK: ImmutableClosureContext_VoidReturn_NoThrow_AsyncCallee() before: immutable
+  // CHECK: ImmutableClosureContext_VoidReturn_NoThrow_AsyncCallee() during: immutable
+  // CHECK: CalleeAsync_NoThrow_VoidReturn()
+  // CHECK: ImmutableClosureContext_VoidReturn_NoThrow_AsyncCallee() result=success()
+  // CHECK: ImmutableClosureContext_VoidReturn_NoThrow_AsyncCallee() after: immutable
+  let immutableField = "immutable"
+  print("\(#function) before: \(immutableField)")
+  let result = Result {
+    try Task.runInline {
+      print("\(#function) during: \(immutableField)")
+      return try await CalleeAsync_NoThrow_VoidReturn()
+    }
+  }
+  print("\(#function) result=\(result)")
+  print("\(#function) after: \(immutableField)")
+}
+
+func ImmutableClosureContext_VoidReturn_TrivialThrow_SyncCallee() {
+  // CHECK: ImmutableClosureContext_VoidReturn_TrivialThrow_SyncCallee() before: immutable
+  // CHECK: ImmutableClosureContext_VoidReturn_TrivialThrow_SyncCallee() during: immutable
+  // CHECK: CalleeSync_TrivialThrow_VoidReturn()
+  // CHECK: ImmutableClosureContext_VoidReturn_TrivialThrow_SyncCallee() result=failure(ETrivial.trivial(Trivial(context: "CalleeSync_TrivialThrow_VoidReturn()")))
+  // CHECK: ImmutableClosureContext_VoidReturn_TrivialThrow_SyncCallee() after: immutable
+  let immutableField = "immutable"
+  print("\(#function) before: \(immutableField)")
+  let result = Result {
+    try Task.runInline {
+      print("\(#function) during: \(immutableField)")
+      try CalleeSync_TrivialThrow_VoidReturn()
+    }
+  }
+  print("\(#function) result=\(result)")
+  print("\(#function) after: \(immutableField)")
+}
+
+func ImmutableClosureContext_TrivialReturn_CantThrow_SyncCallee() {
+  // CHECK: ImmutableClosureContext_TrivialReturn_CantThrow_SyncCallee() before: immutable
+  // CHECK: ImmutableClosureContext_TrivialReturn_CantThrow_SyncCallee() during: immutable
+  // CHECK: CalleeSync_CantThrow_TrivialReturn()
+  // CHECK: ImmutableClosureContext_TrivialReturn_CantThrow_SyncCallee() result=Trivial(context: "CalleeSync_CantThrow_TrivialReturn()")
+  // CHECK: ImmutableClosureContext_TrivialReturn_CantThrow_SyncCallee() after: immutable
+  let immutableField = "immutable"
+  print("\(#function) before: \(immutableField)")
+  let result = Task.runInline {
+    print("\(#function) during: \(immutableField)")
+    return CalleeSync_CantThrow_TrivialReturn()
+  }
+  print("\(#function) result=\(result)")
+  print("\(#function) after: \(immutableField)")
+}
+
+func ImmutableClosureContext_GiganticReturn_CantThrow_SyncCallee() {
+  // CHECK: ImmutableClosureContext_GiganticReturn_CantThrow_SyncCallee() before: immutable
+  // CHECK: ImmutableClosureContext_GiganticReturn_CantThrow_SyncCallee() during: immutable
+  // CHECK: CalleeSync_CantThrow_GiganticReturn()
+  // CHECK: ImmutableClosureContext_GiganticReturn_CantThrow_SyncCallee() result=Gigantic(context: "CalleeSync_CantThrow_GiganticReturn()"
+  // CHECK: ImmutableClosureContext_GiganticReturn_CantThrow_SyncCallee() after: immutable
+  let immutableField = "immutable"
+  print("\(#function) before: \(immutableField)")
+  let result = Task.runInline {
+    print("\(#function) during: \(immutableField)")
+    return CalleeSync_CantThrow_GiganticReturn()
+  }
+  print("\(#function) result=\(result)")
+  print("\(#function) after: \(immutableField)")
+}
+
+func MutableClosureContext_VoidReturn_CantThrow_NoCallee() {
+  // CHECK: MutableClosureContext_VoidReturn_CantThrow_NoCallee() before: unchanged
+  // CHECK: MutableClosureContext_VoidReturn_CantThrow_NoCallee() during: unchanged
+  // CHECK: MutableClosureContext_VoidReturn_CantThrow_NoCallee() after: changed
+  var mutableField = "unchanged"
+  print("\(#function) before: \(mutableField)")
+  Task.runInline {
+    print("\(#function) during: \(mutableField)")
+    mutableField = "changed"
+  }
+  print("\(#function) after: \(mutableField)")
+}
+
+func MutableClosureContext_NontrivialReturn_NoThrow_AsyncCallee() {
+  // CHECK: MutableClosureContext_NontrivialReturn_NoThrow_AsyncCallee() before: unchanged
+  // CHECK: MutableClosureContext_NontrivialReturn_NoThrow_AsyncCallee() during: unchanged
+  // CHECK: X.init(CalleeAsync_NoThrow_NontrivialReturn()|retval)
+  // CHECK: MutableClosureContext_NontrivialReturn_NoThrow_AsyncCallee() result=success(Nontrivial(context: "CalleeAsync_NoThrow_NontrivialReturn()|retval"))
+  // CHECK: X.deinit(CalleeAsync_NoThrow_NontrivialReturn()|retval)
+  // CHECK: MutableClosureContext_NontrivialReturn_NoThrow_AsyncCallee() after: changed
+  var mutableField = "unchanged"
+  print("\(#function) before: \(mutableField)")
+  do {
+    let result = Result {
+      try Task.runInline {
+        print("\(#function) during: \(mutableField)")
+        mutableField = "changed"
+        return try await CalleeAsync_NoThrow_NontrivialReturn()
+      }
+    }
+    print("\(#function) result=\(result)")
+  }
+  print("\(#function) after: \(mutableField)")
+}
+
+func MutableClosureContext_NontrivialReturn_NontrivialThrow_AsyncCallee() {
+  // CHECK: MutableClosureContext_NontrivialReturn_NontrivialThrow_AsyncCallee() before: unchanged
+  // CHECK: MutableClosureContext_NontrivialReturn_NontrivialThrow_AsyncCallee() during: unchanged
+  // CHECK: X.init(CalleeAsync_NontrivialThrow_NontrivialReturn()|retval)
+  // CHECK: X.init(CalleeAsync_NontrivialThrow_NontrivialReturn()|error)
+  // CHECK: X.deinit(CalleeAsync_NontrivialThrow_NontrivialReturn()|retval)
+  // CHECK: MutableClosureContext_NontrivialReturn_NontrivialThrow_AsyncCallee() result=failure(ENontrivial.nontrivial(Nontrivial(context: "CalleeAsync_NontrivialThrow_NontrivialReturn()|error")))
+  // CHECK: X.deinit(CalleeAsync_NontrivialThrow_NontrivialReturn()|error)
+  // CHECK: MutableClosureContext_NontrivialReturn_NontrivialThrow_AsyncCallee() after: changed
+  var mutableField = "unchanged"
+  print("\(#function) before: \(mutableField)")
+  do {
+    let result = Result {
+      try Task.runInline {
+        print("\(#function) during: \(mutableField)")
+        mutableField = "changed"
+        return try await CalleeAsync_NontrivialThrow_NontrivialReturn()
+      }
+    }
+    print("\(#function) result=\(result)")
+  }
+  print("\(#function) after: \(mutableField)")
+}
+// =============================================================================
+// Tests                                                                      }}
+// =============================================================================
+
+// =============================================================================
+// Callees                                                                    {{
+// =============================================================================
+// Some of the combinations of the following:
+// - synchroneity: sync, async
+// - throwness: can't, no, trivial, nontrivial
+// - return: void, trivial, nontrivial, gigantic
+
+// 000
+func CalleeSync_CantThrow_VoidReturn() -> () {
+  print(#function)
+}
+
+// 020
+func CalleeSync_TrivialThrow_VoidReturn() throws -> () {
+  print(#function)
+  throw ETrivial.trivial(Trivial(#function))
+}
+
+// 002
+func CalleeSync_CantThrow_TrivialReturn() -> Trivial {
+  print(#function)
+  return Trivial(#function)
+}
+
+func CalleeSync_CantThrow_GiganticReturn() -> Gigantic {
+  print(#function)
+  return Gigantic(#function)
+}
+
+// 110
+func CalleeAsync_NoThrow_VoidReturn() async throws -> () {
+  print(#function)
+}
+
+// 112
+func CalleeAsync_NoThrow_NontrivialReturn() async throws -> Nontrivial {
+  print(#function)
+  let retval = Nontrivial("\(#function)|retval")
+  return retval
+}
+
+// 132
+func CalleeAsync_NontrivialThrow_NontrivialReturn() async throws -> Nontrivial {
+  print(#function)
+  let retval = Nontrivial("\(#function)|retval")
+  _ = retval
+  throw ENontrivial.nontrivial(Nontrivial("\(#function)|error"))
+}
+// =============================================================================
+// Callees                                                                    }}
+// =============================================================================
+
+// =============================================================================
+// Types                                                                      {{
+// =============================================================================
+class X {
+  var context: String
+  init(_ context: String) {
+    self.context = context
+    print("X.init(\(context))")
+  }
+  deinit {
+    print("X.deinit(\(self.context))")
+  }
+}
+struct Gigantic : CustomStringConvertible {
+  var description: String {
+    return "Gigantic(context: \"\(context)\")"
+  }
+  struct One {
+    var f0: Int = 111
+    var f1: Int = 111
+    var f2: Int = 111
+    var f3: Int = 111
+    var f4: Int = 111
+    var f5: Int = 111
+    var f6: Int = 111
+    var f7: Int = 111
+    var f8: Int = 111
+    var f9: Int = 111
+    var f10: Int = 111
+    var f11: Int = 111
+    var f12: Int = 111
+    var f13: Int = 111
+    var f14: Int = 111
+    var f15: Int = 111
+  }
+  struct Two {
+    var f0: One = One()
+    var f1: One = One()
+    var f2: One = One()
+    var f3: One = One()
+    var f4: One = One()
+    var f5: One = One()
+    var f6: One = One()
+    var f7: One = One()
+    var f8: One = One()
+    var f9: One = One()
+    var f10: One = One()
+    var f11: One = One()
+    var f12: One = One()
+    var f13: One = One()
+    var f14: One = One()
+    var f15: One = One()
+  }
+  var context: String
+  init(_ context: String) {
+    self.context = context
+    self.f0 = Two()
+    self.f1 = Two()
+  }
+  var f0: Two
+  var f1: Two
+}
+extension Result : CustomStringConvertible {
+  public var description: String {
+    switch self {
+      case .success(let success):
+      if success is Void {
+        return "success()"
+      } else if let csc = success as? CustomStringConvertible {
+        return "success(\(csc.description))"
+      } else {
+        return "success(OPAQUE)"
+      }
+      case .failure(let failure):
+      if let csc = failure as? CustomStringConvertible {
+        return "failure(\(csc.description))"
+      } else {
+        return "failure(OPAQUE)"
+      }
+    }
+  }
+}
+struct Nontrivial : CustomStringConvertible {
+  var description: String {
+    return "Nontrivial(context: \"\(context)\")"
+  }
+  var context: String
+  init(_ context: String) {
+    self.context = context
+    self.f0 = X(context)
+    self.f1 = 111
+    self.f2 = 111
+    self.f3 = 111
+    self.f4 = 111
+    self.f5 = 111
+    self.f6 = 111
+    self.f7 = 111
+    self.f8 = 111
+    self.f9 = 111
+  }
+  var f0: X
+  var f1: Int
+  var f2: Int
+  var f3: Int
+  var f4: Int
+  var f5: Int
+  var f6: Int
+  var f7: Int
+  var f8: Int
+  var f9: Int
+}
+struct Trivial : CustomStringConvertible {
+  var description: String {
+    return "Trivial(context: \"\(context)\")"
+  }
+  var context: String
+  init(_ context: String) {
+    self.context = context
+    self.f0 = 111
+    self.f1 = 111
+    self.f2 = 111
+    self.f3 = 111
+    self.f4 = 111
+    self.f5 = 111
+    self.f6 = 111
+    self.f7 = 111
+    self.f8 = 111
+    self.f9 = 111
+  }
+  var f0: Int
+  var f1: Int
+  var f2: Int
+  var f3: Int
+  var f4: Int
+  var f5: Int
+  var f6: Int
+  var f7: Int
+  var f8: Int
+  var f9: Int
+}
+enum ETrivial : Error, CustomStringConvertible {
+  var description: String {
+    switch self {
+    case .nada:
+      return "ETrivial.nada"
+    case .trivial(let trivial):
+      return "ETrivial.trivial(\(trivial))"
+    }
+  }
+  case nada
+  case trivial(Trivial)
+}
+enum ENontrivial : Error, CustomStringConvertible {
+  var description: String {
+    switch self {
+    case .nada:
+      return "ENontrivial.nada"
+    case .trivial(let trivial):
+      return "ENontrivial.trivial(\(trivial))"
+    case .nontrivial(let nontrivial):
+      return "ENontrivial.nontrivial(\(nontrivial))"
+    }
+  }
+  case nada
+  case trivial(Trivial)
+  case nontrivial(Nontrivial)
+}
+// =============================================================================
+// Types                                                                      }}
+// =============================================================================

test/stdlib/run_inline_die.swift

@@ -0,0 +1,21 @@
+// RUN: %target-build-swift -parse-as-library -Xfrontend -disable-availability-checking -module-name a %s -o %t.out
+// RUN: %target-codesign %t.out
+// RUN: %target-run %t.out
+
+// REQUIRES: executable_test
+// REQUIRES: freestanding
+// REQUIRES: concurrency
+// REQUIRES: concurrency_runtime
+
+// This test should always exit with a non-zero code.
+// XFAIL: *
+
+@_spi(_TaskToThreadModel) import _Concurrency
+
+@main struct Main {
+  static func main() async {
+    // Check that we crash when calling runInline from within an async context--
+    // here the async context of an async @main.
+    Task.runInline {}
+  }
+}