Emit enum element arguments directly into the enum payload.

This generates significantly better code directly out of SILGen, at
the cost of having to reimplement a little bit of the argument-emission
logic to handle default arguments.  But it also neatly sidesteps the
problems we have with splitting tuple RValues when the tuple contains
a pack expansion, which will require some significant surgery to RValue
to fix.  That, in turn, fixes rdar://121489308.

lib/SILGen/SILGenApply.cpp

@@ -2677,6 +2677,16 @@ public:
     }
   }
 
+  DelayedArgument(DefaultArgumentExpr *defArg,
+                  AbstractionPattern origParamType,
+                  ClaimedParamsRef params,
+                  SILFunctionTypeRepresentation functionTypeRepresentation)
+    : DelayedArgument(defArg, defArg->getDefaultArgsOwner(),
+                      defArg->getParamIndex(),
+                      defArg->getType()->getCanonicalType(),
+                      origParamType, params, functionTypeRepresentation,
+                      defArg->isImplicitlyAsync()) {}
+
   DelayedArgument(SILLocation loc,
                   ConcreteDeclRef defaultArgsOwner,
                   unsigned destIndex,
@@ -3389,17 +3399,12 @@ public:
     // If this is delayed default argument, prepare to emit the default argument
     // generator later.
     if (arg.isDelayedDefaultArg()) {
-      auto substParamType = arg.getSubstRValueType();
       auto defArg = std::move(arg).asKnownDefaultArg();
 
       auto numParams = getFlattenedValueCount(origParamType,
                                               ImportAsMemberStatus());
-      DelayedArguments.emplace_back(defArg,
+      DelayedArguments.emplace_back(defArg, origParamType,
-                                    defArg->getDefaultArgsOwner(),
+                                    claimNextParameters(numParams), Rep);
-                                    defArg->getParamIndex(),
-                                    substParamType, origParamType,
-                                    claimNextParameters(numParams),
-                                    Rep, defArg->isImplicitlyAsync());
       Args.push_back(ManagedValue());
 
       maybeEmitForeignArgument();
@@ -4329,10 +4334,30 @@ private:
   }
 };
 
+static ManagedValue
+emitDefaultArgument(SILGenFunction &SGF, DefaultArgumentExpr *E,
+                    AbstractionPattern origType, SILType expectedTy,
+                    SGFContext origC) {
+  return SGF.emitAsOrig(E, origType, E->getType()->getCanonicalType(),
+                        expectedTy, origC,
+      [&](SILGenFunction &SGF, SILLocation loc, SGFContext C) {
+    auto result =
+      SGF.emitApplyOfDefaultArgGenerator(loc, E->getDefaultArgsOwner(),
+                                         E->getParamIndex(),
+                                         E->getType()->getCanonicalType(),
+                                         E->isImplicitlyAsync(),
+                                         C);
+    if (result.isInContext())
+      return ManagedValue::forInContext();
+    return std::move(result).getAsSingleValue(SGF, loc);
+  });
+}
+
 void DelayedArgument::emitDefaultArgument(SILGenFunction &SGF,
                                           const DefaultArgumentStorage &info,
                                           SmallVectorImpl<ManagedValue> &args,
                                           size_t &argIndex) {
+  // TODO: call emitDefaultArgument above.
   auto value = SGF.emitApplyOfDefaultArgGenerator(info.loc,
                                                   info.defaultArgsOwner,
                                                   info.destIndex,
@@ -4535,7 +4560,7 @@ public:
   void emit(SILGenFunction &SGF, CleanupLocation l, ForUnwind_t forUnwind) override {
     auto theBox = box;
     if (SGF.getASTContext().SILOpts.supportsLexicalLifetimes(SGF.getModule())) {
-      if (auto *bbi = cast<BeginBorrowInst>(theBox)) {
+      if (auto *bbi = dyn_cast<BeginBorrowInst>(theBox)) {
         SGF.B.createEndBorrow(l, bbi);
         theBox = bbi->getOperand();
       }
@@ -5132,7 +5157,7 @@ RValue CallEmission::applyEnumElementConstructor(SGFContext C) {
   // pattern, to ensure that function types in payloads are re-abstracted
   // correctly.
   auto formalType = callee.getSubstFormalType();
-  auto origFormalType = callee.getOrigFormalType();
+  CanType formalResultType = formalType.getResult();
 
   // We have a fully-applied enum element constructor: open-code the
   // construction.
@@ -5140,9 +5165,6 @@ RValue CallEmission::applyEnumElementConstructor(SGFContext C) {
 
   SILLocation uncurriedLoc = selfArg->Loc;
 
-  origFormalType = origFormalType.getFunctionResultType();
-  CanType formalResultType = formalType.getResult();
-
   // Ignore metatype argument
   SmallVector<ManagedValue, 0> metatypeVal;
   emitPseudoFunctionArguments(SGF, uncurriedLoc,
@@ -5152,34 +5174,17 @@ RValue CallEmission::applyEnumElementConstructor(SGFContext C) {
   assert(metatypeVal.size() == 1);
 
 
-  // Get the payload argument.
+  // Get the payload argument sources, if there are any.
-  ArgumentSource payload;
+  MutableArrayRef<ArgumentSource> payloads;
   if (element->hasAssociatedValues()) {
-    SmallVector<ManagedValue, 4> argVals;
+    payloads = std::move(*callSite).forward().getSources();
-    auto resultFnType = cast<FunctionType>(formalResultType);
-
-    emitPseudoFunctionArguments(SGF, uncurriedLoc,
-                                AbstractionPattern(resultFnType),
-                                resultFnType, argVals,
-                                std::move(*callSite).forward());
-
-    // We need to implode a tuple rvalue for enum construction. This is
-    // essentially an implosion of the internal arguments of a pseudo case
-    // constructor, so we can drop the parameter flags.
-    auto payloadTy = AnyFunctionType::composeTuple(
-        SGF.getASTContext(), resultFnType->getParams(),
-        ParameterFlagHandling::IgnoreNonEmpty);
-
-    auto arg = RValue(SGF, argVals, payloadTy->getCanonicalType());
-    payload = ArgumentSource(uncurriedLoc, std::move(arg));
     formalResultType = cast<FunctionType>(formalResultType).getResult();
-    origFormalType = origFormalType.getFunctionResultType();
   } else {
     assert(!callSite.has_value());
   }
 
   ManagedValue resultMV = SGF.emitInjectEnum(
-      uncurriedLoc, std::move(payload),
+      uncurriedLoc, payloads,
       SGF.getLoweredType(formalResultType),
       element, uncurriedContext);
 
@@ -6035,16 +6040,130 @@ void SILGenFunction::emitRawYield(SILLocation loc,
   B.emitBlock(resumeBB);
 }
 
+static bool isEnumElementPayloadTupled(EnumElementDecl *element,
+                                       MutableArrayRef<ArgumentSource> payloads,
+                                       CanTupleType &formalPayloadTupleType) {
+  auto params = element->getParameterList();
+  assert(params);
+  assert(payloads.size() == params->size());
+  if (payloads.size() == 1 && params->get(0)->getArgumentName().empty())
+    return false;
+
+  SmallVector<TupleTypeElt, 8> tupleElts;
+  for (auto i : indices(payloads)) {
+    tupleElts.push_back({payloads[i].getSubstRValueType(),
+                         params->get(0)->getArgumentName()});
+  }
+  formalPayloadTupleType = cast<TupleType>(
+    CanType(TupleType::get(tupleElts, element->getASTContext())));
+
+  return true;
+}
+
+static ManagedValue
+emitEnumElementPayloads(SILGenFunction &SGF, SILLocation loc,
+                        EnumElementDecl *element,
+                        MutableArrayRef<ArgumentSource> eltPayloads,
+                        AbstractionPattern origPayloadType, SILType payloadTy,
+                        Initialization *dest) {
+  // The payloads array is parallel to the parameters of the enum element.
+  // The abstraction pattern is taken from the element's argument interface
+  // type, so it has extra tuple structure if the argument interface type does.
+  CanTupleType formalPayloadTupleType;
+  auto treatAsTuple =
+    isEnumElementPayloadTupled(element, eltPayloads, formalPayloadTupleType);
+
+  // The Initialization we get passed is always one of several cases in
+  // emitInjectEnum, all of which are splittable.
+  assert(!treatAsTuple || !dest || dest->canSplitIntoTupleElements());
+
+  SmallVector<InitializationPtr, 4> eltInitBuffer;
+  SmallVector<ManagedValue, 4> elts;
+
+  MutableArrayRef<InitializationPtr> eltInits;
+  if (!treatAsTuple) {
+    // nothing required
+  } else if (dest) {
+    eltInits = dest->splitIntoTupleElements(SGF, loc, formalPayloadTupleType,
+                                            eltInitBuffer);
+  } else {
+    elts.reserve(eltPayloads.size());
+  }
+
+  // Do an initial pass, emitting non-default arguments.
+  SmallVector<unsigned, 4> delayedArgIndices;
+  for (auto i : indices(eltPayloads)) {
+    auto &eltPayload = eltPayloads[i];
+    if (eltPayload.isDelayedDefaultArg()) {
+      delayedArgIndices.push_back(i);
+      if (!dest) elts.push_back(ManagedValue());
+      continue;
+    }
+
+    AbstractionPattern origEltType =
+      (treatAsTuple ? origPayloadType.getTupleElementType(i) : origPayloadType);
+    SILType eltTy =
+      (treatAsTuple ? payloadTy.getTupleElementType(i) : payloadTy);
+    Initialization *eltInit =
+      (dest ? (treatAsTuple ? eltInits[i].get() : dest) : nullptr);
+    if (dest) {
+      std::move(eltPayload).forwardInto(SGF, origEltType, eltInit,
+                                        SGF.getTypeLowering(eltTy));
+    } else {
+      auto elt = std::move(eltPayload).getAsSingleValue(SGF, origEltType, eltTy);
+      elts.push_back(elt);
+    }
+  }
+
+  // Emit all of the default arguments in a separate pass.
+  for (auto i : delayedArgIndices) {
+    auto &eltPayload = eltPayloads[i];
+    AbstractionPattern  origEltType =
+      (treatAsTuple ? origPayloadType.getTupleElementType(i) : origPayloadType);
+    SILType eltTy =
+      (treatAsTuple ? payloadTy.getTupleElementType(i) : payloadTy);
+    Initialization *eltInit =
+      (dest ? (treatAsTuple ? eltInits[i].get() : dest) : nullptr);
+
+    auto result = emitDefaultArgument(SGF,
+                                      std::move(eltPayload).asKnownDefaultArg(),
+                                      origEltType, eltTy, SGFContext(eltInit));
+    if (dest) {
+      assert(result.isInContext());
+    } else {
+      elts[i] = result;
+    }
+  }
+
+  // If we're not breaking down a tuple, we can wrap up immediately.
+  if (!treatAsTuple) {
+    if (dest) return ManagedValue::forInContext();
+    assert(elts.size() == 1);
+    return elts[0];
+  }
+
+  // If we've been emitting into split element contexts, finish the
+  // overall tuple initialization.
+  if (dest) {
+    dest->finishInitialization(SGF);
+    return ManagedValue::forInContext();
+  }
+
+  // Otherwise, create a tuple value.
+  return SGF.B.createTuple(loc, payloadTy.getObjectType(), elts);
+}
+
 /// Emits SIL instructions to create an enum value. Attempts to avoid
 /// unnecessary copies by emitting the payload directly into the enum
 /// payload, or into the box in the case of an indirect payload.
 ManagedValue SILGenFunction::emitInjectEnum(SILLocation loc,
-                                            ArgumentSource &&payload,
+                            MutableArrayRef<ArgumentSource> payloads,
                                             SILType enumTy,
                                             EnumElementDecl *element,
                                             SGFContext C) {
   // Easy case -- no payload
-  if (!payload) {
+  if (!element->hasAssociatedValues()) {
+    assert(payloads.empty());
     if (enumTy.isLoadable(F) || !silConv.useLoweredAddresses()) {
       return emitManagedRValueWithCleanup(
           B.createEnum(loc, SILValue(), element, enumTy.getObjectType()));
@@ -6057,25 +6176,29 @@ ManagedValue SILGenFunction::emitInjectEnum(SILLocation loc,
                            });
   }
 
-  ManagedValue payloadMV;
+  AbstractionPattern origPayloadType = [&] {
-  AbstractionPattern origFormalType =
+    if (element == getASTContext().getOptionalSomeDecl()) {
-      (element == getASTContext().getOptionalSomeDecl()
+      assert(payloads.size() == 1);
-           ? AbstractionPattern(payload.getSubstRValueType())
+      auto substPayloadType = payloads[0].getSubstRValueType();
-           : SGM.M.Types.getAbstractionPattern(element));
+      return AbstractionPattern(substPayloadType);
-  auto &payloadTL = getTypeLowering(origFormalType,
+    } else {
-                                    payload.getSubstRValueType());
+      return SGM.M.Types.getAbstractionPattern(element);
+    }
+  }();
 
-  SILType loweredPayloadType = payloadTL.getLoweredType();
+  SILType payloadTy = enumTy.getEnumElementType(element, &F);
 
   // If the payload is indirect, emit it into a heap allocated box.
   //
   // To avoid copies, evaluate it directly into the box, being
   // careful to stage the cleanups so that if the expression
   // throws, we know to deallocate the uninitialized box.
+  ManagedValue boxMV;
   if (element->isIndirect() || element->getParentEnum()->isIndirect()) {
-    auto boxTy = SGM.M.Types.getBoxTypeForEnumElement(getTypeExpansionContext(),
+    CanSILBoxType boxType = payloadTy.castTo<SILBoxType>();
-                                                      enumTy, element);
+    assert(boxType->getLayout()->getFields().size() == 1);
-    auto *box = B.createAllocBox(loc, boxTy);
+    SILType boxPayloadTy = payloadTy.getSILBoxFieldType(&F, 0);
+    auto *box = B.createAllocBox(loc, boxType);
     auto *addr = B.createProjectBox(loc, box, 0);
 
     CleanupHandle initCleanup = enterDestroyCleanup(box);
@@ -6083,20 +6206,26 @@ ManagedValue SILGenFunction::emitInjectEnum(SILLocation loc,
     CleanupHandle uninitCleanup = enterDeallocBoxCleanup(box);
 
     BoxInitialization dest(box, addr, uninitCleanup, initCleanup);
-    std::move(payload).forwardInto(*this, origFormalType, &dest,
+    auto result =
-                                   payloadTL);
+      emitEnumElementPayloads(*this, loc, element, payloads, origPayloadType,
+                              boxPayloadTy, &dest);
+    assert(result.isInContext()); (void) result;
 
-    payloadMV = dest.getManagedBox();
+    boxMV = dest.getManagedBox();
-    loweredPayloadType = payloadMV.getType();
+    payloadTy = boxMV.getType();
   }
 
   // Loadable with payload
   if (enumTy.isLoadable(F) || !silConv.useLoweredAddresses()) {
-    if (!payloadMV) {
+    ManagedValue payloadMV;
+    if (boxMV) {
+      payloadMV = boxMV;
+    } else {
       // If the payload was indirect, we already evaluated it and
       // have a single value. Otherwise, evaluate the payload.
-      payloadMV = std::move(payload).getAsSingleValue(*this, origFormalType,
+      payloadMV = emitEnumElementPayloads(*this, loc, element, payloads,
-                                                      loweredPayloadType);
+                                          origPayloadType, payloadTy,
+                                          /*emit into*/ nullptr);
     }
 
     SILValue argValue = payloadMV.forward(*this);
@@ -6107,34 +6236,36 @@ ManagedValue SILGenFunction::emitInjectEnum(SILLocation loc,
 
   // Address-only with payload
   return B.bufferForExpr(
-      loc, enumTy, getTypeLowering(enumTy), C, [&](SILValue bufferAddr) {
+      loc, enumTy, getTypeLowering(enumTy), C, [&](SILValue enumAddr) {
-        SILValue resultData = B.createInitEnumDataAddr(
+        SILValue payloadAddr = B.createInitEnumDataAddr(
-            loc, bufferAddr, element, loweredPayloadType.getAddressType());
+            loc, enumAddr, element, payloadTy.getAddressType());
 
-        if (payloadMV) {
+        if (boxMV) {
           // If the payload was indirect, we already evaluated it and
           // have a single value. Store it into the result.
-          B.emitStoreValueOperation(loc, payloadMV.forward(*this), resultData,
+          B.emitStoreValueOperation(loc, boxMV.forward(*this), payloadAddr,
                                     StoreOwnershipQualifier::Init);
-        } else if (payloadTL.isLoadable()) {
+        } else if (payloadTy.isLoadable(F)) {
           // The payload of this specific enum case might be loadable
           // even if the overall enum is address-only.
-          payloadMV =
+          auto payloadMV = 
-            std::move(payload).getAsSingleValue(*this, origFormalType,
+            emitEnumElementPayloads(*this, loc, element, payloads,
-                                                loweredPayloadType);
+                                    origPayloadType, payloadTy,
-          B.emitStoreValueOperation(loc, payloadMV.forward(*this), resultData,
+                                    /*emit into*/ nullptr);
+          B.emitStoreValueOperation(loc, payloadMV.forward(*this), payloadAddr,
                                     StoreOwnershipQualifier::Init);
         } else {
           // The payload is address-only. Evaluate it directly into
           // the enum.
-
+          TemporaryInitialization dest(payloadAddr, CleanupHandle::invalid());
-          TemporaryInitialization dest(resultData, CleanupHandle::invalid());
+          auto result =
-          std::move(payload).forwardInto(*this, origFormalType, &dest,
+            emitEnumElementPayloads(*this, loc, element, payloads,
-                                         payloadTL);
+                                    origPayloadType, payloadTy, &dest);
+          assert(result.isInContext()); (void) result;
         }
 
         // The payload is initialized, now apply the tag.
-        B.createInjectEnumAddr(loc, bufferAddr, element);
+        B.createInjectEnumAddr(loc, enumAddr, element);
       });
 }
 

lib/SILGen/SILGenConstructor.cpp

@@ -898,12 +898,21 @@ void SILGenFunction::emitEnumConstructor(EnumElementDecl *element) {
   LoweredParamsInContextGenerator loweredParams(*this);
 
   // Emit the exploded constructor argument.
-  ArgumentSource payload;
+  SmallVector<ArgumentSource, 2> payloads;
   if (element->hasAssociatedValues()) {
-    auto eltArgTy = element->getArgumentInterfaceType()->getCanonicalType();
+    auto elementFnTy =
-    RValue arg = emitImplicitValueConstructorArg(*this, Loc, eltArgTy, element,
+      cast<AnyFunctionType>(
-                                                 loweredParams);
+        cast<AnyFunctionType>(element->getInterfaceType()->getCanonicalType())
-    payload = ArgumentSource(Loc, std::move(arg));
+          .getResult());
+    auto elementParams = elementFnTy.getParams();
+    payloads.reserve(elementParams.size());
+
+    for (auto param: elementParams) {
+      auto paramType = param.getParameterType();
+      RValue arg = emitImplicitValueConstructorArg(*this, Loc, paramType,
+                                                   element, loweredParams);
+      payloads.emplace_back(Loc, std::move(arg));
+    }
   }
 
   // Emit the metatype argument.
@@ -913,7 +922,7 @@ void SILGenFunction::emitEnumConstructor(EnumElementDecl *element) {
 
   // If possible, emit the enum directly into the indirect return.
   SGFContext C = (dest ? SGFContext(dest.get()) : SGFContext());
-  ManagedValue mv = emitInjectEnum(Loc, std::move(payload),
+  ManagedValue mv = emitInjectEnum(Loc, payloads,
                                    enumTI.getLoweredType(),
                                    element, C);
 

lib/SILGen/SILGenConvert.cpp

@@ -1026,6 +1026,34 @@ ManagedValue SILGenFunction::manageOpaqueValue(ManagedValue value,
   return value.copyUnmanaged(*this, loc);
 }
 
+ManagedValue SILGenFunction::emitAsOrig(SILLocation loc,
+                                        AbstractionPattern origType,
+                                        CanType substType,
+                                        SILType expectedTy,
+                                        SGFContext C,
+                                        ValueProducerRef produceValue) {
+  // If the lowered substituted type already matches the substitution,
+  // we can just emit directly.
+  if (getLoweredType(substType).getASTType() == expectedTy.getASTType()) {
+    auto result = produceValue(*this, loc, C);
+
+    // For convenience, force the result into the destination.
+    if (auto init = C.getEmitInto(); init && !result.isInContext()) {
+      result.forwardInto(*this, loc, init);
+      return ManagedValue::forInContext();
+    }
+    return result;
+  }
+
+  auto conversion =
+    Conversion::getSubstToOrig(origType, substType, expectedTy);
+  auto result = emitConvertedRValue(loc, conversion, C, produceValue);
+
+  // emitConvertedRValue always forces results into the context.
+  assert((C.getEmitInto() != nullptr) == result.isInContext());
+  return result;
+}
+
 ManagedValue SILGenFunction::emitConvertedRValue(Expr *E,
                                                  const Conversion &conversion,
                                                  SGFContext C) {

lib/SILGen/SILGenFunction.h

@@ -1300,7 +1300,7 @@ public:
   //===--------------------------------------------------------------------===//
 
   ManagedValue emitInjectEnum(SILLocation loc,
-                              ArgumentSource &&payload,
+                              MutableArrayRef<ArgumentSource> payload,
                               SILType enumTy,
                               EnumElementDecl *element,
                               SGFContext C);
@@ -1562,6 +1562,22 @@ public:
                                    SGFContext C,
                                    ValueProducerRef produceValue);
 
+  /// Call the produceValue function and convert the result to the given
+  /// original abstraction pattern.
+  ///
+  /// The SGFContext provided to the produceValue function includes the
+  /// conversion, if it's non-trivial, and thus permits it to be peepholed
+  /// and combined with other conversions.  This can result in substantially
+  /// more efficient code than just emitting the value and reabstracting
+  /// it afterwards.
+  ///
+  /// If the provided SGFContext includes an initialization, the result
+  /// will always be ManagedValue::forInContext().
+  ManagedValue emitAsOrig(SILLocation loc, AbstractionPattern origType,
+                          CanType substType, SILType expectedTy,
+                          SGFContext C,
+                          ValueProducerRef produceValue);
+
   /// Emit the given expression as an r-value that follows the
   /// abstraction patterns of the original type.
   ManagedValue emitRValueAsOrig(Expr *E, AbstractionPattern origPattern,

test/IRGen/enum_resilience.swift

@@ -135,22 +135,11 @@ public func constructResilientEnumPayload(_ s: Size) -> Medium {
 // CHECK-NEXT: [[VWT_ADDR:%.*]] = getelementptr inbounds ptr, ptr [[METADATA]], [[INT]] -1
 // CHECK-NEXT: [[VWT:%.*]] = load ptr, ptr [[VWT_ADDR]]
 
-// CHECK-NEXT: [[WITNESS_ADDR:%.*]] = getelementptr inbounds %swift.vwtable, ptr [[VWT]], i32 0, i32 8
-// CHECK-NEXT: [[WITNESS_FOR_SIZE:%size]] = load [[INT]], ptr [[WITNESS_ADDR]]
-// CHECK-NEXT: [[ALLOCA:%.*]] = alloca i8, {{.*}} [[WITNESS_FOR_SIZE]], align 16
-// CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i32|i64)}} -1, ptr [[ALLOCA]])
-
 // CHECK:      [[WITNESS_ADDR:%.*]] = getelementptr inbounds ptr, ptr [[VWT]], i32 2
 // CHECK-NEXT: [[WITNESS:%.*]] = load ptr, ptr [[WITNESS_ADDR]]
 // CHECK-arm64e-NEXT: ptrtoint ptr [[WITNESS_ADDR]] to i64
 // CHECK-arm64e-NEXT: call i64 @llvm.ptrauth.blend
-// CHECK-NEXT: call ptr [[WITNESS]](ptr noalias [[ALLOCA]], ptr noalias %1, ptr [[METADATA]])
+// CHECK-NEXT: call ptr [[WITNESS]](ptr noalias %0, ptr noalias %1, ptr [[METADATA]])
-
-// CHECK-NEXT: [[WITNESS_ADDR:%.*]] = getelementptr inbounds ptr, ptr [[VWT]], i32 4
-// CHECK-NEXT: [[WITNESS:%.*]] = load ptr, ptr [[WITNESS_ADDR]]
-// CHECK-arm64e-NEXT: ptrtoint ptr [[WITNESS_ADDR]] to i64
-// CHECK-arm64e-NEXT: call i64 @llvm.ptrauth.blend
-// CHECK-NEXT: call ptr [[WITNESS]](ptr noalias %0, ptr noalias [[ALLOCA]], ptr [[METADATA]])
 
 // CHECK-NEXT: [[TAG:%.*]] = load i32, ptr @"$s14resilient_enum6MediumO8PostcardyAC0A7_struct4SizeVcACmFWC"
 // CHECK-NEXT: [[T0:%.*]] = call swiftcc %swift.metadata_response @"$s14resilient_enum6MediumOMa"([[INT]] 0)
@@ -165,7 +154,6 @@ public func constructResilientEnumPayload(_ s: Size) -> Medium {
 // CHECK-arm64e-NEXT: ptrtoint ptr [[WITNESS_ADDR]] to i64
 // CHECK-arm64e-NEXT: call i64 @llvm.ptrauth.blend
 // CHECK-NEXT: call void [[WITNESS_FN]](ptr noalias %0, i32 [[TAG]], ptr [[METADATA2]])
-// CHECK-NEXT: call void @llvm.lifetime.end.p0({{(i32|i64)}} -1, ptr [[ALLOCA]])
 
 // CHECK-NEXT: ret void
 

test/SILGen/enum.swift

@@ -63,16 +63,13 @@ func AddressOnly_cases(_ s: S) {
   _ = AddressOnly.nought
 
   // CHECK-NEXT:  [[METATYPE:%.*]] = metatype $@thin AddressOnly.Type
-  // CHECK-NEXT:  [[P_BUF:%.*]] = alloc_stack $any P
-  // CHECK-NEXT:  [[PAYLOAD_ADDR:%.*]] = init_existential_addr [[P_BUF]]
-  // CHECK-NEXT:  store %0 to [trivial] [[PAYLOAD_ADDR]]
   // CHECK-NEXT:  [[MERE:%.*]] = alloc_stack $AddressOnly
   // CHECK-NEXT:  [[PAYLOAD:%.*]] = init_enum_data_addr [[MERE]]
-  // CHECK-NEXT:  copy_addr [take] [[P_BUF]] to [init] [[PAYLOAD]] : $*any P
+  // CHECK-NEXT:  [[PAYLOAD_ADDR:%.*]] = init_existential_addr [[PAYLOAD]]
+  // CHECK-NEXT:  store %0 to [trivial] [[PAYLOAD_ADDR]]
   // CHECK-NEXT:  inject_enum_addr [[MERE]]
   // CHECK-NEXT:  destroy_addr [[MERE]]
   // CHECK-NEXT:  dealloc_stack [[MERE]]
-  // CHECK-NEXT:  dealloc_stack [[P_BUF]] : $*any P
   _ = AddressOnly.mere(s)
 
   // Address-only enum vs loadable payload
@@ -105,15 +102,12 @@ func PolyOptionable_cases<T>(_ t: T) {
   _ = PolyOptionable<T>.nought
 
 // CHECK-NEXT:    [[METATYPE:%.*]] = metatype $@thin PolyOptionable<T>.Type
-// CHECK-NEXT:    [[T_BUF:%.*]] = alloc_stack $T
-// CHECK-NEXT:    copy_addr %0 to [init] [[T_BUF]]
 // CHECK-NEXT:    [[MERE:%.*]] = alloc_stack $PolyOptionable<T>
 // CHECK-NEXT:    [[PAYLOAD:%.*]] = init_enum_data_addr [[MERE]]
-// CHECK-NEXT:    copy_addr [take] [[T_BUF]] to [init] [[PAYLOAD]] : $*T
+// CHECK-NEXT:    copy_addr %0 to [init] [[PAYLOAD]] : $*T
 // CHECK-NEXT:    inject_enum_addr [[MERE]]
 // CHECK-NEXT:    destroy_addr [[MERE]]
 // CHECK-NEXT:    dealloc_stack [[MERE]]
-// CHECK-NEXT:    dealloc_stack [[T_BUF]] : $*T
   _ = PolyOptionable<T>.mere(t)
 
 // CHECK-NOT:    destroy_addr %0
@@ -265,3 +259,27 @@ enum rdar81817725 {
         }
     }
 }
+
+enum Indirected {
+  case a
+  case b
+  indirect case c(Int)
+}
+func throwingFunction() throws -> Int { return 0 }
+
+// CHECK-LABEL: sil hidden [ossa] @$s4enum29throwInIndirectConstructorArgAA10IndirectedOyKF
+// CHECK:         [[BOX:%.*]] = alloc_box ${ var Int }
+// CHECK-NEXT:    [[PAYLOAD_ADDR:%.*]] = project_box [[BOX]]
+// CHECK-NEXT:    // function_ref
+// CHECK-NEXT:    [[FN:%.*]] = function_ref
+// CHECK-NEXT:    try_apply [[FN]]()
+// CHECK:       bb1([[RESULT:%.*]] : $Int):
+// CHECK-NEXT:    store [[RESULT]] to [trivial] [[PAYLOAD_ADDR]]
+// CHECK-NEXT:    [[ENUM:%.*]] = enum $Indirected, #Indirected.c!enumelt, [[BOX]]
+// CHECK-NEXT:    return [[ENUM]] : $Indirected
+// CHECK:       bb2([[ERROR:%.*]] : @owned $any Error):
+// CHECK-NEXT:    dealloc_box [[BOX]]
+// CHECK-NEXT:    throw [[ERROR]]
+func throwInIndirectConstructorArg() throws -> Indirected {
+  return .c(try throwingFunction())
+}

test/SILGen/generic_closures.swift

@@ -132,8 +132,8 @@ class NestedGeneric<U> {
   // CHECK-LABEL: sil hidden [ossa] @$s16generic_closures13NestedGenericC20nested_reabstraction{{[_0-9a-zA-Z]*}}F
   //   CHECK:       [[REABSTRACT:%.*]] = function_ref @$sIeg_ytIegr_TR
   //   CHECK:       partial_apply [callee_guaranteed] [[REABSTRACT]]
-  func nested_reabstraction<T>(_ x: T) -> Optionable<() -> ()> {
+  func nested_reabstraction<T>(_ x: T, fn: @escaping () -> ()) -> Optionable<() -> ()> {
-    return .some({})
+    return .some(fn)
   }
 }
 

test/SILGen/indirect_enum.swift

@@ -16,24 +16,21 @@ func TreeA_cases<T>(_ t: T, l: TreeA<T>, r: TreeA<T>) {
   let _ = TreeA<T>.Nil
 
 // CHECK-NEXT:    [[METATYPE:%.*]] = metatype $@thin TreeA<T>.Type
-// CHECK-NEXT:    [[T_BUF:%.*]] = alloc_stack $T
-// CHECK-NEXT:    copy_addr [[ARG1]] to [init] [[T_BUF]] : $*T
 // CHECK-NEXT:    [[BOX:%.*]] = alloc_box $<τ_0_0> { var τ_0_0 } <T>
 // CHECK-NEXT:    [[PB:%.*]] = project_box [[BOX]]
-// CHECK-NEXT:    copy_addr [take] [[T_BUF]] to [init] [[PB]]
+// CHECK-NEXT:    copy_addr [[ARG1]] to [init] [[PB]]
 // CHECK-NEXT:    [[LEAF:%.*]] = enum $TreeA<T>, #TreeA.Leaf!enumelt, [[BOX]]
 // CHECK-NEXT:    destroy_value [[LEAF]]
-// CHECK-NEXT:    dealloc_stack [[T_BUF]] : $*T
   let _ = TreeA<T>.Leaf(t)
 
 // CHECK-NEXT:    [[METATYPE:%.*]] = metatype $@thin TreeA<T>.Type
-// CHECK-NEXT:    [[ARG2_COPY:%.*]] = copy_value [[ARG2]]
-// CHECK-NEXT:    [[ARG3_COPY:%.*]] = copy_value [[ARG3]]
 // CHECK-NEXT:    [[BOX:%.*]] = alloc_box $<τ_0_0> { var (left: TreeA<τ_0_0>, right: TreeA<τ_0_0>) } <T>
 // CHECK-NEXT:    [[PB:%.*]] = project_box [[BOX]]
 // CHECK-NEXT:    [[LEFT:%.*]] = tuple_element_addr [[PB]] : $*(left: TreeA<T>, right: TreeA<T>), 0
 // CHECK-NEXT:    [[RIGHT:%.*]] = tuple_element_addr [[PB]] : $*(left: TreeA<T>, right: TreeA<T>), 1
+// CHECK-NEXT:    [[ARG2_COPY:%.*]] = copy_value [[ARG2]]
 // CHECK-NEXT:    store [[ARG2_COPY]] to [init] [[LEFT]]
+// CHECK-NEXT:    [[ARG3_COPY:%.*]] = copy_value [[ARG3]]
 // CHECK-NEXT:    store [[ARG3_COPY]] to [init] [[RIGHT]]
 // CHECK-NEXT:    [[BRANCH:%.*]] = enum $TreeA<T>, #TreeA.Branch!enumelt, [[BOX]]
 // CHECK-NEXT:    destroy_value [[BRANCH]]
@@ -46,9 +43,9 @@ func TreeA_cases<T>(_ t: T, l: TreeA<T>, r: TreeA<T>) {
 func TreeA_reabstract(_ f: @escaping (Int) -> Int) {
 // CHECK: bb0([[ARG:%.*]] : @guaranteed $@callee_guaranteed (Int) -> Int):
 // CHECK:         [[METATYPE:%.*]] = metatype $@thin TreeA<(Int) -> Int>.Type
-// CHECK-NEXT:    [[ARG_COPY:%.*]] = copy_value [[ARG]]
 // CHECK-NEXT:    [[BOX:%.*]] = alloc_box $<τ_0_0> { var τ_0_0 } <(Int) -> Int>
 // CHECK-NEXT:    [[PB:%.*]] = project_box [[BOX]]
+// CHECK-NEXT:    [[ARG_COPY:%.*]] = copy_value [[ARG]]
 // CHECK:         [[THUNK:%.*]] = function_ref @$sS2iIegyd_S2iIegnr_TR
 // CHECK-NEXT:    [[FN:%.*]] = partial_apply [callee_guaranteed] [[THUNK]]([[ARG_COPY]])
 // CHECK-NEXT:    [[FNC:%.*]] = convert_function [[FN]]
@@ -77,36 +74,27 @@ func TreeB_cases<T>(_ t: T, l: TreeB<T>, r: TreeB<T>) {
   let _ = TreeB<T>.Nil
 
 // CHECK-NEXT:    [[METATYPE:%.*]] = metatype $@thin TreeB<T>.Type
-// CHECK-NEXT:    [[T_BUF:%.*]] = alloc_stack $T
-// CHECK-NEXT:    copy_addr %0 to [init] [[T_BUF]]
 // CHECK-NEXT:    [[LEAF:%.*]] = alloc_stack $TreeB<T>
 // CHECK-NEXT:    [[PAYLOAD:%.*]] = init_enum_data_addr [[LEAF]] : $*TreeB<T>, #TreeB.Leaf!enumelt
-// CHECK-NEXT:    copy_addr [take] [[T_BUF]] to [init] [[PAYLOAD]]
+// CHECK-NEXT:    copy_addr %0 to [init] [[PAYLOAD]]
 // CHECK-NEXT:    inject_enum_addr [[LEAF]] : $*TreeB<T>, #TreeB.Leaf!enumelt
 // CHECK-NEXT:    destroy_addr [[LEAF]]
 // CHECK-NEXT:    dealloc_stack [[LEAF]]
-// CHECK-NEXT:    dealloc_stack [[T_BUF]]
   let _ = TreeB<T>.Leaf(t)
 
 // CHECK-NEXT:    [[METATYPE:%.*]] = metatype $@thin TreeB<T>.Type
-// CHECK-NEXT:    [[ARG1_COPY:%.*]] = alloc_stack $TreeB<T>
-// CHECK-NEXT:    copy_addr %1 to [init] [[ARG1_COPY]] : $*TreeB<T>
-// CHECK-NEXT:    [[ARG2_COPY:%.*]] = alloc_stack $TreeB<T>
-// CHECK-NEXT:    copy_addr %2 to [init] [[ARG2_COPY]] : $*TreeB<T>
 // CHECK-NEXT:    [[BOX:%.*]] = alloc_box $<τ_0_0> { var (left: TreeB<τ_0_0>, right: TreeB<τ_0_0>) } <T>
 // CHECK-NEXT:    [[PB:%.*]] = project_box [[BOX]]
 // CHECK-NEXT:    [[LEFT:%.*]] = tuple_element_addr [[PB]]
 // CHECK-NEXT:    [[RIGHT:%.*]] = tuple_element_addr [[PB]]
-// CHECK-NEXT:    copy_addr [take] [[ARG1_COPY]] to [init] [[LEFT]] : $*TreeB<T>
+// CHECK-NEXT:    copy_addr %1 to [init] [[LEFT]] : $*TreeB<T>
-// CHECK-NEXT:    copy_addr [take] [[ARG2_COPY]] to [init] [[RIGHT]] : $*TreeB<T>
+// CHECK-NEXT:    copy_addr %2 to [init] [[RIGHT]] : $*TreeB<T>
 // CHECK-NEXT:    [[BRANCH:%.*]] = alloc_stack $TreeB<T>
 // CHECK-NEXT:    [[PAYLOAD:%.*]] = init_enum_data_addr [[BRANCH]]
 // CHECK-NEXT:    store [[BOX]] to [init] [[PAYLOAD]]
 // CHECK-NEXT:    inject_enum_addr [[BRANCH]] : $*TreeB<T>, #TreeB.Branch!enumelt
 // CHECK-NEXT:    destroy_addr [[BRANCH]]
 // CHECK-NEXT:    dealloc_stack [[BRANCH]]
-// CHECK-NEXT:    dealloc_stack [[ARG2_COPY]]
-// CHECK-NEXT:    dealloc_stack [[ARG1_COPY]]
   let _ = TreeB<T>.Branch(left: l, right: r)
 
 // CHECK:         return
@@ -126,13 +114,13 @@ func TreeInt_cases(_ t: Int, l: TreeInt, r: TreeInt) {
   let _ = TreeInt.Leaf(t)
 
 // CHECK-NEXT:    [[METATYPE:%.*]] = metatype $@thin TreeInt.Type
-// CHECK-NEXT:    [[ARG2_COPY:%.*]] = copy_value [[ARG2]] : $TreeInt
-// CHECK-NEXT:    [[ARG3_COPY:%.*]] = copy_value [[ARG3]] : $TreeInt
 // CHECK-NEXT:    [[BOX:%.*]] = alloc_box ${ var (left: TreeInt, right: TreeInt) }
 // CHECK-NEXT:    [[PB:%.*]] = project_box [[BOX]]
 // CHECK-NEXT:    [[LEFT:%.*]] = tuple_element_addr [[PB]]
 // CHECK-NEXT:    [[RIGHT:%.*]] = tuple_element_addr [[PB]]
+// CHECK-NEXT:    [[ARG2_COPY:%.*]] = copy_value [[ARG2]] : $TreeInt
 // CHECK-NEXT:    store [[ARG2_COPY]] to [init] [[LEFT]]
+// CHECK-NEXT:    [[ARG3_COPY:%.*]] = copy_value [[ARG3]] : $TreeInt
 // CHECK-NEXT:    store [[ARG3_COPY]] to [init] [[RIGHT]]
 // CHECK-NEXT:    [[BRANCH:%.*]] = enum $TreeInt, #TreeInt.Branch!enumelt, [[BOX]]
 // CHECK-NEXT:    destroy_value [[BRANCH]]

test/SILGen/variadic-generic-tuples.swift

@@ -399,3 +399,27 @@ func test() {
   let tuple = identityOnVariadicTuples((1, 2, 3))
   takesVariadicTuple(tuple: tuple)
 }
+
+func createTuple<each T>(including: repeat Stored<each T>, from: Int) -> (repeat each T) {
+  fatalError()
+}
+
+// rdar://121489308
+func testTupleExpansionInEnumConstructor<each T>(
+  from: repeat Stored<each T>,
+  to: @escaping (Result<(repeat each T), Error>) -> ()
+) {
+  _ = {
+    let tuple = createTuple(including: repeat each from,
+                            from: 42)
+    to(.success(tuple))
+  }
+}
+// CHECK-LABEL: sil {{.*}}@$s4main35testTupleExpansionInEnumConstructor4from2toyAA6StoredVyxGxQp_ys6ResultOyxxQp_ts5Error_pGctRvzlFyycfU_ :
+// CHECK:         [[VAR:%.*]] = alloc_stack [lexical] $(repeat each T), let, name "tuple"
+//   (a few moments later)
+// CHECK:         metatype $@thin Result<(repeat each T), any Error>.Type
+// CHECK:         [[RESULT_TEMP:%.*]] = alloc_stack $Result<(repeat each T), any Error>
+// CHECK-NEXT:    [[PAYLOAD_ADDR:%.*]] = init_enum_data_addr [[RESULT_TEMP]] : $*Result<(repeat each T), any Error>, #Result.success
+// CHECK-NEXT:    copy_addr [[VAR]] to [init] [[PAYLOAD_ADDR]] : $*(repeat each T)
+// CHECK-NEXT:    inject_enum_addr [[RESULT_TEMP]] : $*Result<(repeat each T), any Error>, #Result.success

test/SILOptimizer/access_marker_verify.swift

@@ -334,14 +334,9 @@ func testInitGenericEnum<T>(t: T) -> GenericEnum<T>? {
 // CHECK:   alloc_box $<τ_0_0> { var GenericEnum<τ_0_0> } <T>, var, name "self"
 // CHECK:   mark_uninitialized [delegatingself] %3 : $<τ_0_0> { var GenericEnum<τ_0_0> } <T>
 // CHECK:   [[PROJ:%.*]] = project_box
-// CHECK:   [[ADR1:%.*]] = alloc_stack $T
-// CHECK-NOT: begin_access
-// CHECK:   copy_addr %1 to [init] [[ADR1]] : $*T
 // CHECK:   [[STK:%.*]] = alloc_stack $GenericEnum<T>
 // CHECK:   [[ENUMDATAADDR:%.*]] = init_enum_data_addr [[STK]]
-// CHECK:   [[ACCESSENUM:%.*]] = begin_access [modify] [unsafe] [[ENUMDATAADDR]] : $*T
+// CHECK:   copy_addr %1 to [init] [[ENUMDATAADDR]] : $*T
-// CHECK:   copy_addr [take] [[ADR1]] to [init] [[ACCESSENUM]] : $*T
-// CHECK:   end_access [[ACCESSENUM]] : $*T
 // CHECK:   inject_enum_addr
 // CHECK:   [[ACCESS:%.*]] = begin_access [modify] [unknown] [[PROJ]]
 // CHECK:   copy_addr [take] %{{.*}} to [[ACCESS]] : $*GenericEnum<T>
@@ -362,9 +357,9 @@ func testIndirectEnum() -> IndirectEnum {
 }
 // CHECK-LABEL: sil hidden [ossa] @$s20access_marker_verify16testIndirectEnumAA0eF0OyF : $@convention(thin) () -> @owned IndirectEnum {
 // CHECK: bb0:
-// CHECK:   apply
 // CHECK:   alloc_box ${ var Int }
 // CHECK:   [[PROJ:%.*]] = project_box
+// CHECK:   apply
 // CHECK:   [[ACCESS:%.*]] = begin_access [modify] [unsafe] [[PROJ]]
 // CHECK:   store %{{.*}} to [trivial] [[ACCESS]] : $*Int
 // CHECK:   end_access