Use the first element of structs and tuples as a source

for extra inhabitants. For structs in particular, this eliminates a major source of abstraction penatlies. For example, an optional struct containing an object pointer is now represented the same way as an optional object pointer, which is critical for correctly importing CF types as Unmanaged<T>!. In time, we should generalize this to consider all elements as sources for extra inhabitants, as well as exploiting spare bits in the representation, but getting the single-element case right really provides the bulk of the benefit. This commit restores r17242 and r17243 with a fix to use value witnesses that actually forward the right type metadata down. We were already generating these value witnesses in the dependent struct VWT pattern, but I was being too clever and trying to use the underlying value witness directly. Swift SVN r17267
2025-12-14 20:36:38 +01:00 · 2014-05-02 20:17:14 +00:00
parent 0f8733867b
commit c57dac63ae
8 changed files with 304 additions and 85 deletions
--- a/include/swift/Runtime/Metadata.h
+++ b/include/swift/Runtime/Metadata.h
@@ -569,6 +569,15 @@ struct ValueWitnessTable {
  /// The number of extra inhabitants, that is, bit patterns that do not form
  /// valid values of the type, in this type's binary representation.
  unsigned getNumExtraInhabitants() const;
+
+  /// Assert that this value witness table is an extra-inhabitants
+  /// value witness table and return it as such.
+  ///
+  /// This has an awful name because it's supposed to be internal to
+  /// this file.  Code outside this file should use LLVM's cast/dyn_cast.
+  /// We don't want to use those here because we need to avoid accidentally
+  /// introducing ABI dependencies on LLVM structures.
+  const struct ExtraInhabitantsValueWitnessTable *_asXIVWT() const;
 };
  
 /// A value-witness table with extra inhabitants entry points.
@@ -588,15 +597,23 @@ struct ExtraInhabitantsValueWitnessTable : ValueWitnessTable {
                            value_witness_types::extraInhabitantFlags eif)
    : ValueWitnessTable(base), storeExtraInhabitant(sei),
      getExtraInhabitantIndex(geii), extraInhabitantFlags(eif) {}
+
+  static bool classof(const ValueWitnessTable *table) {
+    return table->flags.hasExtraInhabitants();
+  }
 };

+inline const ExtraInhabitantsValueWitnessTable *
+ValueWitnessTable::_asXIVWT() const {
+  assert(ExtraInhabitantsValueWitnessTable::classof(this));
+  return static_cast<const ExtraInhabitantsValueWitnessTable *>(this);
+}
+  
 inline unsigned ValueWitnessTable::getNumExtraInhabitants() const {
  // If the table does not have extra inhabitant witnesses, then there are zero.
  if (!flags.hasExtraInhabitants())
    return 0;
-  return static_cast<const ExtraInhabitantsValueWitnessTable &>(*this)
-    .extraInhabitantFlags
-    .getNumExtraInhabitants();
+  return this->_asXIVWT()->extraInhabitantFlags.getNumExtraInhabitants();
 }

 // Standard value-witness tables.
@@ -795,6 +812,13 @@ public:
  FOR_ALL_FUNCTION_VALUE_WITNESSES(FORWARD_WITNESS)
  #undef FORWARD_WITNESS

+  int vw_getExtraInhabitantIndex(const OpaqueValue *value) const  {
+    return getValueWitnesses()->_asXIVWT()->getExtraInhabitantIndex(value, this);
+  }
+  void vw_storeExtraInhabitant(OpaqueValue *value, int index) const {
+    getValueWitnesses()->_asXIVWT()->storeExtraInhabitant(value, index, this);
+  }
+  
  /// Get the nominal type descriptor if this metadata describes a nominal type,
  /// or return null if it does not.
  const NominalTypeDescriptor *getNominalTypeDescriptor() const;
--- a/lib/IRGen/FixedTypeInfo.h
+++ b/lib/IRGen/FixedTypeInfo.h
@@ -124,6 +124,17 @@ public:
    return getFixedExtraInhabitantCount(IGM) > 0;
  }

+  /// Given a payload value that might be an extra enhabitant, return
+  /// a value that can be compared for bitwise equality with an extra
+  /// inhabitant.
+  ///
+  /// This is necessary because we don't want to promise to initialize
+  /// every bit in the value when setting up an extra inhabitant.
+  virtual llvm::Value *maskFixedExtraInhabitant(IRGenFunction &IGF,
+                                                llvm::Value *payload) const {
+    return payload;
+  }
+
  /// Create a constant of the given bit width holding one of the extra
  /// inhabitants of the type.
  /// The index must be less than the value returned by
--- a/lib/IRGen/GenEnum.cpp
+++ b/lib/IRGen/GenEnum.cpp
@@ -379,7 +379,8 @@ namespace {
      }

      return getSingleton()->getExtraInhabitantIndex(IGF,
-                                             getSingletonAddress(IGF, src), T);
+                                             getSingletonAddress(IGF, src),
+                                             getSingletonType(IGF.IGM, T));
    }
    
    void storeExtraInhabitant(IRGenFunction &IGF,
@@ -390,7 +391,8 @@ namespace {
        return;
      }
      getSingleton()->storeExtraInhabitant(IGF, index,
-                                           getSingletonAddress(IGF, dest), T);
+                                           getSingletonAddress(IGF, dest),
+                                           getSingletonType(IGF.IGM, T));
    }
    
    unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
@@ -999,6 +1001,8 @@ namespace {
    
    CopyDestroyStrategy CopyDestroyKind;

+    unsigned NumExtraInhabitantTagValues = ~0U;
+    
  public:
    SinglePayloadEnumImplStrategy(IRGenModule &IGM,
                                  TypeInfoKind tik, unsigned NumElements,
@@ -1037,6 +1041,13 @@ namespace {
      // TODO: Same for single unknown-refcounted pointers.
    }

+    /// Return the number of tag values represented with extra
+    /// inhabitants in the payload.
+    unsigned getNumExtraInhabitantTagValues() const {
+      assert(NumExtraInhabitantTagValues != ~0U);
+      return NumExtraInhabitantTagValues;
+    }
+    
    /// The payload for a single-payload enum is always placed in front and
    /// will never have interleaved tag bits, so we can just bitcast the enum
    /// address to the payload type for either injection or projection of the
@@ -1137,8 +1148,7 @@ namespace {
      if (payloadTy)
        payload = value.claimNext();
      llvm::BasicBlock *payloadDest = blockForCase(getPayloadElement());
-      unsigned extraInhabitantCount
-        = getFixedPayloadTypeInfo().getFixedExtraInhabitantCount(IGF.IGM);
+      unsigned extraInhabitantCount = getNumExtraInhabitantTagValues();
      
      // If there are extra tag bits, switch over them first.
      SmallVector<llvm::BasicBlock*, 2> tagBitBlocks;
@@ -1193,7 +1203,9 @@ namespace {
        = getFixedPayloadTypeInfo().getFixedSize().getValueInBits();
      if (extraInhabitantCount > 0) {
        assert(payload && "extra inhabitants with empty payload?!");
-        auto *swi = IGF.Builder.CreateSwitch(payload, payloadDest);
+        auto *switchValue =
+          getFixedPayloadTypeInfo().maskFixedExtraInhabitant(IGF, payload);
+        auto *swi = IGF.Builder.CreateSwitch(switchValue, payloadDest);
        for (unsigned i = 0; i < extraInhabitantCount && elti != eltEnd; ++i) {
          auto v = getFixedPayloadTypeInfo().getFixedExtraInhabitantValue(
                                                       IGF.IGM, payloadBits, i);
@@ -1363,8 +1375,7 @@ namespace {
      // Non-payload cases use extra inhabitants, if any, or are discriminated
      // by setting the tag bits.
      unsigned tagIndex = getSimpleElementTagIndex(elt);
-      unsigned numExtraInhabitants
-        = getFixedPayloadTypeInfo().getFixedExtraInhabitantCount(IGM);
+      unsigned numExtraInhabitants = getNumExtraInhabitantTagValues();
      llvm::ConstantInt *payload = nullptr;
      unsigned extraTagValue;
      if (tagIndex < numExtraInhabitants) {
@@ -1465,11 +1476,15 @@ namespace {
      
      // If we used extra inhabitants to represent empty case discriminators,
      // weed them out.
-      unsigned numExtraInhabitants
-        = getFixedPayloadTypeInfo().getFixedExtraInhabitantCount(IGF.IGM);
+      unsigned numExtraInhabitants = getNumExtraInhabitantTagValues();
      if (numExtraInhabitants > 0) {
+        unsigned bitWidth =
+          getFixedPayloadTypeInfo().getFixedSize().getValueInBits();
+
        auto *payloadBB = llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
-        auto *swi = IGF.Builder.CreateSwitch(payload, payloadBB);
+        auto *switchValue =
+          getFixedPayloadTypeInfo().maskFixedExtraInhabitant(IGF, payload);
+        auto *swi = IGF.Builder.CreateSwitch(switchValue, payloadBB);
        
        auto elements = getPayloadElement()->getParentEnum()->getAllElements();
        unsigned inhabitant = 0;
@@ -1482,9 +1497,7 @@ namespace {
              break;
          }
          auto xi = getFixedPayloadTypeInfo().getFixedExtraInhabitantValue(
-                      IGF.IGM,
-                      getFixedPayloadTypeInfo().getFixedSize().getValueInBits(),
-                      inhabitant);
+                      IGF.IGM, bitWidth, inhabitant);
          swi->addCase(xi, falseBB);
        }
        
@@ -2001,15 +2014,8 @@ namespace {
    }
    
    unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
-      unsigned payloadXI
-        = getFixedPayloadTypeInfo().getFixedExtraInhabitantCount(IGM);
-      
-      unsigned numEmptyCases = ElementsWithNoPayload.size();
-      
-      if (payloadXI <= numEmptyCases)
-        return 0;
-      
-      return payloadXI - numEmptyCases;
+      return getFixedPayloadTypeInfo().getFixedExtraInhabitantCount(IGM)
+               - getNumExtraInhabitantTagValues();
    }
    
    llvm::ConstantInt *
@@ -2018,7 +2024,7 @@ namespace {
                                 unsigned index) const override {
      return getFixedPayloadTypeInfo()
        .getFixedExtraInhabitantValue(IGM, bits,
-                                      index + ElementsWithNoPayload.size());
+                                      index + getNumExtraInhabitantTagValues());
    }
    
    llvm::Value *
@@ -3550,9 +3556,9 @@ namespace {
    // Determine how many tag bits we need. Given N extra inhabitants, we
    // represent the first N tags using those inhabitants. For additional tags,
    // we use discriminator bit(s) to inhabit the full bit size of the payload.
-    unsigned tagsWithoutInhabitants = numTags <= fixedExtraInhabitants
-      ? 0 : numTags - fixedExtraInhabitants;
+    NumExtraInhabitantTagValues = std::min(numTags, fixedExtraInhabitants);
    
+    unsigned tagsWithoutInhabitants = numTags - NumExtraInhabitantTagValues;    
    if (tagsWithoutInhabitants == 0) {
      ExtraTagBitCount = 0;
      NumExtraTagValues = 0;
--- a/lib/IRGen/GenStruct.cpp
+++ b/lib/IRGen/GenStruct.cpp
@@ -123,6 +123,62 @@ namespace {
      }
      return nullptr;
    }
+
+    // For now, just use extra inhabitants from the first field.
+    // FIXME: generalize
+    bool mayHaveExtraInhabitants(IRGenModule &IGM) const override {
+      if (asImpl().getFields().empty()) return false;
+      return asImpl().getFields()[0].getTypeInfo().mayHaveExtraInhabitants(IGM);
+    }
+
+    // This is dead code in NonFixedStructTypeInfo.
+    unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const {
+      if (asImpl().getFields().empty()) return 0;
+      auto &fieldTI = cast<FixedTypeInfo>(asImpl().getFields()[0].getTypeInfo());
+      return fieldTI.getFixedExtraInhabitantCount(IGM);
+    }
+
+    // This is dead code in NonFixedStructTypeInfo.
+    llvm::ConstantInt *getFixedExtraInhabitantValue(IRGenModule &IGM,
+                                                    unsigned bits,
+                                                    unsigned index) const {
+      auto &fieldTI = cast<FixedTypeInfo>(asImpl().getFields()[0].getTypeInfo());
+      return fieldTI.getFixedExtraInhabitantValue(IGM, bits, index);
+    }
+
+    // This is dead code in NonFixedStructTypeInfo.
+    llvm::Value *maskFixedExtraInhabitant(IRGenFunction &IGF,
+                                          llvm::Value *structValue) const {
+      // Truncate down to the width of the field, mask it recursively,
+      // and then zext back out to the payload size.
+      auto &fieldTI = cast<FixedTypeInfo>(asImpl().getFields()[0].getTypeInfo());
+      unsigned fieldWidth = fieldTI.getFixedSize().getValueInBits();
+      auto fieldTy = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), fieldWidth);
+      auto fieldValue = IGF.Builder.CreateTrunc(structValue, fieldTy);
+      fieldValue = fieldTI.maskFixedExtraInhabitant(IGF, fieldValue);
+      return IGF.Builder.CreateZExt(fieldValue, structValue->getType());
+    }
+
+    llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF,
+                                         Address structAddr,
+                                         CanType structType) const override {
+      auto &field = asImpl().getFields()[0];
+      Address fieldAddr =
+        asImpl().projectFieldAddress(IGF, structAddr, structType, field.Field);
+      return field.getTypeInfo().getExtraInhabitantIndex(IGF, fieldAddr,
+                                          field.getType(IGF.IGM, structType));
+    }
+
+    void storeExtraInhabitant(IRGenFunction &IGF,
+                              llvm::Value *index,
+                              Address structAddr,
+                              CanType structType) const override {
+      auto &field = asImpl().getFields()[0];
+      Address fieldAddr =
+        asImpl().projectFieldAddress(IGF, structAddr, structType, field.Field);
+      field.getTypeInfo().storeExtraInhabitant(IGF, index, fieldAddr,
+                                          field.getType(IGF.IGM, structType));
+    }
  };


@@ -149,14 +205,6 @@ namespace {
    Nothing_t getNonFixedOffsets(IRGenFunction &IGF) const {
      return Nothing;
    }
-        
-    // FIXME: Suppress use of extra inhabitants for single-payload enum layout
-    // until we're ready to handle the runtime logic for exporting extra
-    // inhabitants through generic structs.
-    bool mayHaveExtraInhabitants(IRGenModule&) const override { return false; }
-    unsigned getFixedExtraInhabitantCount(IRGenModule&) const override {
-      return 0;
-    }
  };

  /// A type implementation for non-loadable but fixed-size struct types.
@@ -176,14 +224,6 @@ namespace {
      return Nothing;
    }
    Nothing_t getNonFixedOffsets(IRGenFunction &IGF) const { return Nothing; }
-                                                     
-    // FIXME: Suppress use of extra inhabitants for single-payload enum layout
-    // until we're ready to handle the runtime logic for exporting extra
-    // inhabitants through generic structs.
-    bool mayHaveExtraInhabitants(IRGenModule&) const override { return false; }
-    unsigned getFixedExtraInhabitantCount(IRGenModule&) const override {
-      return 0;
-    }
  };
  
  /// Find the beginning of the field offset vector in a struct's metadata.
--- a/lib/IRGen/GenTuple.cpp
+++ b/lib/IRGen/GenTuple.cpp
@@ -122,6 +122,62 @@ namespace {
                              Address src, CanType T) const override {
      llvm_unreachable("unexploded tuple as argument?");
    }
+
+    // For now, just use extra inhabitants from the first element.
+    // FIXME: generalize
+    bool mayHaveExtraInhabitants(IRGenModule &IGM) const override {
+      if (asImpl().getFields().empty()) return false;
+      return asImpl().getFields()[0].getTypeInfo().mayHaveExtraInhabitants(IGM);
+    }
+
+    // This is dead code in NonFixedTupleTypeInfo.
+    unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const {
+      if (asImpl().getFields().empty()) return 0;
+      auto &eltTI = cast<FixedTypeInfo>(asImpl().getFields()[0].getTypeInfo());
+      return eltTI.getFixedExtraInhabitantCount(IGM);
+    }
+
+    // This is dead code in NonFixedTupleTypeInfo.
+    llvm::ConstantInt *getFixedExtraInhabitantValue(IRGenModule &IGM,
+                                                    unsigned bits,
+                                                    unsigned index) const {
+      auto &eltTI = cast<FixedTypeInfo>(asImpl().getFields()[0].getTypeInfo());
+      return eltTI.getFixedExtraInhabitantValue(IGM, bits, index);
+    }
+
+    // This is dead code in NonFixedTupleTypeInfo.
+    llvm::Value *maskFixedExtraInhabitant(IRGenFunction &IGF,
+                                          llvm::Value *tupleValue) const {
+      // Truncate down to the width of the element, mask it recursively,
+      // and then zext back out to the payload size.
+      auto &eltTI = cast<FixedTypeInfo>(asImpl().getFields()[0].getTypeInfo());
+      unsigned eltWidth = eltTI.getFixedSize().getValueInBits();
+      auto eltTy = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), eltWidth);
+      auto eltValue = IGF.Builder.CreateTrunc(tupleValue, eltTy);      
+      eltValue = eltTI.maskFixedExtraInhabitant(IGF, eltValue);
+      return IGF.Builder.CreateZExt(eltValue, tupleValue->getType());
+    }
+
+    llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF,
+                                         Address tupleAddr,
+                                         CanType tupleType) const override {
+      Address eltAddr =
+        asImpl().projectElementAddress(IGF, tupleAddr, tupleType, 0);
+      auto &elt = asImpl().getFields()[0];
+      return elt.getTypeInfo().getExtraInhabitantIndex(IGF, eltAddr,
+                                               elt.getType(IGF.IGM, tupleType));
+    }
+  
+    void storeExtraInhabitant(IRGenFunction &IGF,
+                              llvm::Value *index,
+                              Address tupleAddr,
+                              CanType tupleType) const override {
+      Address eltAddr =
+        asImpl().projectElementAddress(IGF, tupleAddr, tupleType, 0);
+      auto &elt = asImpl().getFields()[0];
+      elt.getTypeInfo().storeExtraInhabitant(IGF, index, eltAddr,
+                                             elt.getType(IGF.IGM, tupleType));
+    }
  };

  /// Type implementation for loadable tuples.
@@ -139,14 +195,6 @@ namespace {
    Nothing_t getNonFixedOffsets(IRGenFunction &IGF) const { return Nothing; }
    Nothing_t getNonFixedOffsets(IRGenFunction &IGF,
                                 CanType T) const { return Nothing; }
-    
-    // FIXME: Suppress use of extra inhabitants for single-payload enum layout
-    // until we're ready to handle the runtime logic for exporting extra
-    // inhabitants through tuple metadata.
-    bool mayHaveExtraInhabitants(IRGenModule&) const override { return false; }
-    unsigned getFixedExtraInhabitantCount(IRGenModule&) const override {
-      return 0;
-    }
  };

  /// Type implementation for fixed-size but non-loadable tuples.
@@ -167,14 +215,6 @@ namespace {
    Nothing_t getNonFixedOffsets(IRGenFunction &IGF) const { return Nothing; }
    Nothing_t getNonFixedOffsets(IRGenFunction &IGF,
                                 CanType T) const { return Nothing; }
-    
-    // FIXME: Suppress use of extra inhabitants for single-payload enum layout
-    // until we're ready to handle the runtime logic for exporting extra
-    // inhabitants through tuple metadata.
-    bool mayHaveExtraInhabitants(IRGenModule&) const override { return false; }
-    unsigned getFixedExtraInhabitantCount(IRGenModule&) const override {
-      return 0;
-    }
  };

  /// An accessor for the non-fixed offsets for a tuple type.
--- a/stdlib/runtime/Metadata.cpp
+++ b/stdlib/runtime/Metadata.cpp
@@ -15,6 +15,7 @@
 //===----------------------------------------------------------------------===//

 #include "llvm/Support/MathExtras.h"
+#include "swift/Basic/LLVM.h"
 #include "swift/Basic/Range.h"
 #include "swift/Runtime/HeapObject.h"
 #include "swift/Runtime/Metadata.h"
@@ -673,7 +674,9 @@ swift::swift_getFunctionTypeMetadata(const Metadata *argMetadata,
 namespace {
  class TupleCacheEntry : public CacheEntry<TupleCacheEntry> {
  public:
-    ValueWitnessTable Witnesses;
+    // NOTE: if you change the layout of this type, you'll also need
+    // to update tuple_getValueWitnesses().
+    ExtraInhabitantsValueWitnessTable Witnesses;
    FullMetadata<TupleTypeMetadata> Metadata;

    TupleCacheEntry(size_t numArguments) : CacheEntry(numArguments) {
@@ -695,7 +698,7 @@ static MetadataCache<TupleCacheEntry> TupleTypes;
 /// Given a metatype pointer, produce the value-witness table for it.
 /// This is equivalent to metatype->ValueWitnesses but more efficient.
 static const ValueWitnessTable *tuple_getValueWitnesses(const Metadata *metatype) {
-  return ((const ValueWitnessTable*) asFullMetadata(metatype)) - 1;
+  return ((const ExtraInhabitantsValueWitnessTable*) asFullMetadata(metatype)) - 1;
 }

 /// Generic tuple value witness for 'projectBuffer'.
@@ -1016,6 +1019,29 @@ static const Metadata *tuple_typeOf(OpaqueValue *obj,
  return metatype;
 }

+static void tuple_storeExtraInhabitant(OpaqueValue *tuple,
+                                       int index,
+                                       const Metadata *_metatype) {
+  auto &metatype = *(const TupleTypeMetadata*) _metatype;
+  auto &eltInfo = metatype.getElements()[0];
+
+  assert(eltInfo.Offset == 0);
+  OpaqueValue *elt = tuple;
+
+  eltInfo.Type->vw_storeExtraInhabitant(elt, index);
+}
+
+static int tuple_getExtraInhabitantIndex(const OpaqueValue *tuple,
+                                         const Metadata *_metatype) {
+  auto &metatype = *(const TupleTypeMetadata*) _metatype;
+  auto &eltInfo = metatype.getElements()[0];
+
+  assert(eltInfo.Offset == 0);
+  const OpaqueValue *elt = tuple;
+
+  return eltInfo.Type->vw_getExtraInhabitantIndex(elt);
+}
+
 /// Various standard witness table for tuples.
 static const ValueWitnessTable tuple_witnesses_pod_inline = {
 #define TUPLE_WITNESS(NAME) &tuple_##NAME<true, true>,
@@ -1199,6 +1225,16 @@ swift::swift_getTupleTypeMetadata(size_t numElements,
  FOR_ALL_FUNCTION_VALUE_WITNESSES(ASSIGN_TUPLE_WITNESS)
 #undef ASSIGN_TUPLE_WITNESS

+  // We have extra inhabitants if the first element does.
+  // FIXME: generalize this.
+  if (auto firstEltEIVWT = dyn_cast<ExtraInhabitantsValueWitnessTable>(
+                                          elements[0]->getValueWitnesses())) {
+    witnesses->flags = witnesses->flags.withExtraInhabitants(true);
+    witnesses->extraInhabitantFlags = firstEltEIVWT->extraInhabitantFlags;
+    witnesses->storeExtraInhabitant = tuple_storeExtraInhabitant;
+    witnesses->getExtraInhabitantIndex = tuple_getExtraInhabitantIndex;
+  }
+
  auto finalMetadata = TupleTypes.add(entry)->getData();
 #if SWIFT_DEBUG_RUNTIME
  printf(" -> %p\n", finalMetadata);
@@ -1240,6 +1276,19 @@ void swift::swift_initStructMetadata_UniversalStrategy(size_t numFields,
  vwtable->size = layout.size;
  vwtable->flags = layout.flags;
  vwtable->stride = layout.stride;
+
+  // We have extra inhabitants if the first element does.
+  // FIXME: generalize this.
+  if (auto firstFieldVWT = dyn_cast<ExtraInhabitantsValueWitnessTable>(
+                                        fieldTypes[0]->getValueWitnesses())) {
+    vwtable->flags = vwtable->flags.withExtraInhabitants(true);
+    auto xiVWT = cast<ExtraInhabitantsValueWitnessTable>(vwtable);
+    xiVWT->extraInhabitantFlags = firstFieldVWT->extraInhabitantFlags;
+
+    // The compiler should already have initialized these.
+    assert(xiVWT->storeExtraInhabitant);
+    assert(xiVWT->getExtraInhabitantIndex);
+  }
 }

 /*** Classes ***************************************************************/
--- a/test/IRGen/enum.sil
+++ b/test/IRGen/enum.sil
@@ -598,7 +598,7 @@ entry:
 // -- Test packing and unpacking aggregates.

 enum AggregateSinglePayload {
-  case x(Builtin.Int21, Builtin.Int64)
+  case x(Builtin.Int64, Builtin.Int64)
  case y
  case z
 }
@@ -608,10 +608,10 @@ sil @aggregate_single_payload_unpack : $@thin (AggregateSinglePayload) -> () {
 entry(%u : $AggregateSinglePayload):
  switch_enum %u : $AggregateSinglePayload, case #AggregateSinglePayload.x!enumelt.1: x_dest, default end

-// CHECK:   {{%.*}} = trunc i128 %0 to i21
+// CHECK:   {{%.*}} = trunc i128 %0 to i64
 // CHECK:   [[SHIFT_1:%.*]] = lshr i128 %0, 64
 // CHECK:   {{%.*}} = trunc i128 [[SHIFT_1]] to i64
-x_dest(%v : $(Builtin.Int21, Builtin.Int64)):
+x_dest(%v : $(Builtin.Int64, Builtin.Int64)):
  br end

 end:
@@ -619,22 +619,22 @@ end:
  return %x : $()
 }

-// CHECK-LABEL: define { i21, i64 } @aggregate_single_payload_unsafe_unpack(i128, i1) {
-// CHECK:  %2 = trunc i128 %0 to i21
+// CHECK-LABEL: define { i64, i64 } @aggregate_single_payload_unsafe_unpack(i128, i1) {
+// CHECK:  %2 = trunc i128 %0 to i64
 // CHECK:  %3 = lshr i128 %0, 64
 // CHECK:  %4 = trunc i128 %3 to i64
-// CHECK:  %5 = insertvalue { i21, i64 } undef, i21 %2, 0
-// CHECK:  %6 = insertvalue { i21, i64 } %5, i64 %4, 1
-// CHECK:  ret { i21, i64 } %6
-sil @aggregate_single_payload_unsafe_unpack : $@thin (AggregateSinglePayload) -> (Builtin.Int21, Builtin.Int64) {
+// CHECK:  %5 = insertvalue { i64, i64 } undef, i64 %2, 0
+// CHECK:  %6 = insertvalue { i64, i64 } %5, i64 %4, 1
+// CHECK:  ret { i64, i64 } %6
+sil @aggregate_single_payload_unsafe_unpack : $@thin (AggregateSinglePayload) -> (Builtin.Int64, Builtin.Int64) {
 entry(%u : $AggregateSinglePayload):
  %x = unchecked_enum_data %u : $AggregateSinglePayload, #AggregateSinglePayload.x!enumelt.1
-  return %x : $(Builtin.Int21, Builtin.Int64)
+  return %x : $(Builtin.Int64, Builtin.Int64)
 }

-// CHECK-LABEL: define { i128, i1 } @aggregate_single_payload_inject(i21, i64) {
+// CHECK-LABEL: define { i128, i1 } @aggregate_single_payload_inject(i64, i64) {
 // CHECK: entry:
-// CHECK:   [[ZEXT_0:%.*]] = zext i21 %0 to i128
+// CHECK:   [[ZEXT_0:%.*]] = zext i64 %0 to i128
 // CHECK:   [[ZEXT_1:%.*]] = zext i64 %1 to i128
 // CHECK:   [[SHIFT_1:%.*]] = shl i128 [[ZEXT_1]], 64
 // CHECK:   [[PAYLOAD:%.*]] = or i128 [[ZEXT_0]], [[SHIFT_1]]
@@ -642,10 +642,10 @@ entry(%u : $AggregateSinglePayload):
 // CHECK:   [[RES:%.*]] = insertvalue { i128, i1 } [[RES_0]], i1 false, 1
 // CHECK:   ret { i128, i1 } [[RES]]
 // CHECK: }
-sil @aggregate_single_payload_inject : $(Builtin.Int21, Builtin.Int64) -> AggregateSinglePayload {
-entry(%0 : $Builtin.Int21, %1 : $Builtin.Int64):
-  %t = tuple (%0 : $Builtin.Int21, %1 : $Builtin.Int64)
-  %u = enum $AggregateSinglePayload, #AggregateSinglePayload.x!enumelt.1, %t : $(Builtin.Int21, Builtin.Int64)
+sil @aggregate_single_payload_inject : $(Builtin.Int64, Builtin.Int64) -> AggregateSinglePayload {
+entry(%0 : $Builtin.Int64, %1 : $Builtin.Int64):
+  %t = tuple (%0 : $Builtin.Int64, %1 : $Builtin.Int64)
+  %u = enum $AggregateSinglePayload, #AggregateSinglePayload.x!enumelt.1, %t : $(Builtin.Int64, Builtin.Int64)
  return %u : $AggregateSinglePayload
 }

@@ -659,7 +659,7 @@ enum AggregateSinglePayload2 {
  case z
 }

-// CHECK: define void @aggregate_single_payload_unpack_2(i256, i1) {
+// CHECK: define void @aggregate_single_payload_unpack_2(i256) {
 sil @aggregate_single_payload_unpack_2 : $@thin (AggregateSinglePayload2) -> () {
 entry(%u : $AggregateSinglePayload2):
  switch_enum %u : $AggregateSinglePayload2, case #AggregateSinglePayload2.x!enumelt.1: x_dest, default end
@@ -679,7 +679,7 @@ end:
  return %x : $()
 }

-// CHECK: define { i256, i1 } @aggregate_single_payload_2_inject(i21, i64, i64, i64) {
+// CHECK: define i256 @aggregate_single_payload_2_inject(i21, i64, i64, i64) {
 // CHECK: entry:
 // CHECK:   [[ZEXT_0:%.*]] = zext i21 %0 to i256
 // CHECK:   [[ZEXT_1:%.*]] = zext i64 %1 to i256
@@ -691,9 +691,7 @@ end:
 // CHECK:   [[SHIFT_2_TO:%.*]] = shl i256 [[ZEXT_2_TO]], 192
 // CHECK:   [[PAYLOAD_2:%.*]] = or i256 [[SHIFT_2_FROM]], [[SHIFT_2_TO]]
 // CHECK:   [[PAYLOAD:%.*]] = or i256 [[PAYLOAD_0_1]], [[PAYLOAD_2]]
-// CHECK:   [[RES_0:%.*]] = insertvalue { i256, i1 } undef, i256 [[PAYLOAD]], 0
-// CHECK:   [[RES:%.*]] = insertvalue { i256, i1 } [[RES_0]], i1 false, 1
-// CHECK:   ret { i256, i1 } [[RES]]
+// CHECK:   ret i256 [[PAYLOAD]]
 // CHECK: }
 sil @aggregate_single_payload_2_inject : $(CharLike, IntLike, RangeLike) -> AggregateSinglePayload2 {
 entry(%0 : $CharLike, %1 : $IntLike, %2 : $RangeLike):
@@ -702,6 +700,53 @@ entry(%0 : $CharLike, %1 : $IntLike, %2 : $RangeLike):
  return %u : $AggregateSinglePayload2
 }

+enum AggregateSinglePayload3 {
+  case x(Builtin.Int21, Builtin.Int64)
+  case y
+  case z
+}
+
+// CHECK-LABEL: define void @aggregate_single_payload_unpack_3(i128) {
+sil @aggregate_single_payload_unpack_3 : $@thin (AggregateSinglePayload3) -> () {
+entry(%u : $AggregateSinglePayload3):
+  switch_enum %u : $AggregateSinglePayload3, case #AggregateSinglePayload3.x!enumelt.1: x_dest, default end
+
+// CHECK:   {{%.*}} = trunc i128 %0 to i21
+// CHECK:   [[SHIFT_1:%.*]] = lshr i128 %0, 64
+// CHECK:   {{%.*}} = trunc i128 [[SHIFT_1]] to i64
+x_dest(%v : $(Builtin.Int21, Builtin.Int64)):
+  br end
+
+end:
+  %x = tuple ()
+  return %x : $()
+}
+
+// CHECK-LABEL: define i128 @aggregate_single_payload_inject_x3(i21, i64) {
+// CHECK: entry:
+// CHECK:   [[ZEXT_0:%.*]] = zext i21 %0 to i128
+// CHECK:   [[ZEXT_1:%.*]] = zext i64 %1 to i128
+// CHECK:   [[SHIFT_1:%.*]] = shl i128 [[ZEXT_1]], 64
+// CHECK:   [[PAYLOAD:%.*]] = or i128 [[ZEXT_0]], [[SHIFT_1]]
+// CHECK:   ret i128 [[PAYLOAD]]
+// CHECK: }
+sil @aggregate_single_payload_inject_x3 : $(Builtin.Int21, Builtin.Int64) -> AggregateSinglePayload3 {
+entry(%0 : $Builtin.Int21, %1 : $Builtin.Int64):
+  %t = tuple (%0 : $Builtin.Int21, %1 : $Builtin.Int64)
+  %u = enum $AggregateSinglePayload3, #AggregateSinglePayload3.x!enumelt.1, %t : $(Builtin.Int21, Builtin.Int64)
+  return %u : $AggregateSinglePayload3
+}
+
+// CHECK-LABEL: define i128 @aggregate_single_payload_inject_y3() {
+// CHECK: entry:
+// CHECK:   ret i128 2097152
+// CHECK: }
+//   2097152 == 0x200000
+sil @aggregate_single_payload_inject_y3 : $() -> AggregateSinglePayload3 {
+entry:
+  %u = enum $AggregateSinglePayload3, #AggregateSinglePayload3.y!enumelt
+  return %u : $AggregateSinglePayload3
+}

 enum SinglePayloadSpareBit {
  case x(Builtin.Int63)
--- a/test/IRGen/generic_structs.sil
+++ b/test/IRGen/generic_structs.sil
@@ -26,7 +26,7 @@ import Builtin
 // CHECK: @_TMPdV15generic_structs13SingleDynamic = global {{[{].*\* [}]}} {
 // -- template header
 // CHECK:   void (i8*, i8*)* [[SINGLE_DYNAMIC_FILL_METADATA:@fill_generic_metadata[0-9]*]],
-// CHECK:   i32 216, i16 1, i16 8, [8 x i8*] zeroinitializer,
+// CHECK:   i32 240, i16 1, i16 8, [8 x i8*] zeroinitializer,
 // -- placeholder for vwtable pointer
 // CHECK:   i8* null,
 // -- address point
@@ -37,7 +37,11 @@ import Builtin
 // CHECK:   i8* bitcast (void ([24 x i8]*, %swift.type*)* @_TwXXV15generic_structs13SingleDynamic to i8*),
 // -- ...
 // -- placeholder for size, flags, stride
-// CHECK:   i8* null, i8* null, i8* null }
+// CHECK:   i8* null, i8* null, i8* null
+// -- extra inhabitants
+// CHECK:   i8* bitcast (void (%swift.opaque*, i32, %swift.type*)* @_TwxsV15generic_structs13SingleDynamic to i8*),
+// CHECK:   i8* bitcast (i32 (%swift.opaque*, %swift.type*)* @_TwxgV15generic_structs13SingleDynamic to i8*),
+// CHECK:   i8* null }
 // CHECK-NOT: @_TWVV15generic_structs13SingleDynamic

 // -- Nominal type descriptor for generic struct with protocol requirements