//===--- GenEnum.cpp - Swift IR Generation For 'enum' Types -------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
//  This file implements IR generation for algebraic data types (ADTs,
//  or 'enum' types) in Swift.  This includes creating the IR type as
//  well as emitting the basic access operations.
//
//  The current scheme is that all such types with are represented
//  with an initial word indicating the variant, followed by an enum
//  of all the possibilities.  This is obviously completely acceptable
//  to everyone and will not benefit from further refinement.
//
//  As a completely unimportant premature optimization, we do emit
//  types with only a single variant as simple structs wrapping that
//  variant.
//
//===----------------------------------------------------------------------===//

#include "swift/AST/Types.h"
#include "swift/AST/Decl.h"
#include "swift/Basic/Fallthrough.h"
#include "swift/Basic/Optional.h"
#include "swift/IRGen/Options.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalVariable.h"

// @@@@
#include "llvm/Support/raw_ostream.h"

#include "IRGenModule.h"
#include "LoadableTypeInfo.h"
#include "NonFixedTypeInfo.h"
#include "GenMeta.h"
#include "GenProto.h"
#include "GenType.h"
#include "GenEnum.h"
#include "IRGenDebugInfo.h"
#include "ScalarTypeInfo.h"

using namespace swift;
using namespace irgen;

namespace {

/// An implementation strategy for an enum, which handles how the enum is
/// laid out and how to construct and destructure values inside the enum.
class EnumImplStrategy {
public:
  struct Element {
    EnumElementDecl *decl;
    const TypeInfo *ti;
  };
  
  enum TypeInfoKind {
    Opaque,   ///< The enum has a NonFixedTypeInfo.
    Fixed,    ///< The enum has a FixedTypeInfo.
    Loadable, ///< The enum has a LoadableTypeInfo.
  };
  
protected:
  std::vector<Element> ElementsWithPayload;
  std::vector<Element> ElementsWithRecursivePayload;
  std::vector<Element> ElementsWithNoPayload;
  const TypeInfo *TI = nullptr;
  TypeInfoKind TIK;
  unsigned NumElements;
  
  EnumImplStrategy(IRGenModule &IGM,
                   TypeInfoKind tik,
                    unsigned NumElements,
                    std::vector<Element> &&ElementsWithPayload,
                    std::vector<Element> &&ElementsWithRecursivePayload,
                    std::vector<Element> &&ElementsWithNoPayload)
  : ElementsWithPayload(std::move(ElementsWithPayload)),
    ElementsWithRecursivePayload(std::move(ElementsWithRecursivePayload)),
    ElementsWithNoPayload(std::move(ElementsWithNoPayload)),
    TIK(tik),
    NumElements(NumElements)
  {}
  
  /// Save the TypeInfo created for the enum.
  TypeInfo *registerEnumTypeInfo(TypeInfo *mutableTI) {
    TI = mutableTI;
    return mutableTI;
  }
  
  /// Constructs a TypeInfo for an enum of the best possible kind for its
  /// layout, FixedEnumTypeInfo or LoadableEnumTypeInfo.
  TypeInfo *getFixedEnumTypeInfo(llvm::StructType *T, Size S, llvm::BitVector SB,
                                  Alignment A, IsPOD_t isPOD);
  
public:
  virtual ~EnumImplStrategy() { }
  
  /// Construct a layout strategy appropriate to the enum type.
  static EnumImplStrategy *get(TypeConverter &TC,
                               CanType type,
                               EnumDecl *theEnum);
  
  /// Given an incomplete StructType for the enum, completes layout of the
  /// storage type, calculates its size and alignment, and produces the
  /// TypeInfo for the enum.
  virtual TypeInfo *completeEnumTypeLayout(TypeConverter &TC,
                                            CanType type,
                                            EnumDecl *theEnum,
                                            llvm::StructType *enumTy) = 0;
  
  const TypeInfo &getTypeInfo() const {
    assert(TI);
    return *TI;
  }
  
  llvm::StructType *getStorageType() const {
    return cast<llvm::StructType>(getTypeInfo().getStorageType());
  }
  
  IsPOD_t isPOD(ResilienceScope scope) const {
    return getTypeInfo().isPOD(scope);
  }
  
  /// \group Indirect enum operations
  
  /// Project the address of the data for a case. Does not check or modify
  /// the referenced enum value.
  /// Corresponds to the SIL 'enum_data_addr' instruction.
  virtual Address projectDataForStore(IRGenFunction &IGF,
                                      EnumElementDecl *elt,
                                      Address enumAddr) const = 0;
  
  /// Overlay the tag value for a case onto a data value in memory.
  /// Corresponds to the SIL 'inject_enum_addr' instruction.
  virtual void storeTag(IRGenFunction &IGF,
                        EnumElementDecl *elt,
                        Address enumAddr) const = 0;
  
  /// Clears tag bits from within the payload of an enum in memory and
  /// projects the address of the data for a case. Does not check
  /// the referenced enum value.
  /// Performs the block argument binding for a SIL
  /// 'destructive_switch_enum_addr' instruction.
  virtual Address destructiveProjectDataForLoad(IRGenFunction &IGF,
                                                EnumElementDecl *elt,
                                                Address enumAddr) const = 0;
  
  /// Emit a branch on the case contained by an enum explosion.
  /// Performs the branching for a SIL 'destructive_switch_enum_addr'
  /// instruction.
  virtual void emitIndirectSwitch(IRGenFunction &IGF,
                                  Address enumAddr,
                                  ArrayRef<std::pair<EnumElementDecl*,
                                                     llvm::BasicBlock*>> dests,
                                  llvm::BasicBlock *defaultDest) const = 0;
  
  /// \group Loadable enum operations
  
  /// Emit the construction sequence for an enum case into an explosion.
  /// Corresponds to the SIL 'enum' instruction.
  virtual void emitValueInjection(IRGenFunction &IGF,
                                  EnumElementDecl *elt,
                                  Explosion &params,
                                  Explosion &out) const = 0;
  
  /// Emit a branch on the case contained by an enum explosion.
  /// Performs the branching for a SIL 'switch_enum' instruction.
  virtual void emitValueSwitch(IRGenFunction &IGF,
                               Explosion &value,
                               ArrayRef<std::pair<EnumElementDecl*,
                                                  llvm::BasicBlock*>> dests,
                               llvm::BasicBlock *defaultDest) const = 0;
  
  /// Project a case value out of an enum explosion. This does not check that
  /// the explosion actually contains a value of the given case.
  /// Performs the block argument binding for a SIL 'switch_enum'
  /// instruction.
  virtual void emitValueProject(IRGenFunction &IGF,
                                Explosion &inEnum,
                                EnumElementDecl *theCase,
                                Explosion &out) const = 0;
  
  /// \group Delegated TypeInfo operations
  
  virtual void getSchema(ExplosionSchema &schema) const = 0;
  virtual void destroy(IRGenFunction &IGF, Address addr) const = 0;
  
  virtual bool isIndirectArgument(ExplosionKind kind) const {
    return TIK < Loadable;
  }
  
  virtual void initializeFromParams(IRGenFunction &IGF, Explosion &params,
                                    Address dest) const {
    if (TIK >= Loadable)
      return initialize(IGF, params, dest);
    Address src = TI->getAddressForPointer(params.claimNext());
    TI->initializeWithTake(IGF, dest, src);
  }
  
  virtual void assignWithCopy(IRGenFunction &IGF, Address dest,
                              Address src) const = 0;
  virtual void assignWithTake(IRGenFunction &IGF, Address dest,
                              Address src) const = 0;
  virtual void initializeWithCopy(IRGenFunction &IGF, Address dest,
                                  Address src) const = 0;
  virtual void initializeWithTake(IRGenFunction &IGF, Address dest,
                                  Address src) const = 0;
  
  virtual void initializeMetadata(IRGenFunction &IGF,
                                           llvm::Value *metadata,
                                           llvm::Value *vwtable) const = 0;

  virtual bool mayHaveExtraInhabitants(IRGenModule &IGM) const = 0;

  virtual llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF,
                                               Address src) const = 0;
  virtual void storeExtraInhabitant(IRGenFunction &IGF,
                                    llvm::Value *index,
                                    Address dest) const = 0;
  
  /// \group Delegated FixedTypeInfo operations
  
  virtual unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const = 0;
  
  virtual llvm::ConstantInt *
  getFixedExtraInhabitantValue(IRGenModule &IGM,
                               unsigned bits,
                               unsigned index) const = 0;
  
  /// \group Delegated LoadableTypeInfo operations
  
  virtual unsigned getExplosionSize(ExplosionKind kind) const = 0;
  virtual void loadAsCopy(IRGenFunction &IGF, Address addr,
                          Explosion &e) const = 0;
  virtual void loadAsTake(IRGenFunction &IGF, Address addr,
                          Explosion &e) const = 0;
  virtual void assign(IRGenFunction &IGF, Explosion &e,
                      Address addr) const = 0;
  virtual void initialize(IRGenFunction &IGF, Explosion &e,
                          Address addr) const = 0;
  virtual void reexplode(IRGenFunction &IGF, Explosion &src,
                         Explosion &dest) const = 0;
  virtual void copy(IRGenFunction &IGF, Explosion &src,
                    Explosion &dest) const = 0;
  virtual void consume(IRGenFunction &IGF, Explosion &src) const = 0;
  virtual llvm::Value *packEnumPayload(IRGenFunction &IGF,
                                        Explosion &in,
                                        unsigned bitWidth,
                                        unsigned offset) const = 0;
  
  virtual void unpackEnumPayload(IRGenFunction &IGF,
                                  llvm::Value *payload,
                                  Explosion &dest,
                                  unsigned offset) const = 0;
};

  /// Implementation strategy for singleton enums, with zero or one cases.
  class SingletonEnumImplStrategy final : public EnumImplStrategy {
    const TypeInfo *getSingleton() const {
      return ElementsWithPayload.empty() ? nullptr : ElementsWithPayload[0].ti;
    }
    
    const FixedTypeInfo *getFixedSingleton() const {
      return cast_or_null<FixedTypeInfo>(getSingleton());
    }
    
    const LoadableTypeInfo *getLoadableSingleton() const {
      return cast_or_null<LoadableTypeInfo>(getSingleton());
    }

    Address getSingletonAddress(IRGenFunction &IGF, Address addr) const {
      return IGF.Builder.CreateBitCast(addr,
                             getSingleton()->getStorageType()->getPointerTo());
    }
    
  public:
    SingletonEnumImplStrategy(IRGenModule &IGM,
                              TypeInfoKind tik, unsigned NumElements,
                              std::vector<Element> &&WithPayload,
                              std::vector<Element> &&WithRecursivePayload,
                              std::vector<Element> &&WithNoPayload)
      : EnumImplStrategy(IGM, tik, NumElements,
                         std::move(WithPayload),
                         std::move(WithRecursivePayload),
                         std::move(WithNoPayload))
    {
      assert(NumElements <= 1);
      assert(ElementsWithPayload.size() <= 1);
    }
    
    TypeInfo *completeEnumTypeLayout(TypeConverter &TC,
                                      CanType type,
                                      EnumDecl *theEnum,
                                      llvm::StructType *enumTy) override;

    void emitSingletonSwitch(IRGenFunction &IGF,
                    ArrayRef<std::pair<EnumElementDecl*,
                                       llvm::BasicBlock*>> dests,
                    llvm::BasicBlock *defaultDest) const {
      // No dispatch necessary. Branch straight to the destination.
      assert(dests.size() <= 1 && "impossible switch table for singleton enum");
      llvm::BasicBlock *dest = dests.size() == 1
        ? dests[0].second : defaultDest;
      IGF.Builder.CreateBr(dest);
    }
    
    void emitValueSwitch(IRGenFunction &IGF,
                         Explosion &value,
                         ArrayRef<std::pair<EnumElementDecl*,
                                            llvm::BasicBlock*>> dests,
                         llvm::BasicBlock *defaultDest) const override {
      value.claim(getExplosionSize(value.getKind()));
      emitSingletonSwitch(IGF, dests, defaultDest);
    }
    
    void emitIndirectSwitch(IRGenFunction &IGF,
                            Address addr,
                            ArrayRef<std::pair<EnumElementDecl*,
                                               llvm::BasicBlock*>> dests,
                            llvm::BasicBlock *defaultDest) const override {
      emitSingletonSwitch(IGF, dests, defaultDest);
    }
    
    void emitValueProject(IRGenFunction &IGF,
                          Explosion &in,
                          EnumElementDecl *theCase,
                          Explosion &out) const override {
      // The projected value is the payload.
      if (getLoadableSingleton())
        getLoadableSingleton()->reexplode(IGF, in, out);
    }

    void emitValueInjection(IRGenFunction &IGF,
                            EnumElementDecl *elt,
                            Explosion &params,
                            Explosion &out) const override {
      // If the element carries no data, neither does the injection.
      // Otherwise, the result is identical.
      if (getLoadableSingleton())
        getLoadableSingleton()->reexplode(IGF, params, out);
    }
    
    Address projectDataForStore(IRGenFunction &IGF,
                                EnumElementDecl *elt,
                                Address enumAddr) const override {
      return getSingletonAddress(IGF, enumAddr);
    }
    
    Address destructiveProjectDataForLoad(IRGenFunction &IGF,
                                          EnumElementDecl *elt,
                                          Address enumAddr) const override {
      return getSingletonAddress(IGF, enumAddr);
    }
    
    void storeTag(IRGenFunction &IGF,
                  EnumElementDecl *elt,
                  Address enumAddr) const override {
      // No tag, nothing to do.
    }
    
    void getSchema(ExplosionSchema &schema) const {
      if (!getSingleton()) return;
      // If the payload is loadable, forward its explosion schema.
      if (TIK >= Loadable)
        return getSingleton()->getSchema(schema);
      // Otherwise, use an indirect aggregate schema with our storage
      // type.
      schema.add(ExplosionSchema::Element::forAggregate(getStorageType(),
                                      getSingleton()->getBestKnownAlignment()));
    }
    
    unsigned getExplosionSize(ExplosionKind kind) const {
      if (!getLoadableSingleton()) return 0;
      return getLoadableSingleton()->getExplosionSize(kind);
    }
    
    void loadAsCopy(IRGenFunction &IGF, Address addr, Explosion &e) const {
      if (!getLoadableSingleton()) return;
      getLoadableSingleton()->loadAsCopy(IGF, getSingletonAddress(IGF, addr),e);
    }

    void loadForSwitch(IRGenFunction &IGF, Address addr, Explosion &e) const {
      // Switching on a singleton does not require a value.
      return;
    }

    void loadAsTake(IRGenFunction &IGF, Address addr, Explosion &e) const {
      if (!getLoadableSingleton()) return;
      getLoadableSingleton()->loadAsTake(IGF, getSingletonAddress(IGF, addr),e);
    }
    
    void assign(IRGenFunction &IGF, Explosion &e, Address addr) const {
      if (!getLoadableSingleton()) return;
      getLoadableSingleton()->assign(IGF, e, getSingletonAddress(IGF, addr));
    }
    
    void assignWithCopy(IRGenFunction &IGF, Address dest, Address src) const {
      if (!getSingleton()) return;
      dest = getSingletonAddress(IGF, dest);
      src = getSingletonAddress(IGF, src);
      getSingleton()->assignWithCopy(IGF, dest, src);
    }
    
    void assignWithTake(IRGenFunction &IGF, Address dest, Address src) const {
      if (!getSingleton()) return;
      dest = getSingletonAddress(IGF, dest);
      src = getSingletonAddress(IGF, src);
      getSingleton()->assignWithTake(IGF, dest, src);
    }
    
    void initialize(IRGenFunction &IGF, Explosion &e, Address addr) const {
      if (!getLoadableSingleton()) return;
      getLoadableSingleton()->initialize(IGF, e, getSingletonAddress(IGF, addr));
    }
    
    void initializeWithCopy(IRGenFunction &IGF, Address dest, Address src)
    const override {
      if (!getSingleton()) return;
      dest = getSingletonAddress(IGF, dest);
      src = getSingletonAddress(IGF, src);
      getSingleton()->initializeWithCopy(IGF, dest, src);
    }
    
    void initializeWithTake(IRGenFunction &IGF, Address dest, Address src)
    const override {
      if (!getSingleton()) return;
      dest = getSingletonAddress(IGF, dest);
      src = getSingletonAddress(IGF, src);
      getSingleton()->initializeWithTake(IGF, dest, src);
    }
    
    void reexplode(IRGenFunction &IGF, Explosion &src, Explosion &dest) const {
      if (getLoadableSingleton()) getLoadableSingleton()->reexplode(IGF, src, dest);
    }
    
    void copy(IRGenFunction &IGF, Explosion &src, Explosion &dest) const {
      if (getLoadableSingleton()) getLoadableSingleton()->copy(IGF, src, dest);
    }
    
    void consume(IRGenFunction &IGF, Explosion &src) const {
      if (getLoadableSingleton()) getLoadableSingleton()->consume(IGF, src);
    }
    
    void destroy(IRGenFunction &IGF, Address addr) const {
      if (getSingleton() && !getSingleton()->isPOD(ResilienceScope::Local))
        getSingleton()->destroy(IGF, getSingletonAddress(IGF, addr));
    }
    
    llvm::Value *packEnumPayload(IRGenFunction &IGF,
                                  Explosion &in,
                                  unsigned bitWidth,
                                  unsigned offset) const override {
      if (getLoadableSingleton())
        return getLoadableSingleton()->packEnumPayload(IGF, in,
                                                        bitWidth, offset);
      return PackEnumPayload::getEmpty(IGF.IGM, bitWidth);
    }
    
    void unpackEnumPayload(IRGenFunction &IGF,
                            llvm::Value *payload,
                            Explosion &dest,
                            unsigned offset) const override {
      if (!getLoadableSingleton()) return;
      getLoadableSingleton()->unpackEnumPayload(IGF, payload, dest, offset);
    }
    
    void initializeMetadata(IRGenFunction &IGF,
                                     llvm::Value *metadata,
                                     llvm::Value *vwtable) const override {
      // Fixed-size enums don't need dynamic witness table initialization.
      if (TIK >= Fixed) return;

      assert(!ElementsWithPayload.empty() &&
             "empty singleton enum should not be dynamic!");

      // Get the value witness table for the element.
      CanType eltTy
       = ElementsWithPayload[0].decl->getArgumentType()->getCanonicalType();
      llvm::Value *eltMetadata = IGF.emitTypeMetadataRef(eltTy);
      llvm::Value *eltVWT
        = IGF.emitValueWitnessTableRefForMetadata(eltMetadata);
      
      Address vwtAddr(vwtable, IGF.IGM.getPointerAlignment());
      Address eltVWTAddr(eltVWT, IGF.IGM.getPointerAlignment());
      
      auto copyWitnessFromElt = [&](ValueWitness witness) -> llvm::Value* {
        Address dest = IGF.Builder.CreateConstArrayGEP(vwtAddr,
                                   unsigned(witness), IGF.IGM.getPointerSize());
        Address src = IGF.Builder.CreateConstArrayGEP(eltVWTAddr,
                                   unsigned(witness), IGF.IGM.getPointerSize());
        auto val = IGF.Builder.CreateLoad(src);
        IGF.Builder.CreateStore(val, dest);
        return val;
      };

      copyWitnessFromElt(ValueWitness::Size);
      auto flags = copyWitnessFromElt(ValueWitness::Flags);
      copyWitnessFromElt(ValueWitness::Stride);
      
      // If the original type had extra inhabitants, carry over its
      // extra inhabitant flags.
      auto xiBB = llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
      auto noXIBB = llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
      
      auto xiFlag = IGF.Builder.CreatePtrToInt(flags, IGF.IGM.SizeTy);
      auto xiMask
        = IGF.IGM.getSize(Size(ValueWitnessFlags::Enum_HasExtraInhabitants));
      xiFlag = IGF.Builder.CreateAnd(xiFlag, xiMask);
      auto xiBool = IGF.Builder.CreateICmpNE(xiFlag,
                                             IGF.IGM.getSize(Size(0)));
      IGF.Builder.CreateCondBr(xiBool, xiBB, noXIBB);
      
      IGF.Builder.emitBlock(xiBB);
      copyWitnessFromElt(ValueWitness::ExtraInhabitantFlags);
      IGF.Builder.CreateBr(noXIBB);
      
      IGF.Builder.emitBlock(noXIBB);
    }
    
    bool mayHaveExtraInhabitants(IRGenModule &IGM) const override {
      // FIXME: Hold off on registering extra inhabitants for dynamic enums
      // until initializeMetadata handles them.
      if (!getSingleton())
        return false;
      return getSingleton()->mayHaveExtraInhabitants(IGM);
    }
    
    llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF,
                                         Address src) const override {
      if (!getSingleton()) {
        // Any empty value is a valid value.
        return llvm::ConstantInt::getSigned(IGF.IGM.Int32Ty, -1);
      }

      return getSingleton()->getExtraInhabitantIndex(IGF,
                                                 getSingletonAddress(IGF, src));
    }
    
    void storeExtraInhabitant(IRGenFunction &IGF,
                              llvm::Value *index,
                              Address dest) const override {
      if (!getSingleton()) {
        // Nothing to store for empty singletons.
        return;
      }
      getSingleton()->storeExtraInhabitant(IGF, index,
                                           getSingletonAddress(IGF, dest));
    }
    
    unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
      assert(TIK >= Fixed);
      if (!getSingleton())
        return 0;
      return getFixedSingleton()->getFixedExtraInhabitantCount(IGM);
    }
    
    llvm::ConstantInt *
    getFixedExtraInhabitantValue(IRGenModule &IGM,
                                 unsigned bits,
                                 unsigned index) const override {
      assert(TIK >= Fixed);
      assert(getSingleton() && "empty singletons have no extra inhabitants");
      return getFixedSingleton()
        ->getFixedExtraInhabitantValue(IGM, bits, index);
    }
  };
  
  /// Implementation strategy for no-payload enums, in other words, 'C-like'
  /// enums where none of the cases have data.
  class NoPayloadEnumImplStrategyBase
    : public SingleScalarTypeInfo<NoPayloadEnumImplStrategyBase,
                                  EnumImplStrategy>
  {
  protected:
    llvm::IntegerType *getDiscriminatorType() const {
      llvm::StructType *Struct = getStorageType();
      return cast<llvm::IntegerType>(Struct->getElementType(0));
    }
    
    /// Map the given element to the appropriate value in the
    /// discriminator type.
    virtual llvm::ConstantInt *getDiscriminatorIndex(EnumElementDecl *target)
      const = 0;
    
  public:
    NoPayloadEnumImplStrategyBase(IRGenModule &IGM,
                                  TypeInfoKind tik, unsigned NumElements,
                           std::vector<Element> &&WithPayload,
                           std::vector<Element> &&WithRecursivePayload,
                           std::vector<Element> &&WithNoPayload)
      : SingleScalarTypeInfo(IGM, tik, NumElements,
                             std::move(WithPayload),
                             std::move(WithRecursivePayload),
                             std::move(WithNoPayload))
    {
      assert(ElementsWithPayload.empty());
      assert(!ElementsWithNoPayload.empty());
    }
    
    void emitValueSwitch(IRGenFunction &IGF,
                         Explosion &value,
                         ArrayRef<std::pair<EnumElementDecl*,
                                            llvm::BasicBlock*>> dests,
                         llvm::BasicBlock *defaultDest) const override {
      llvm::Value *discriminator = value.claimNext();
      
      // Create an unreachable block for the default if the original SIL
      // instruction had none.
      bool unreachableDefault = false;
      if (!defaultDest) {
        unreachableDefault = true;
        defaultDest = llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
      }
      
      auto *i = IGF.Builder.CreateSwitch(discriminator, defaultDest,
                                         dests.size());
      for (auto &dest : dests)
        i->addCase(getDiscriminatorIndex(dest.first), dest.second);
      
      if (unreachableDefault) {
        IGF.Builder.emitBlock(defaultDest);
        IGF.Builder.CreateUnreachable();
      }
    }
    
    void emitIndirectSwitch(IRGenFunction &IGF,
                            Address addr,
                            ArrayRef<std::pair<EnumElementDecl*,
                                               llvm::BasicBlock*>> dests,
                            llvm::BasicBlock *defaultDest) const override {
      Explosion value(ExplosionKind::Minimal);
      loadAsTake(IGF, addr, value);
      emitValueSwitch(IGF, value, dests, defaultDest);
    }
    
    void emitValueProject(IRGenFunction &IGF,
                          Explosion &in,
                          EnumElementDecl *elt,
                          Explosion &out) const override {
      // All of the cases project an empty explosion.
      in.claim(getExplosionSize(in.getKind()));
    }
    
    void emitValueInjection(IRGenFunction &IGF,
                            EnumElementDecl *elt,
                            Explosion &params,
                            Explosion &out) const {
      out.add(getDiscriminatorIndex(elt));
    }
    
    Address projectDataForStore(IRGenFunction &IGF,
                                EnumElementDecl *elt,
                                Address enumAddr) const override {
      llvm_unreachable("cannot project data for no-payload cases");
    }
    Address destructiveProjectDataForLoad(IRGenFunction &IGF,
                                          EnumElementDecl *elt,
                                          Address enumAddr) const override {
      llvm_unreachable("cannot project data for no-payload cases");
    }
    
    void storeTag(IRGenFunction &IGF, EnumElementDecl *elt, Address enumAddr)
    const override {
      llvm::Value *discriminator = getDiscriminatorIndex(elt);
      Address discriminatorAddr
        = IGF.Builder.CreateStructGEP(enumAddr, 0, Size(0));
      IGF.Builder.CreateStore(discriminator, discriminatorAddr);
    }
    
    void initializeMetadata(IRGenFunction &IGF,
                                     llvm::Value *metadata,
                                     llvm::Value *vwtable) const override {
      // No-payload enums are always fixed-size so never need dynamic value
      // witness table initialization.
    }
    
    /// \group Required for SingleScalarTypeInfo
    
    llvm::Type *getScalarType() const {
      return getDiscriminatorType();
    }
    
    static Address projectScalar(IRGenFunction &IGF, Address addr) {
      return IGF.Builder.CreateStructGEP(addr, 0, Size(0));
    }

    void emitScalarRetain(IRGenFunction &IGF, llvm::Value *value) const {}
    void emitScalarRelease(IRGenFunction &IGF, llvm::Value *value) const {}
    
    void initializeWithTake(IRGenFunction &IGF, Address dest, Address src)
    const override {
      // No-payload enums are always POD, so we can always initialize by
      // primitive copy.
      llvm::Value *val = IGF.Builder.CreateLoad(src);
      IGF.Builder.CreateStore(val, dest);
    }
    
    static constexpr IsPOD_t IsScalarPOD = IsPOD;
  };
  
  /// Implementation strategy for native Swift no-payload enums.
  class NoPayloadEnumImplStrategy final
    : public NoPayloadEnumImplStrategyBase
  {
  protected:
    llvm::ConstantInt *getDiscriminatorIndex(EnumElementDecl *target)
    const override {
      // The elements are assigned discriminators in declaration order.
      // FIXME: using a linear search here is fairly ridiculous.
      unsigned index = 0;
      for (auto elt : target->getParentEnum()->getAllElements()) {
        if (elt == target) break;
        index++;
      }
      return llvm::ConstantInt::get(getDiscriminatorType(), index);
    }
    
  public:
    NoPayloadEnumImplStrategy(IRGenModule &IGM,
                              TypeInfoKind tik, unsigned NumElements,
                              std::vector<Element> &&WithPayload,
                              std::vector<Element> &&WithRecursivePayload,
                              std::vector<Element> &&WithNoPayload)
      : NoPayloadEnumImplStrategyBase(IGM, tik, NumElements,
                                      std::move(WithPayload),
                                      std::move(WithRecursivePayload),
                                      std::move(WithNoPayload))
    {
      assert(ElementsWithPayload.empty());
      assert(!ElementsWithNoPayload.empty());
    }

    TypeInfo *completeEnumTypeLayout(TypeConverter &TC,
                                     CanType type,
                                     EnumDecl *theEnum,
                                     llvm::StructType *enumTy) override;
    
    /// \group Extra inhabitants for no-payload enums.

    // No-payload enums have all values above their greatest discriminator
    // value that fit inside their storage size available as extra inhabitants.
    
    bool mayHaveExtraInhabitants(IRGenModule &IGM) const override {
      return getFixedExtraInhabitantCount(IGM) > 0;
    }
    
    unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
      unsigned bits = cast<FixedTypeInfo>(TI)->getFixedSize().getValueInBits();
      assert(bits < 32 && "freakishly huge no-payload enum");
      return (1U << bits) - ElementsWithNoPayload.size();
    }
    
    llvm::ConstantInt *
    getFixedExtraInhabitantValue(IRGenModule &IGM,
                                 unsigned bits,
                                 unsigned index) const override {
      unsigned value = index + ElementsWithNoPayload.size();
      return llvm::ConstantInt::get(IGM.getLLVMContext(),
                                    APInt(bits, value));
    }
    
    llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF,
                                         Address src) const override {
      auto &C = IGF.IGM.getLLVMContext();

      // Load the value.
      auto payloadTy = llvm::IntegerType::get(C,
                      cast<FixedTypeInfo>(TI)->getFixedSize().getValueInBits());
      src = IGF.Builder.CreateBitCast(src, payloadTy->getPointerTo());
      llvm::Value *val = IGF.Builder.CreateLoad(src);
      
      // Subtract the number of cases.
      val = IGF.Builder.CreateSub(val,
              llvm::ConstantInt::get(payloadTy, ElementsWithNoPayload.size()));
      
      // If signed less than zero, we have a valid value. Otherwise, we have
      // an extra inhabitant.
      auto valid
        = IGF.Builder.CreateICmpSLT(val, llvm::ConstantInt::get(payloadTy, 0));
      val = IGF.Builder.CreateSelect(valid,
                                     llvm::ConstantInt::getSigned(payloadTy, -1),
                                     val);
      
      val = IGF.Builder.CreateZExtOrTrunc(val, IGF.IGM.Int32Ty);
      return val;
    }
    
    void storeExtraInhabitant(IRGenFunction &IGF,
                              llvm::Value *index,
                              Address dest) const override {
      auto &C = IGF.IGM.getLLVMContext();
      auto payloadTy = llvm::IntegerType::get(C,
                      cast<FixedTypeInfo>(TI)->getFixedSize().getValueInBits());
      dest = IGF.Builder.CreateBitCast(dest, payloadTy->getPointerTo());

      index = IGF.Builder.CreateZExtOrTrunc(index, payloadTy);
      index = IGF.Builder.CreateAdd(index,
                llvm::ConstantInt::get(payloadTy, ElementsWithNoPayload.size()));
      IGF.Builder.CreateStore(index, dest);
    }
  };
  
  /// Implementation strategy for no-payload enums with C-compatible
  /// enums where none of the cases have data.
  class CCompatibleEnumImplStrategy final
    : public NoPayloadEnumImplStrategyBase
  {
  protected:
    llvm::ConstantInt *getDiscriminatorIndex(EnumElementDecl *target)
    const override {
      // The elements are assigned discriminators ABI-compatible with their
      // raw values from C.
      assert(target->hasRawValueExpr()
             && "c-compatible enum elt has no raw value?!");
      auto intExpr = cast<IntegerLiteralExpr>(target->getRawValueExpr());
      auto intType = getDiscriminatorType();
      
      APInt intValue = IntegerLiteralExpr::getValue(intExpr->getDigitsText(),
                                                    intType->getBitWidth());
      
      if (intExpr->isNegative())
        intValue = -intValue;
      
      return llvm::ConstantInt::get(intType->getContext(), intValue);
    }
    
  public:
    CCompatibleEnumImplStrategy(IRGenModule &IGM,
                                TypeInfoKind tik, unsigned NumElements,
                                std::vector<Element> &&WithPayload,
                                std::vector<Element> &&WithRecursivePayload,
                                std::vector<Element> &&WithNoPayload)
      : NoPayloadEnumImplStrategyBase(IGM, tik, NumElements,
                                      std::move(WithPayload),
                                      std::move(WithRecursivePayload),
                                      std::move(WithNoPayload))
    {
      assert(ElementsWithPayload.empty());
      assert(!ElementsWithNoPayload.empty());
    }
    
    TypeInfo *completeEnumTypeLayout(TypeConverter &TC,
                                     CanType type,
                                     EnumDecl *theEnum,
                                     llvm::StructType *enumTy) override;

    /// \group Extra inhabitants for C-compatible enums.

    // C-compatible enums have scattered inhabitants. For now, expose no
    // extra inhabitants.
    
    bool mayHaveExtraInhabitants(IRGenModule &IGM) const override {
      return false;
    }
    
    unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
      return 0;
    }
    
    llvm::ConstantInt *
    getFixedExtraInhabitantValue(IRGenModule &IGM,
                                 unsigned bits,
                                 unsigned index) const override {
      llvm_unreachable("no extra inhabitants");
    }
    
    llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF,
                                         Address src) const override {
      llvm_unreachable("no extra inhabitants");
    }
    
    void storeExtraInhabitant(IRGenFunction &IGF,
                              llvm::Value *index,
                              Address dest) const override {
      llvm_unreachable("no extra inhabitants");
    }
  };
  
  /// Common base class for enums with one or more cases with data.
  class PayloadEnumImplStrategyBase : public EnumImplStrategy {
  protected:
    llvm::IntegerType *payloadTy = nullptr, *extraTagTy = nullptr;
    
    // The number of extra tag bits outside of the payload required to
    // discriminate enum cases.
    unsigned ExtraTagBitCount = ~0u;
    // The number of possible values for the extra tag bits that are used.
    // Log2(NumExtraTagValues - 1) + 1 == ExtraTagBitCount
    unsigned NumExtraTagValues = ~0u;

    void setTaggedEnumBody(IRGenModule &IGM,
                           llvm::StructType *bodyStruct,
                           unsigned payloadBits, unsigned extraTagBits) {
      // LLVM's ABI rules for I.O.U.S. (Integer Of Unusual Size) types is to
      // pad them out as if aligned to the largest native integer type, even
      // inside "packed" structs, so to accurately lay things out, we use
      // i8 arrays for the payload and extra tag bits.
      auto payloadArrayTy = llvm::ArrayType::get(IGM.Int8Ty,
                                                 (payloadBits+7U)/8U);
      
      SmallVector<llvm::Type*, 2> body;
      
      // Handle the case when the payload has no storage.
      // This may come up when a generic type with payload is instantiated on an
      // empty type.
      if (payloadBits > 0) {
        payloadTy = llvm::IntegerType::get(IGM.getLLVMContext(),
                                           payloadBits);
        body.push_back(payloadArrayTy);
      } else {
        payloadTy = nullptr;
      }
      
      if (extraTagBits > 0) {
        auto extraTagArrayTy = llvm::ArrayType::get(IGM.Int8Ty,
                                                    (extraTagBits+7U)/8U);
        body.push_back(extraTagArrayTy);
        extraTagTy = llvm::IntegerType::get(IGM.getLLVMContext(),
                                            extraTagBits);
      } else {
        extraTagTy = nullptr;
      }
      bodyStruct->setBody(body, /*isPacked*/true);
    }
    
  public:
    PayloadEnumImplStrategyBase(IRGenModule &IGM,
                                TypeInfoKind tik, unsigned NumElements,
                                std::vector<Element> &&WithPayload,
                                std::vector<Element> &&WithRecursivePayload,
                                std::vector<Element> &&WithNoPayload)
      : EnumImplStrategy(IGM, tik, NumElements,
                         std::move(WithPayload),
                         std::move(WithRecursivePayload),
                         std::move(WithNoPayload))
    {
      assert(ElementsWithPayload.size() >= 1);
    }
    
    void getSchema(ExplosionSchema &schema) const override {
      if (TIK < Loadable) {
        schema.add(ExplosionSchema::Element::forAggregate(getStorageType(),
                                                  TI->getBestKnownAlignment()));
        return;
      }
      
      if (payloadTy)
        schema.add(ExplosionSchema::Element::forScalar(payloadTy));
      if (ExtraTagBitCount > 0)
        schema.add(ExplosionSchema::Element::forScalar(extraTagTy));
    }
    
    unsigned getExplosionSize(ExplosionKind kind) const override {
      return unsigned(ExtraTagBitCount > 0) + unsigned(payloadTy != nullptr);
    }
    
    Address projectPayload(IRGenFunction &IGF, Address addr) const {
      assert(payloadTy && "has empty payload");

      return IGF.Builder.CreateBitCast(addr, payloadTy->getPointerTo());
    }

    Address projectExtraTagBits(IRGenFunction &IGF, Address addr) const {
      assert(ExtraTagBitCount > 0 && "does not have extra tag bits");

      if (!payloadTy) {
        return IGF.Builder.CreateBitCast(addr, extraTagTy->getPointerTo());
      }
      
      addr = IGF.Builder.CreateStructGEP(addr, 1,
                                         Size(payloadTy->getBitWidth()/8U));
      return IGF.Builder.CreateBitCast(addr, extraTagTy->getPointerTo());
    }

    void loadForSwitch(IRGenFunction &IGF, Address addr, Explosion &e)
    const {
      assert(TIK >= Fixed);
      if (payloadTy)
        e.add(IGF.Builder.CreateLoad(projectPayload(IGF, addr)));
      if (ExtraTagBitCount > 0)
        e.add(IGF.Builder.CreateLoad(projectExtraTagBits(IGF, addr)));
    }

    void loadAsTake(IRGenFunction &IGF, Address addr, Explosion &e)
    const override {
      assert(TIK >= Loadable);
      loadForSwitch(IGF, addr, e);
    }
    
    void loadAsCopy(IRGenFunction &IGF, Address addr, Explosion &e)
    const override {
      assert(TIK >= Loadable);
      Explosion tmp(e.getKind());
      loadAsTake(IGF, addr, tmp);
      copy(IGF, tmp, e);
    }
    
    void assign(IRGenFunction &IGF, Explosion &e, Address addr) const override {
      assert(TIK >= Loadable);
      Explosion old(e.getKind());
      if (!isPOD(ResilienceScope::Local))
        loadAsTake(IGF, addr, old);
      initialize(IGF, e, addr);
      if (!isPOD(ResilienceScope::Local))
        consume(IGF, old);
    }
    
    void initialize(IRGenFunction &IGF, Explosion &e, Address addr)
    const override {
      assert(TIK >= Loadable);
      if (payloadTy)
        IGF.Builder.CreateStore(e.claimNext(), projectPayload(IGF, addr));
      if (ExtraTagBitCount > 0)
        IGF.Builder.CreateStore(e.claimNext(), projectExtraTagBits(IGF, addr));
    }
    
    void reexplode(IRGenFunction &IGF, Explosion &src, Explosion &dest)
    const override {
      assert(TIK >= Loadable);
      dest.add(src.claim(getExplosionSize(ExplosionKind::Minimal)));
    }
    
  protected:
    /// Do a primitive copy of the enum from one address to another.
    void emitPrimitiveCopy(IRGenFunction &IGF, Address dest, Address src) const{
      // If the layout is fixed, load and store the fixed-size payload and tag.
      if (TIK >= Fixed) {
        llvm::Value *payload, *extraTag;
        std::tie(payload, extraTag)
          = emitPrimitiveLoadPayloadAndExtraTag(IGF, src);
        emitPrimitiveStorePayloadAndExtraTag(IGF, dest, payload, extraTag);
        return;
      }
      
      // Otherwise, do a memcpy of the dynamic size of the type.
      IGF.Builder.CreateMemCpy(dest.getAddress(), src.getAddress(),
                               TI->getSize(IGF),
                               std::min(dest.getAlignment().getValue(),
                                        src.getAlignment().getValue()));
    }
    
    void emitPrimitiveStorePayloadAndExtraTag(IRGenFunction &IGF, Address dest,
                                              llvm::Value *payload,
                                              llvm::Value *extraTag) const {
      if (payloadTy)
        IGF.Builder.CreateStore(payload, projectPayload(IGF, dest));
      if (ExtraTagBitCount > 0)
        IGF.Builder.CreateStore(extraTag, projectExtraTagBits(IGF, dest));
    }
    
    std::pair<llvm::Value*, llvm::Value*>
    getPayloadAndExtraTagFromExplosion(Explosion &src) const {
      llvm::Value *payload = src.claimNext();
      llvm::Value *extraTag = ExtraTagBitCount > 0 ? src.claimNext() : nullptr;
      return {payload, extraTag};
    }
    
    std::pair<llvm::Value*, llvm::Value*>
    emitPrimitiveLoadPayloadAndExtraTag(IRGenFunction &IGF, Address addr) const{
      llvm::Value *payload = nullptr;
      llvm::Value *extraTag = nullptr;
      if (payloadTy)
        payload = IGF.Builder.CreateLoad(projectPayload(IGF, addr));
      if (ExtraTagBitCount > 0)
        extraTag = IGF.Builder.CreateLoad(projectExtraTagBits(IGF, addr));
      return {payload, extraTag};
    }
  };

  class SinglePayloadEnumImplStrategy final
    : public PayloadEnumImplStrategyBase
  {
    EnumElementDecl *getPayloadElement() const {
      return ElementsWithPayload[0].decl;
    }
    
    const TypeInfo &getPayloadTypeInfo() const {
      return *ElementsWithPayload[0].ti;
    }
    const FixedTypeInfo &getFixedPayloadTypeInfo() const {
      return cast<FixedTypeInfo>(*ElementsWithPayload[0].ti);
    }
    const LoadableTypeInfo &getLoadablePayloadTypeInfo() const {
      return cast<LoadableTypeInfo>(*ElementsWithPayload[0].ti);
    }
    
    CanType PayloadTy;
    
    llvm::Value *emitPayloadMetadata(IRGenFunction &IGF) const {
      return IGF.emitTypeMetadataRef(PayloadTy);
    }
    
    /// More efficient value semantics implementations for certain enum layouts.
    enum CopyDestroyStrategy {
      /// No special behavior.
      Normal,
      /// The payload is POD, so copying is bitwise, and destruction is a noop.
      POD,
      /// The payload is a single Swift reference-counted value, and we have
      /// a single no-payload case which uses the null extra inhabitant, so
      /// copy and destroy can pass through to swift_retain/swift_release.
      NullableSwiftRefcounted,
      /// The payload is a single unknown-reference-counted value, and we have
      /// a single no-payload case which uses the null extra inhabitant, so
      /// copy and destroy can pass through to swift_retain/swift_release.
      ///TODO: NullableUnknownRefcounted,
    };
    
    CopyDestroyStrategy CopyDestroyKind;
    
  public:
    SinglePayloadEnumImplStrategy(IRGenModule &IGM,
                                  TypeInfoKind tik, unsigned NumElements,
                                   std::vector<Element> &&WithPayload,
                                   std::vector<Element> &&WithRecursivePayload,
                                   std::vector<Element> &&WithNoPayload)
      : PayloadEnumImplStrategyBase(IGM, tik, NumElements,
                                     std::move(WithPayload),
                                     std::move(WithRecursivePayload),
                                     std::move(WithNoPayload)),
        CopyDestroyKind(Normal)
    {
      assert(ElementsWithPayload.size() == 1);
      
      // If the payload is POD, then we can use POD value semantics.
      if (ElementsWithPayload[0].ti->isPOD(ResilienceScope::Component))
        CopyDestroyKind = POD;
      // If the payload is a single refcounted pointer and we have a single
      // empty case, then the layout will be a nullable pointer, and we can
      // pass enum values directly into swift_retain/swift_release as-is.
      else if (tik >= TypeInfoKind::Loadable
          && ElementsWithPayload[0].ti->isSingleRetainablePointer(
                                                   ResilienceScope::Component)
          && ElementsWithNoPayload.size() == 1
          // FIXME: All single-retainable-pointer types should eventually have
          // extra inhabitants.
          && cast<FixedTypeInfo>(ElementsWithPayload[0].ti)
            ->getFixedExtraInhabitantCount(IGM) > 0) {
        CopyDestroyKind = NullableSwiftRefcounted;
      }
      
      // TODO: Same for single unknown-refcounted pointers.
    }
    
    /// The payload for a single-payload enum is always placed in front and
    /// will never have interleaved tag bits, so we can just bitcast the enum
    /// address to the payload type for either injection or projection of the
    /// enum.
    Address projectPayloadData(IRGenFunction &IGF, Address addr) const {
      return IGF.Builder.CreateBitCast(addr,
                         getPayloadTypeInfo().getStorageType()->getPointerTo());
    }
    Address projectDataForStore(IRGenFunction &IGF, EnumElementDecl *elt,
                                Address enumAddr) const override {
      assert(elt == getPayloadElement() && "cannot project no-data case");
      return projectPayloadData(IGF, enumAddr);
    }
    Address destructiveProjectDataForLoad(IRGenFunction &IGF,
                                          EnumElementDecl *elt,
                                          Address enumAddr) const override {
      assert(elt == getPayloadElement() && "cannot project no-data case");
      return projectPayloadData(IGF, enumAddr);
    }
    
    TypeInfo *completeEnumTypeLayout(TypeConverter &TC,
                                      CanType type,
                                      EnumDecl *theEnum,
                                      llvm::StructType *enumTy) override;
  private:
    TypeInfo *completeFixedLayout(TypeConverter &TC,
                                  CanType type,
                                  EnumDecl *theEnum,
                                  llvm::StructType *enumTy);
    TypeInfo *completeDynamicLayout(TypeConverter &TC,
                                    CanType type,
                                    EnumDecl *theEnum,
                                    llvm::StructType *enumTy);
    
  public:
    llvm::Value *packEnumPayload(IRGenFunction &IGF, Explosion &src,
                                  unsigned bitWidth,
                                  unsigned offset) const override {
      PackEnumPayload pack(IGF, bitWidth);
      // Pack payload.
      pack.addAtOffset(src.claimNext(), offset);
      
      // Pack tag bits, if any.
      if (ExtraTagBitCount > 0) {
        unsigned extraTagOffset
          = getFixedPayloadTypeInfo().getFixedSize().getValueInBits() + offset;
        
        pack.addAtOffset(src.claimNext(), extraTagOffset);
      }
      
      return pack.get();
    }
    
    void unpackEnumPayload(IRGenFunction &IGF, llvm::Value *outerPayload,
                            Explosion &dest,
                            unsigned offset) const override {
      UnpackEnumPayload unpack(IGF, outerPayload);
      
      // Unpack our inner payload.
      dest.add(unpack.claimAtOffset(payloadTy, offset));
      
      // Unpack our extra tag bits, if any.
      if (ExtraTagBitCount > 0) {
        unsigned extraTagOffset
          = getFixedPayloadTypeInfo().getFixedSize().getValueInBits() + offset;
        
        dest.add(unpack.claimAtOffset(extraTagTy, extraTagOffset));
      }
    }
    
    void emitValueSwitch(IRGenFunction &IGF,
                         Explosion &value,
                         ArrayRef<std::pair<EnumElementDecl*,
                                            llvm::BasicBlock*>> dests,
                         llvm::BasicBlock *defaultDest) const override {
      auto &C = IGF.IGM.getLLVMContext();
      
      // Create a map of the destination blocks for quicker lookup.
      llvm::DenseMap<EnumElementDecl*,llvm::BasicBlock*> destMap(dests.begin(),
                                                                  dests.end());
      // Create an unreachable branch for unreachable switch defaults.
      auto *unreachableBB = llvm::BasicBlock::Create(C);
      
      // If there was no default branch in SIL, use the unreachable branch as
      // the default.
      if (!defaultDest)
        defaultDest = unreachableBB;
      
      auto blockForCase = [&](EnumElementDecl *theCase) -> llvm::BasicBlock* {
        auto found = destMap.find(theCase);
        if (found == destMap.end())
          return defaultDest;
        else
          return found->second;
      };
      
      llvm::Value *payload = nullptr;
      if (payloadTy)
        payload = value.claimNext();
      llvm::BasicBlock *payloadDest = blockForCase(getPayloadElement());
      unsigned extraInhabitantCount
        = getFixedPayloadTypeInfo().getFixedExtraInhabitantCount(IGF.IGM);
      
      // If there are extra tag bits, switch over them first.
      SmallVector<llvm::BasicBlock*, 2> tagBitBlocks;
      if (ExtraTagBitCount > 0) {
        llvm::Value *tagBits = value.claimNext();
        
        auto *swi = IGF.Builder.CreateSwitch(tagBits, unreachableBB,
                                             NumExtraTagValues);
        
        // If we have extra inhabitants, we need to check for them in the
        // zero-tag case. Otherwise, we switch directly to the payload case.
        if (extraInhabitantCount > 0) {
          auto bb = llvm::BasicBlock::Create(C);
          tagBitBlocks.push_back(bb);
          swi->addCase(llvm::ConstantInt::get(C,APInt(ExtraTagBitCount,0)), bb);
        } else {
          tagBitBlocks.push_back(payloadDest);
          swi->addCase(llvm::ConstantInt::get(C,APInt(ExtraTagBitCount,0)),
                       payloadDest);
        }
        
        for (unsigned i = 1; i < NumExtraTagValues; ++i) {
          auto bb = llvm::BasicBlock::Create(C);
          tagBitBlocks.push_back(bb);
          swi->addCase(llvm::ConstantInt::get(C,APInt(ExtraTagBitCount,i)), bb);
        }
        
        // Continue by emitting the extra inhabitant dispatch, if any.
        if (extraInhabitantCount > 0)
          IGF.Builder.emitBlock(tagBitBlocks[0]);
      }
      
      auto elements = getPayloadElement()->getParentEnum()->getAllElements();
      auto elti = elements.begin(), eltEnd = elements.end();
      if (*elti == getPayloadElement())
        ++elti;
      
      // Advance the enum element iterator, skipping the payload case.
      auto nextCase = [&]() -> EnumElementDecl* {
        assert(elti != eltEnd);
        auto result = *elti;
        ++elti;
        if (elti != eltEnd && *elti == getPayloadElement())
          ++elti;
        return result;
      };
      
      // If there are no extra tag bits, or they're set to zero, then we either
      // have a payload, or an empty case represented using an extra inhabitant.
      // Check the extra inhabitant cases if we have any.
      unsigned payloadBits
        = getFixedPayloadTypeInfo().getFixedSize().getValueInBits();
      if (extraInhabitantCount > 0) {
        assert(payload && "extra inhabitants with empty payload?!");
        auto *swi = IGF.Builder.CreateSwitch(payload, payloadDest);
        for (unsigned i = 0; i < extraInhabitantCount && elti != eltEnd; ++i) {
          auto v = getFixedPayloadTypeInfo().getFixedExtraInhabitantValue(
                                                       IGF.IGM, payloadBits, i);
          swi->addCase(v, blockForCase(nextCase()));
        }
      }
      
      // We should have handled the payload case either in extra inhabitant
      // or in extra tag dispatch by now.
      assert(IGF.Builder.hasPostTerminatorIP() &&
             "did not handle payload case");
      
      // If there's an empty payload, each tag value corresponds to a single
      // empty case.
      if (!payload) {
        for (unsigned i = 1, e = tagBitBlocks.size(); i < e; ++i) {
          assert(elti != eltEnd &&
                 "ran out of cases before running out of extra tags?");
          IGF.Builder.emitBlock(tagBitBlocks[i]);
          IGF.Builder.CreateBr(blockForCase(nextCase()));
        }
      } else {
        // Handle the cases covered by each tag bit value.
        unsigned casesPerTag = 1 << ExtraTagBitCount;
        for (unsigned i = 1, e = tagBitBlocks.size(); i < e; ++i) {
          assert(elti != eltEnd &&
                 "ran out of cases before running out of extra tags?");
          IGF.Builder.emitBlock(tagBitBlocks[i]);
          auto swi = IGF.Builder.CreateSwitch(payload, unreachableBB);
          for (unsigned tag = 0; tag < casesPerTag && elti != eltEnd; ++tag) {
            auto v = llvm::ConstantInt::get(C, APInt(payloadBits, tag));
            swi->addCase(v, blockForCase(nextCase()));
          }
        }
      }
      
      // Delete the unreachable default block if we didn't use it, or emit it
      // if we did.
      if (unreachableBB->use_empty()) {
        delete unreachableBB;
      } else {
        IGF.Builder.emitBlock(unreachableBB);
        IGF.Builder.CreateUnreachable();
      }
    }
    
    void emitDynamicSwitch(IRGenFunction &IGF,
                           Address addr,
                           ArrayRef<std::pair<EnumElementDecl*,
                                              llvm::BasicBlock*>> dests,
                           llvm::BasicBlock *defaultDest) const {
      auto payloadMetadata = emitPayloadMetadata(IGF);
      auto numEmptyCases = llvm::ConstantInt::get(IGF.IGM.Int32Ty,
                                                  ElementsWithNoPayload.size());
      auto opaqueAddr = IGF.Builder.CreateBitCast(addr.getAddress(),
                                                  IGF.IGM.OpaquePtrTy);
      
      // Create a map of the destination blocks for quicker lookup.
      llvm::DenseMap<EnumElementDecl*,llvm::BasicBlock*> destMap(dests.begin(),
                                                                  dests.end());

      // If there was no default branch in SIL, use an unreachable branch as
      // the default.
      llvm::BasicBlock *unreachableBB = nullptr;
      if (!defaultDest) {
        unreachableBB = llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
        defaultDest = unreachableBB;
      }
      
      // Ask the runtime to find the case index.
      auto caseIndex = IGF.Builder.CreateCall3(
                            IGF.IGM.getGetEnumCaseSinglePayloadFn(),
                            opaqueAddr, payloadMetadata, numEmptyCases);

      // Switch on the index.
      auto *swi = IGF.Builder.CreateSwitch(caseIndex, defaultDest);

      // Add the payload case.
      auto payloadCase = destMap.find(getPayloadElement());
      if (payloadCase != destMap.end())
        swi->addCase(llvm::ConstantInt::getSigned(IGF.IGM.Int32Ty, -1),
                     payloadCase->second);
      
      // Add the empty cases.
      unsigned emptyCaseIndex = 0;
      for (auto &empty : ElementsWithNoPayload) {
        auto emptyCase = destMap.find(empty.decl);
        if (emptyCase != destMap.end())
          swi->addCase(llvm::ConstantInt::get(IGF.IGM.Int32Ty, emptyCaseIndex),
                       emptyCase->second);
        ++emptyCaseIndex;
      }
      
      // Emit the unreachable block, if any.
      if (unreachableBB) {
        IGF.Builder.emitBlock(unreachableBB);
        IGF.Builder.CreateUnreachable();
      }
    }
    
    void emitIndirectSwitch(IRGenFunction &IGF,
                            Address addr,
                            ArrayRef<std::pair<EnumElementDecl*,
                                               llvm::BasicBlock*>> dests,
                            llvm::BasicBlock *defaultDest) const override {
      if (TIK >= Fixed) {
        // Load the fixed-size representation and switch directly.
        Explosion value(ExplosionKind::Minimal);
        loadForSwitch(IGF, addr, value);
        return emitValueSwitch(IGF, value, dests, defaultDest);
      }
      
      // Use the runtime to dynamically switch.
      emitDynamicSwitch(IGF, addr, dests, defaultDest);
    }
    
    void emitValueProject(IRGenFunction &IGF,
                          Explosion &inEnum,
                          EnumElementDecl *theCase,
                          Explosion &out) const override {
      // Only the payload case has anything to project. The other cases are
      // empty.
      if (theCase != getPayloadElement()) {
        inEnum.claim(getExplosionSize(inEnum.getKind()));
        return;
      }
      
      if (payloadTy) {
        llvm::Value *payload = inEnum.claimNext();
        getLoadablePayloadTypeInfo().unpackEnumPayload(IGF, payload, out, 0);
      } else {
        assert(getLoadablePayloadTypeInfo()
                 .getSchema(ExplosionKind::Minimal)
                 .empty()
               && "empty payload with non-empty explosion schema?!");
      }
      if (ExtraTagBitCount > 0)
        inEnum.claimNext();
    }
    
  private:
    // Get the index of an enum element among the non-payload cases.
    unsigned getSimpleElementTagIndex(EnumElementDecl *elt) const {
      assert(elt != getPayloadElement() && "is payload element");
      unsigned i = 0;
      // FIXME: linear search
      for (auto *enumElt : elt->getParentEnum()->getAllElements()) {
        if (elt == enumElt)
          return i;
        if (enumElt != getPayloadElement())
          ++i;
      }
      llvm_unreachable("element was not a member of enum");
    }
    
    // Get the payload and extra tag (if any) parts of the discriminator for
    // a no-data case.
    std::pair<llvm::Value *, llvm::Value *>
    getNoPayloadCaseValue(IRGenFunction &IGF, EnumElementDecl *elt) const {
      assert(elt != getPayloadElement());
      
      unsigned payloadSize
        = getFixedPayloadTypeInfo().getFixedSize().getValueInBits();
      
      // Non-payload cases use extra inhabitants, if any, or are discriminated
      // by setting the tag bits.
      unsigned tagIndex = getSimpleElementTagIndex(elt);
      unsigned numExtraInhabitants
        = getFixedPayloadTypeInfo().getFixedExtraInhabitantCount(IGF.IGM);
      llvm::Value *payload = nullptr;
      unsigned extraTagValue;
      if (tagIndex < numExtraInhabitants) {
        payload = getFixedPayloadTypeInfo().getFixedExtraInhabitantValue(
                                               IGF.IGM, payloadSize, tagIndex);
        extraTagValue = 0;
      } else {
        tagIndex -= numExtraInhabitants;
        
        // Factor the extra tag value from the payload value.
        unsigned payloadValue;
        if (payloadSize >= 32) {
          payloadValue = tagIndex;
          extraTagValue = 1U;
        } else {
          payloadValue = tagIndex & ((1U << payloadSize) - 1U);
          extraTagValue = (tagIndex >> payloadSize) + 1U;
        }
        
        if (payloadTy)
          payload = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(),
                                           APInt(payloadSize, payloadValue));
      }

      llvm::Value *extraTag = nullptr;
      if (ExtraTagBitCount > 0) {
        extraTag = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(),
                                        APInt(ExtraTagBitCount, extraTagValue));
      } else {
        assert(extraTagValue == 0 &&
               "non-zero extra tag value with no tag bits");
      }
      return {payload, extraTag};
    }
    
  public:
    void emitValueInjection(IRGenFunction &IGF,
                            EnumElementDecl *elt,
                            Explosion &params,
                            Explosion &out) const {
      // The payload case gets its native representation. If there are extra
      // tag bits, set them to zero.
      unsigned payloadSize
        = getFixedPayloadTypeInfo().getFixedSize().getValueInBits();
      
      if (elt == getPayloadElement()) {
        if (payloadTy) {
          auto &loadablePayloadTI = getLoadablePayloadTypeInfo();
          llvm::Value *payload
            = loadablePayloadTI.packEnumPayload(IGF, params, payloadSize, 0);
          out.add(payload);
        }

        if (ExtraTagBitCount > 0)
          out.add(getZeroExtraTagConstant(IGF.IGM));
        return;
      }
      
      // Non-payload cases use extra inhabitants, if any, or are discriminated
      // by setting the tag bits.
      llvm::Value *payload, *extraTag;
      std::tie(payload, extraTag) = getNoPayloadCaseValue(IGF, elt);
      if (payloadTy) {
        assert(payload);
        out.add(payload);
      }
      if (ExtraTagBitCount > 0) {
        assert(extraTag);
        out.add(extraTag);
      }
    }
    
  private:
    /// Emits the test(s) that determine whether the fixed-size enum contains a
    /// payload or an empty case. Emits the basic block for the "true" case and
    /// returns the unemitted basic block for the "false" case.
    llvm::BasicBlock *
    testFixedEnumContainsPayload(IRGenFunction &IGF,
                                  llvm::Value *payload,
                                  llvm::Value *extraBits) const {
      auto *falseBB = llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
      
      // We only need to apply the payload operation if the enum contains a
      // value of the payload case.
      
      // If we have extra tag bits, they will be zero if we contain a payload.
      if (ExtraTagBitCount > 0) {
        assert(extraBits);
        llvm::Value *zero = llvm::ConstantInt::get(extraBits->getType(), 0);
        llvm::Value *isZero = IGF.Builder.CreateICmp(llvm::CmpInst::ICMP_EQ,
                                                     extraBits, zero);
        
        auto *trueBB = llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
        IGF.Builder.CreateCondBr(isZero, trueBB, falseBB);
        
        IGF.Builder.emitBlock(trueBB);
      }
      
      // If we used extra inhabitants to represent empty case discriminators,
      // weed them out.
      unsigned numExtraInhabitants
        = getFixedPayloadTypeInfo().getFixedExtraInhabitantCount(IGF.IGM);
      if (numExtraInhabitants > 0) {
        auto *payloadBB = llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
        auto *swi = IGF.Builder.CreateSwitch(payload, payloadBB);
        
        auto elements = getPayloadElement()->getParentEnum()->getAllElements();
        unsigned inhabitant = 0;
        for (auto i = elements.begin(), end = elements.end();
             i != end && inhabitant < numExtraInhabitants;
             ++i, ++inhabitant) {
          if (*i == getPayloadElement()) {
            ++i;
            if (i == end)
              break;
          }
          auto xi = getFixedPayloadTypeInfo().getFixedExtraInhabitantValue(
                      IGF.IGM,
                      getFixedPayloadTypeInfo().getFixedSize().getValueInBits(),
                      inhabitant);
          swi->addCase(xi, falseBB);
        }
        
        IGF.Builder.emitBlock(payloadBB);
      }
      
      return falseBB;
    }
    
    /// Emits the test(s) that determine whether the enum contains a payload
    /// or an empty case. For a fixed-size enum, this does a primitive load
    /// of the representation and calls down to testFixedEnumContainsPayload.
    /// For a dynamic enum, this queries the value witness table of the payload
    /// type. Emits the basic block for the "true" case and
    /// returns the unemitted basic block for the "false" case.
    llvm::BasicBlock *
    testEnumContainsPayload(IRGenFunction &IGF,
                             Address addr) const {
      auto &C = IGF.IGM.getLLVMContext();
      
      if (TIK >= Fixed) {
        llvm::Value *payload, *extraTag;
        std::tie(payload, extraTag)
          = emitPrimitiveLoadPayloadAndExtraTag(IGF, addr);
        return testFixedEnumContainsPayload(IGF, payload, extraTag);
      }

      auto *payloadBB = llvm::BasicBlock::Create(C);
      auto *noPayloadBB = llvm::BasicBlock::Create(C);

      // Look up the metadata for the payload.
      llvm::Value *metadata = emitPayloadMetadata(IGF);

      // Ask the runtime what case we have.
      llvm::Value *opaqueAddr = IGF.Builder.CreateBitCast(addr.getAddress(),
                                                          IGF.IGM.OpaquePtrTy);
      llvm::Value *numCases = llvm::ConstantInt::get(IGF.IGM.Int32Ty,
                                                 ElementsWithNoPayload.size());
      llvm::Value *which = IGF.Builder.CreateCall3(
                                       IGF.IGM.getGetEnumCaseSinglePayloadFn(),
                                       opaqueAddr, metadata, numCases);
      
      // If it's -1 then we have the payload.
      llvm::Value *hasPayload = IGF.Builder.CreateICmpEQ(which,
                             llvm::ConstantInt::getSigned(IGF.IGM.Int32Ty, -1));
      IGF.Builder.CreateCondBr(hasPayload, payloadBB, noPayloadBB);
      
      IGF.Builder.emitBlock(payloadBB);
      return noPayloadBB;
    }
    
  public:
    void copy(IRGenFunction &IGF, Explosion &src, Explosion &dest)
    const override {
      assert(TIK >= Loadable);
      
      switch (CopyDestroyKind) {
      case POD:
        reexplode(IGF, src, dest);
        return;

      case Normal: {
        // Copy the payload, if we have it.
        llvm::Value *payload, *extraTag;
        std::tie(payload, extraTag) = getPayloadAndExtraTagFromExplosion(src);
        
        llvm::BasicBlock *endBB = testFixedEnumContainsPayload(IGF, payload, extraTag);
        
        if (payload) {
          Explosion payloadValue(ExplosionKind::Minimal);
          Explosion payloadCopy(ExplosionKind::Minimal);
          auto &loadableTI = getLoadablePayloadTypeInfo();
          loadableTI.unpackEnumPayload(IGF, payload, payloadValue, 0);
          loadableTI.copy(IGF, payloadValue, payloadCopy);
          payloadCopy.claimAll(); // FIXME: repack if not bit-identical
        }
        
        IGF.Builder.CreateBr(endBB);
        IGF.Builder.emitBlock(endBB);
        
        // Copy to the new explosion.
        if (payload)
          dest.add(payload);
        if (extraTag) dest.add(extraTag);
        return;
      }
      
      case NullableSwiftRefcounted: {
        // Bitcast to swift.refcounted*, and hand to swift_retain.
        llvm::Value *val = src.claimNext();
        llvm::Value *ptr
          = IGF.Builder.CreateIntToPtr(val, IGF.IGM.RefCountedPtrTy);
        IGF.emitRetainCall(ptr);
        dest.add(val);
        return;
      }
      }
    }
    
    void consume(IRGenFunction &IGF, Explosion &src) const override {
      assert(TIK >= Loadable);

      switch (CopyDestroyKind) {
      case POD:
        src.claim(getExplosionSize(src.getKind()));
        return;

      case Normal: {
        // Check that we have a payload.
        llvm::Value *payload, *extraTag;
        std::tie(payload, extraTag) = getPayloadAndExtraTagFromExplosion(src);
        
        llvm::BasicBlock *endBB
          = testFixedEnumContainsPayload(IGF, payload, extraTag);
        
        // If we did, consume it.
        if (payload) {
          Explosion payloadValue(ExplosionKind::Minimal);
          auto &loadableTI = getLoadablePayloadTypeInfo();
          loadableTI.unpackEnumPayload(IGF, payload, payloadValue, 0);
          loadableTI.consume(IGF, payloadValue);
        }
        
        IGF.Builder.CreateBr(endBB);
        IGF.Builder.emitBlock(endBB);
        return;
      }

      case NullableSwiftRefcounted: {
        // Bitcast to swift.refcounted*, and hand to swift_release.
        llvm::Value *val = src.claimNext();
        llvm::Value *ptr
          = IGF.Builder.CreateIntToPtr(val, IGF.IGM.RefCountedPtrTy);
        IGF.emitRelease(ptr);
        return;
      }
      }
      
    }

    void destroy(IRGenFunction &IGF, Address addr) const override {
      switch (CopyDestroyKind) {
      case POD:
        return;
          
      case Normal: {
        // Check that there is a payload at the address.
        llvm::BasicBlock *endBB = testEnumContainsPayload(IGF, addr);
        
        // If there is, project and destroy it.
        Address payloadAddr = projectPayloadData(IGF, addr);
        getPayloadTypeInfo().destroy(IGF, payloadAddr);
        
        IGF.Builder.CreateBr(endBB);
        IGF.Builder.emitBlock(endBB);
        return;
      }
          
      case NullableSwiftRefcounted: {
        // Load the value as swift.refcounted, then hand to swift_release.
        addr = IGF.Builder.CreateBitCast(addr,
                                       IGF.IGM.RefCountedPtrTy->getPointerTo());
        llvm::Value *ptr = IGF.Builder.CreateLoad(addr);
        IGF.emitRelease(ptr);
        return;
      }
      }
    }
    
  private:
    llvm::ConstantInt *getZeroExtraTagConstant(IRGenModule &IGM) const {
      assert(TIK >= Fixed && "not fixed layout");
      assert(ExtraTagBitCount > 0 && "no extra tag bits?!");
      return llvm::ConstantInt::get(IGM.getLLVMContext(),
                                    APInt(ExtraTagBitCount, 0));
    }
    
    /// Initialize the extra tag bits, if any, to zero to indicate a payload.
    void emitInitializeExtraTagBitsForPayload(IRGenFunction &IGF,
                                              Address dest) const {
      if (TIK >= Fixed) {
        // We statically know whether we have extra tag bits.
        // Store zero directly to the fixed-layout extra tag field.
        if (ExtraTagBitCount > 0) {
          auto *zeroTag = getZeroExtraTagConstant(IGF.IGM);
          IGF.Builder.CreateStore(zeroTag, projectExtraTagBits(IGF, dest));
        }
        return;
      }
      
      // Ask the runtime to store the tag.
      llvm::Value *opaqueAddr = IGF.Builder.CreateBitCast(dest.getAddress(),
                                                          IGF.IGM.OpaquePtrTy);
      llvm::Value *metadata = emitPayloadMetadata(IGF);
      IGF.Builder.CreateCall4(IGF.IGM.getStoreEnumTagSinglePayloadFn(),
                              opaqueAddr, metadata,
                              llvm::ConstantInt::getSigned(IGF.IGM.Int32Ty, -1),
                              llvm::ConstantInt::get(IGF.IGM.Int32Ty,
                                                 ElementsWithNoPayload.size()));
    }
    
    /// Emit an reassignment sequence from an enum at one address to another.
    void emitIndirectAssign(IRGenFunction &IGF,
       Address dest, Address src,
       IsTake_t isTake)
    const {
      auto &C = IGF.IGM.getLLVMContext();
      
      switch (CopyDestroyKind) {
      case POD:
        return emitPrimitiveCopy(IGF, dest, src);
          
      case Normal: {
        llvm::BasicBlock *endBB = llvm::BasicBlock::Create(C);

        Address destData = projectPayloadData(IGF, dest);
        Address srcData = projectPayloadData(IGF, src);
        // See whether the current value at the destination has a payload.

        llvm::BasicBlock *noDestPayloadBB
          = testEnumContainsPayload(IGF, dest);
        
        // Here, the destination has a payload. Now see if the source also has
        // one.
        llvm::BasicBlock *destNoSrcPayloadBB
          = testEnumContainsPayload(IGF, src);
        
        // Here, both source and destination have payloads. Do the reassignment
        // of the payload in-place.
        if (isTake)
          getPayloadTypeInfo().assignWithTake(IGF, destData, srcData);
        else
          getPayloadTypeInfo().assignWithCopy(IGF, destData, srcData);
        IGF.Builder.CreateBr(endBB);
        
        // If the destination has a payload but the source doesn't, we can destroy
        // the payload and primitive-store the new no-payload value.
        IGF.Builder.emitBlock(destNoSrcPayloadBB);
        getPayloadTypeInfo().destroy(IGF, destData);
        emitPrimitiveCopy(IGF, dest, src);
        IGF.Builder.CreateBr(endBB);
        
        // Now, if the destination has no payload, check if the source has one.
        IGF.Builder.emitBlock(noDestPayloadBB);
        llvm::BasicBlock *noDestNoSrcPayloadBB
          = testEnumContainsPayload(IGF, src);

        // Here, the source has a payload but the destination doesn't. We can
        // copy-initialize the source over the destination, then primitive-store
        // the zero extra tag (if any).
        if (isTake)
          getPayloadTypeInfo().initializeWithTake(IGF, destData, srcData);
        else
          getPayloadTypeInfo().initializeWithCopy(IGF, destData, srcData);
        emitInitializeExtraTagBitsForPayload(IGF, dest);
        IGF.Builder.CreateBr(endBB);
        
        // If neither destination nor source have payloads, we can just primitive-
        // store the new empty-case value.
        IGF.Builder.emitBlock(noDestNoSrcPayloadBB);
        emitPrimitiveCopy(IGF, dest, src);
        IGF.Builder.CreateBr(endBB);
        
        IGF.Builder.emitBlock(endBB);
        return;
      }
      
      case NullableSwiftRefcounted: {
        // Do the assignment as for a refcounted pointer.
        Address destAddr = IGF.Builder.CreateBitCast(dest,
                                      IGF.IGM.RefCountedPtrTy->getPointerTo());
        Address srcAddr = IGF.Builder.CreateBitCast(src,
                                       IGF.IGM.RefCountedPtrTy->getPointerTo());
        // Load the old pointer at the destination.
        llvm::Value *oldPtr = IGF.Builder.CreateLoad(destAddr);
        // Store the new pointer.
        llvm::Value *srcPtr = IGF.Builder.CreateLoad(srcAddr);
        if (!isTake)
          IGF.emitRetainCall(srcPtr);
        IGF.Builder.CreateStore(srcPtr, destAddr);
        // Release the old value.
        IGF.emitRelease(oldPtr);
        return;
      }
      }
      
    }
    
    /// Emit an initialization sequence, initializing an enum at one address
    /// with another at a different address.
    void emitIndirectInitialize(IRGenFunction &IGF,
                                Address dest, Address src, IsTake_t isTake)
    const {
      auto &C = IGF.IGM.getLLVMContext();
      
      switch (CopyDestroyKind) {
      case POD:
        return emitPrimitiveCopy(IGF, dest, src);

      case Normal: {
        llvm::BasicBlock *endBB = llvm::BasicBlock::Create(C);
        
        Address destData = projectPayloadData(IGF, dest);
        Address srcData = projectPayloadData(IGF, src);
        
        // See whether the source value has a payload.
        llvm::BasicBlock *noSrcPayloadBB
          = testEnumContainsPayload(IGF, src);
        
        // Here, the source value has a payload. Initialize the destination with
        // it, and set the extra tag if any to zero.
        if (isTake)
          getPayloadTypeInfo().initializeWithTake(IGF, destData, srcData);
        else
          getPayloadTypeInfo().initializeWithCopy(IGF, destData, srcData);
        emitInitializeExtraTagBitsForPayload(IGF, dest);
        IGF.Builder.CreateBr(endBB);
        
        // If the source value has no payload, we can primitive-store the
        // empty-case value.
        IGF.Builder.emitBlock(noSrcPayloadBB);
        emitPrimitiveCopy(IGF, dest, src);
        IGF.Builder.CreateBr(endBB);
        
        IGF.Builder.emitBlock(endBB);
        return;
      }
          
      case NullableSwiftRefcounted: {
        // Do the initialization as for a refcounted pointer.
        Address destAddr = IGF.Builder.CreateBitCast(dest,
                                      IGF.IGM.RefCountedPtrTy->getPointerTo());
        Address srcAddr = IGF.Builder.CreateBitCast(src,
                                       IGF.IGM.RefCountedPtrTy->getPointerTo());

        llvm::Value *srcPtr = IGF.Builder.CreateLoad(srcAddr);
        if (!isTake)
          IGF.emitRetainCall(srcPtr);
        IGF.Builder.CreateStore(srcPtr, destAddr);
        return;
      }
      }
    }
    
  public:
    void assignWithCopy(IRGenFunction &IGF, Address dest, Address src)
    const override {
      emitIndirectAssign(IGF, dest, src, IsNotTake);
    }
    
    void assignWithTake(IRGenFunction &IGF, Address dest, Address src)
    const override {
      emitIndirectAssign(IGF, dest, src, IsTake);
    }
    
    void initializeWithCopy(IRGenFunction &IGF, Address dest, Address src)
    const override {
      emitIndirectInitialize(IGF, dest, src, IsNotTake);
    }
    
    void initializeWithTake(IRGenFunction &IGF, Address dest, Address src)
    const override {
      emitIndirectInitialize(IGF, dest, src, IsTake);
    }
    
    void storeTag(IRGenFunction &IGF,
                  EnumElementDecl *elt,
                  Address enumAddr) const override {
      if (TIK < Fixed) {
        // If the enum isn't fixed-layout, get the runtime to do this for us.
        llvm::Value *payload = emitPayloadMetadata(IGF);
        llvm::Value *caseIndex;
        if (elt == getPayloadElement()) {
          caseIndex = llvm::ConstantInt::getSigned(IGF.IGM.Int32Ty, -1);
        } else {
          auto found = std::find_if(ElementsWithNoPayload.begin(),
                                    ElementsWithNoPayload.end(),
                                    [&](Element a) { return a.decl == elt; });
          assert(found != ElementsWithNoPayload.end() &&
                 "case not in enum?!");
          unsigned caseIndexVal = found - ElementsWithNoPayload.begin();
          caseIndex = llvm::ConstantInt::get(IGF.IGM.Int32Ty, caseIndexVal);
        }
        
        llvm::Value *numEmptyCases = llvm::ConstantInt::get(IGF.IGM.Int32Ty,
                                                  ElementsWithNoPayload.size());
        
        llvm::Value *opaqueAddr
          = IGF.Builder.CreateBitCast(enumAddr.getAddress(),
                                      IGF.IGM.OpaquePtrTy);
        
        IGF.Builder.CreateCall4(IGF.IGM.getStoreEnumTagSinglePayloadFn(),
                                opaqueAddr, payload, caseIndex, numEmptyCases);
        
        return;
      }
      
      if (elt == getPayloadElement()) {
        // The data occupies the entire payload. If we have extra tag bits,
        // zero them out.
        if (ExtraTagBitCount > 0)
          IGF.Builder.CreateStore(getZeroExtraTagConstant(IGF.IGM),
                                  projectExtraTagBits(IGF, enumAddr));
        return;
      }
      
      // Store the discriminator for the no-payload case.
      llvm::Value *payload, *extraTag;
      std::tie(payload, extraTag) = getNoPayloadCaseValue(IGF, elt);
      
      if (payloadTy)
        IGF.Builder.CreateStore(payload, projectPayload(IGF, enumAddr));
      if (ExtraTagBitCount > 0)
        IGF.Builder.CreateStore(extraTag, projectExtraTagBits(IGF, enumAddr));
    }
    
    void initializeMetadata(IRGenFunction &IGF,
                                     llvm::Value *metadata,
                                     llvm::Value *vwtable) const override
    {
      // Fixed-size enums don't need dynamic witness table initialization.
      if (TIK >= Fixed) return;

      // Ask the runtime to do our layout using the payload metadata and number
      // of empty cases.
      auto payloadMetadata = emitPayloadMetadata(IGF);
      auto emptyCasesVal = llvm::ConstantInt::get(IGF.IGM.Int32Ty,
                                                  ElementsWithNoPayload.size());
      
      IGF.Builder.CreateCall3(
                    IGF.IGM.getInitEnumValueWitnessTableSinglePayloadFn(),
                    vwtable, payloadMetadata, emptyCasesVal);
    }
    
    /// \group Extra inhabitants
    
    // Extra inhabitants from the payload that we didn't use for our empty cases
    // are available to outer enums.
    // FIXME: If we spilled extra tag bits, we could offer spare bits from the
    // tag.
    
    bool mayHaveExtraInhabitants(IRGenModule &IGM) const override {
      if (TIK >= Fixed)
        return getFixedExtraInhabitantCount(IGM) > 0;

      return getPayloadTypeInfo().mayHaveExtraInhabitants(IGM);
    }
    
    unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
      unsigned payloadXI
        = getFixedPayloadTypeInfo().getFixedExtraInhabitantCount(IGM);
      
      unsigned numEmptyCases = ElementsWithNoPayload.size();
      
      if (payloadXI <= numEmptyCases)
        return 0;
      
      return payloadXI - numEmptyCases;
    }
    
    llvm::ConstantInt *
    getFixedExtraInhabitantValue(IRGenModule &IGM,
                                 unsigned bits,
                                 unsigned index) const override {
      return getFixedPayloadTypeInfo()
        .getFixedExtraInhabitantValue(IGM, bits,
                                      index + ElementsWithNoPayload.size());
    }
    
    llvm::Value *
    getExtraInhabitantIndex(IRGenFunction &IGF,
                            Address src) const override {
      auto payload = projectPayloadData(IGF, src);
      llvm::Value *index
        = getPayloadTypeInfo().getExtraInhabitantIndex(IGF, payload);
      
      // Offset the payload extra inhabitant index by the number of inhabitants
      // we used. If less than zero, it's a valid value of the enum type.
      index = IGF.Builder.CreateSub(index,
         llvm::ConstantInt::get(IGF.IGM.Int32Ty, ElementsWithNoPayload.size()));
      auto valid = IGF.Builder.CreateICmpSLT(index,
                                   llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0));
      index = IGF.Builder.CreateSelect(valid,
                              llvm::ConstantInt::getSigned(IGF.IGM.Int32Ty, -1),
                              index);
      return index;
    }
    
    void storeExtraInhabitant(IRGenFunction &IGF,
                              llvm::Value *index,
                              Address dest) const override {
      // Offset the index to skip the extra inhabitants we used.
      index = IGF.Builder.CreateAdd(index,
          llvm::ConstantInt::get(IGF.IGM.Int32Ty, ElementsWithNoPayload.size()));
      
      auto payload = projectPayloadData(IGF, dest);
      getPayloadTypeInfo().storeExtraInhabitant(IGF, index, payload);
    }
  };

  class MultiPayloadEnumImplStrategy final
    : public PayloadEnumImplStrategyBase
  {
    // The spare bits shared by all payloads, if any.
    // Invariant: The size of the bit vector is the size of the payload in bits,
    // rounded up to a byte boundary.
    llvm::BitVector CommonSpareBits;

    // The number of tag values used for no-payload cases.
    unsigned NumEmptyElementTags = ~0u;

  public:
    MultiPayloadEnumImplStrategy(IRGenModule &IGM,
                                 TypeInfoKind tik, unsigned NumElements,
                                  std::vector<Element> &&WithPayload,
                                  std::vector<Element> &&WithRecursivePayload,
                                  std::vector<Element> &&WithNoPayload)
      : PayloadEnumImplStrategyBase(IGM, tik, NumElements,
                                     std::move(WithPayload),
                                     std::move(WithRecursivePayload),
                                     std::move(WithNoPayload))
    {
      assert(ElementsWithPayload.size() > 1);
    }
    
    TypeInfo *completeEnumTypeLayout(TypeConverter &TC,
                                      CanType type,
                                      EnumDecl *theEnum,
                                      llvm::StructType *enumTy) override;
    
  private:
    /// The number of empty cases representable by each tag value.
    /// Equal to the size of the payload minus the spare bits used for tags.
    unsigned getNumCaseBits() const {
      return CommonSpareBits.size() - CommonSpareBits.count();
    }
    
    unsigned getNumCasesPerTag() const {
      unsigned numCaseBits = getNumCaseBits();
      return numCaseBits >= 32
        ? 0x80000000 : 1 << numCaseBits;
    }
    
    /// Extract the payload-discriminating tag from a payload and optional
    /// extra tag value.
    llvm::Value *extractPayloadTag(IRGenFunction &IGF,
                                   llvm::Value *payload,
                                   llvm::Value *extraTagBits) const {
      unsigned numSpareBits = CommonSpareBits.count();
      llvm::Value *tag = nullptr;
      unsigned numTagBits = numSpareBits + ExtraTagBitCount;
      
      // Get the tag bits from spare bits, if any.
      if (numSpareBits > 0) {
        tag = emitGatherSpareBits(IGF, CommonSpareBits, payload, 0, numTagBits);
      }
      
      // Get the extra tag bits, if any.
      if (ExtraTagBitCount > 0) {
        assert(extraTagBits);
        if (!tag) {
          return extraTagBits;
        } else {
          extraTagBits = IGF.Builder.CreateZExt(extraTagBits, tag->getType());
          extraTagBits = IGF.Builder.CreateShl(extraTagBits,
                                               numTagBits - ExtraTagBitCount);
          return IGF.Builder.CreateOr(tag, extraTagBits);
        }
      }
      assert(!extraTagBits);
      return tag;
    }
    
  public:
    void emitValueSwitch(IRGenFunction &IGF,
                         Explosion &value,
                         ArrayRef<std::pair<EnumElementDecl*,
                                            llvm::BasicBlock*>> dests,
                         llvm::BasicBlock *defaultDest) const override {
      auto &C = IGF.IGM.getLLVMContext();
      
      // Create a map of the destination blocks for quicker lookup.
      llvm::DenseMap<EnumElementDecl*,llvm::BasicBlock*> destMap(dests.begin(),
                                                                  dests.end());
      
      // Create an unreachable branch for unreachable switch defaults.
      auto *unreachableBB = llvm::BasicBlock::Create(C);
      
      // If there was no default branch in SIL, use the unreachable branch as
      // the default.
      if (!defaultDest)
        defaultDest = unreachableBB;
      
      auto blockForCase = [&](EnumElementDecl *theCase) -> llvm::BasicBlock* {
        auto found = destMap.find(theCase);
        if (found == destMap.end())
          return defaultDest;
        else
          return found->second;
      };
      
      llvm::Value *payload = value.claimNext();
      llvm::Value *extraTagBits = nullptr;
      if (ExtraTagBitCount > 0)
        extraTagBits = value.claimNext();
      
      // Extract and switch on the tag bits.
      llvm::Value *tag = extractPayloadTag(IGF, payload, extraTagBits);
      unsigned numTagBits
        = cast<llvm::IntegerType>(tag->getType())->getBitWidth();
      
      auto *tagSwitch = IGF.Builder.CreateSwitch(tag, unreachableBB,
                             ElementsWithPayload.size() + NumEmptyElementTags);
      
      // Switch over the tag bits for payload cases.
      unsigned tagIndex = 0;
      for (auto &payloadCasePair : ElementsWithPayload) {
        EnumElementDecl *payloadCase = payloadCasePair.decl;
        tagSwitch->addCase(llvm::ConstantInt::get(C,APInt(numTagBits,tagIndex)),
                           blockForCase(payloadCase));
        ++tagIndex;
      }
      
      // Switch over the no-payload cases.
      unsigned casesPerTag = getNumCasesPerTag();
      
      auto elti = ElementsWithNoPayload.begin(),
           eltEnd = ElementsWithNoPayload.end();
      
      for (unsigned i = 0; i < NumEmptyElementTags; ++i) {
        assert(elti != eltEnd &&
               "ran out of cases before running out of extra tags?");
        auto *tagBB = llvm::BasicBlock::Create(C);
        tagSwitch->addCase(llvm::ConstantInt::get(C,APInt(numTagBits,tagIndex)),
                           tagBB);
        
        // Switch over the cases for this tag.
        IGF.Builder.emitBlock(tagBB);
        auto *caseSwitch = IGF.Builder.CreateSwitch(payload, unreachableBB);
        for (unsigned idx = 0; idx < casesPerTag && elti != eltEnd; ++idx) {
          auto v = interleaveSpareBits(IGF.IGM, CommonSpareBits,
                                       CommonSpareBits.size(),
                                       tagIndex, idx);
          caseSwitch->addCase(v, blockForCase(elti->decl));
          ++elti;
        }
        
        ++tagIndex;
      }
      
      // Delete the unreachable default block if we didn't use it, or emit it
      // if we did.
      if (unreachableBB->use_empty()) {
        delete unreachableBB;
      } else {
        IGF.Builder.emitBlock(unreachableBB);
        IGF.Builder.CreateUnreachable();
      }
    }

    void emitIndirectSwitch(IRGenFunction &IGF,
                            Address addr,
                            ArrayRef<std::pair<EnumElementDecl*,
                            llvm::BasicBlock*>> dests,
                            llvm::BasicBlock *defaultDest) const override {
      if (TIK >= Fixed) {
        // Load the fixed-size representation and switch directly.
        Explosion value(ExplosionKind::Minimal);
        loadForSwitch(IGF, addr, value);
        return emitValueSwitch(IGF, value, dests, defaultDest);
      }
      
      // Use the runtime to dynamically switch.
      llvm_unreachable("dynamic switch for multi-payload enum not implemented");
    }
    
  private:
    void projectPayloadValue(IRGenFunction &IGF,
                        llvm::Value *payload,
                        unsigned payloadTag,
                        const LoadableTypeInfo &payloadTI,
                        Explosion &out) const {
      // If we have spare bits, we have to mask out any set tag bits packed
      // there.
      if (CommonSpareBits.any()) {
        unsigned spareBitCount = CommonSpareBits.count();
        if (spareBitCount < 32)
          payloadTag &= (1U << spareBitCount) - 1U;
        if (payloadTag != 0) {
          APInt mask = ~getAPIntFromBitVector(CommonSpareBits);
          auto maskVal = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(),
                                                mask);
          payload = IGF.Builder.CreateAnd(payload, maskVal);
        }
      }
      
      // Unpack the payload.
      payloadTI.unpackEnumPayload(IGF, payload, out, 0);
    }
    
  public:
    void emitValueProject(IRGenFunction &IGF,
                          Explosion &inValue,
                          EnumElementDecl *theCase,
                          Explosion &out) const override {
      auto foundPayload = std::find_if(ElementsWithPayload.begin(),
                                       ElementsWithPayload.end(),
             [&](const Element &e) { return e.decl == theCase; });
      
      // Non-payload cases project to an empty explosion.
      if (foundPayload == ElementsWithPayload.end()) {
        inValue.claim(getExplosionSize(inValue.getKind()));
        return;
      }
      
      llvm::Value *payload = inValue.claimNext();
      // We don't need the tag bits.
      if (ExtraTagBitCount > 0)
        inValue.claimNext();
      
      // Unpack the payload.
      projectPayloadValue(IGF, payload,
                          foundPayload - ElementsWithPayload.begin(),
                     cast<LoadableTypeInfo>(*foundPayload->ti), out);
    }
    
    llvm::Value *packEnumPayload(IRGenFunction &IGF, Explosion &src,
                                  unsigned bitWidth,
                                  unsigned offset) const override {
      PackEnumPayload pack(IGF, bitWidth);
      // Pack the payload.
      pack.addAtOffset(src.claimNext(), offset);
      // Pack the extra bits, if any.
      if (ExtraTagBitCount > 0) {
        pack.addAtOffset(src.claimNext(), CommonSpareBits.size() + offset);
      }
      return pack.get();
    }
    
    void unpackEnumPayload(IRGenFunction &IGF, llvm::Value *outerPayload,
                            Explosion &dest, unsigned offset) const override {
      UnpackEnumPayload unpack(IGF, outerPayload);
      // Unpack the payload.
      dest.add(unpack.claimAtOffset(payloadTy, offset));
      // Unpack the extra bits, if any.
      if (ExtraTagBitCount > 0) {
        dest.add(unpack.claimAtOffset(extraTagTy,
                                      CommonSpareBits.size() + offset));
      }
    }
    
  private:
    void emitPayloadInjection(IRGenFunction &IGF,
                              const FixedTypeInfo &payloadTI,
                              Explosion &params, Explosion &out,
                              unsigned tag) const {
      // Pack the payload.
      auto &loadablePayloadTI = cast<LoadableTypeInfo>(payloadTI); // FIXME
      llvm::Value *payload = loadablePayloadTI.packEnumPayload(IGF, params,
                                                    CommonSpareBits.size(), 0);
      
      // If we have spare bits, pack tag bits into them.
      unsigned numSpareBits = CommonSpareBits.count();
      if (numSpareBits > 0) {
        llvm::ConstantInt *tagMask
          = interleaveSpareBits(IGF.IGM, CommonSpareBits,CommonSpareBits.size(),
                                tag, 0);
        payload = IGF.Builder.CreateOr(payload, tagMask);
      }
      
      out.add(payload);
      
      // If we have extra tag bits, pack the remaining tag bits into them.
      if (ExtraTagBitCount > 0) {
        tag >>= numSpareBits;
        auto extra = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(),
                                            APInt(ExtraTagBitCount, tag));
        out.add(extra);
      }
    }
    
    std::pair<llvm::Value*, llvm::Value*>
    getNoPayloadCaseValue(IRGenFunction &IGF, unsigned index) const {
      // Figure out the tag and payload for the empty case.
      unsigned numCaseBits = getNumCaseBits();
      unsigned tag, tagIndex;
      if (numCaseBits >= 32) {
        tag = ElementsWithPayload.size();
        tagIndex = index;
      } else {
        tag = (index >> numCaseBits) + ElementsWithPayload.size();
        tagIndex = index & ((1 << numCaseBits) - 1);
      }
      
      llvm::Value *payload;
      llvm::Value *extraTag = nullptr;
      unsigned numSpareBits = CommonSpareBits.count();
      if (numSpareBits > 0) {
        // If we have spare bits, pack tag bits into them.
        payload = interleaveSpareBits(IGF.IGM,
                                      CommonSpareBits, CommonSpareBits.size(),
                                      tag, tagIndex);
      } else {
        // Otherwise the payload is just the index.
        payload = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(),
                                         APInt(CommonSpareBits.size(), tagIndex));
      }
      
      // If we have extra tag bits, pack the remaining tag bits into them.
      if (ExtraTagBitCount > 0) {
        tag >>= numSpareBits;
        extraTag = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(),
                                            APInt(ExtraTagBitCount, tag));
      }
      return {payload, extraTag};
    }
    
    void emitNoPayloadInjection(IRGenFunction &IGF, Explosion &out,
                                unsigned index) const {
      llvm::Value *payload, *extraTag;
      std::tie(payload, extraTag) = getNoPayloadCaseValue(IGF, index);
      out.add(payload);
      if (ExtraTagBitCount > 0) {
        assert(extraTag);
        out.add(extraTag);
      }
    }
    
    void forNontrivialPayloads(IRGenFunction &IGF,
               llvm::Value *payload, llvm::Value *extraTagBits,
               std::function<void (unsigned, const TypeInfo &)> f) const
    {
      auto *endBB = llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
      
      llvm::Value *tag = extractPayloadTag(IGF, payload, extraTagBits);
      auto *swi = IGF.Builder.CreateSwitch(tag, endBB);
      auto *tagTy = cast<llvm::IntegerType>(tag->getType());
      
      // Handle nontrivial tags.
      unsigned tagIndex = 0;
      for (auto &payloadCasePair : ElementsWithPayload) {
        auto &payloadTI = *payloadCasePair.ti;
        
        // Trivial payloads don't need any work.
        if (payloadTI.isPOD(ResilienceScope::Local)) {
          ++tagIndex;
          continue;
        }
        
        // Unpack and handle nontrivial payloads.
        auto *caseBB = llvm::BasicBlock::Create(IGF.IGM.getLLVMContext());
        swi->addCase(llvm::ConstantInt::get(tagTy, tagIndex), caseBB);
        
        IGF.Builder.emitBlock(caseBB);
        f(tagIndex, payloadTI);
        IGF.Builder.CreateBr(endBB);
        
        ++tagIndex;
      }
      
      IGF.Builder.emitBlock(endBB);
    }
    
  public:
    void emitValueInjection(IRGenFunction &IGF,
                            EnumElementDecl *elt,
                            Explosion &params,
                            Explosion &out) const {
      // See whether this is a payload or empty case we're emitting.
      auto payloadI = std::find_if(ElementsWithPayload.begin(),
                                   ElementsWithPayload.end(),
                               [&](const Element &e) { return e.decl == elt; });
      if (payloadI != ElementsWithPayload.end())
        return emitPayloadInjection(IGF, cast<FixedTypeInfo>(*payloadI->ti),
                                    params, out,
                                    payloadI - ElementsWithPayload.begin());

      auto emptyI = std::find_if(ElementsWithNoPayload.begin(),
                                 ElementsWithNoPayload.end(),
                               [&](const Element &e) { return e.decl == elt; });
      assert(emptyI != ElementsWithNoPayload.end() && "case not in enum");
      emitNoPayloadInjection(IGF, out, emptyI - ElementsWithNoPayload.begin());
    }
    
    void copy(IRGenFunction &IGF, Explosion &src, Explosion &dest)
    const override {
      if (isPOD(ResilienceScope::Local)) {
        reexplode(IGF, src, dest);
        return;
      }
      
      llvm::Value *payload = src.claimNext();
      llvm::Value *extraTagBits = ExtraTagBitCount > 0
        ? src.claimNext() : nullptr;
      
      forNontrivialPayloads(IGF, payload, extraTagBits,
        [&](unsigned tagIndex, const TypeInfo &ti) {
          auto &lti = cast<LoadableTypeInfo>(ti);
          Explosion value(ExplosionKind::Minimal);
          projectPayloadValue(IGF, payload, tagIndex, lti, value);

          Explosion tmp(value.getKind());
          lti.copy(IGF, value, tmp);
          tmp.claimAll(); // FIXME: repack if not bit-identical
        });
      
      dest.add(payload);
      if (extraTagBits)
        dest.add(extraTagBits);
    }
    
    void consume(IRGenFunction &IGF, Explosion &src) const override {
      if (isPOD(ResilienceScope::Local)) {
        src.claim(getExplosionSize(src.getKind()));
        return;
      }
      
      llvm::Value *payload = src.claimNext();
      llvm::Value *extraTagBits = ExtraTagBitCount > 0
        ? src.claimNext() : nullptr;
      
      forNontrivialPayloads(IGF, payload, extraTagBits,
        [&](unsigned tagIndex, const TypeInfo &ti) {
          auto &lti = cast<LoadableTypeInfo>(ti);
          Explosion value(ExplosionKind::Minimal);
          projectPayloadValue(IGF, payload, tagIndex, lti, value);

          lti.consume(IGF, value);
        });
    }
  
  private:
    /// Emit an reassignment sequence from an enum at one address to another.
    void emitIndirectAssign(IRGenFunction &IGF,
                            Address dest, Address src, IsTake_t isTake) const {
      auto &C = IGF.IGM.getLLVMContext();
      
      if (isPOD(ResilienceScope::Local))
        return emitPrimitiveCopy(IGF, dest, src);
      
      // If the enum is loadable, it's better to do this directly using values,
      // so we don't need to RMW tag bits in place.
      if (TI->isLoadable()) {
        Explosion tmpSrc(ExplosionKind::Minimal),
                  tmpOld(ExplosionKind::Minimal);
        if (isTake)
          loadAsTake(IGF, src, tmpSrc);
        else
          loadAsCopy(IGF, src, tmpSrc);
        
        loadAsTake(IGF, dest, tmpOld);
        initialize(IGF, tmpSrc, dest);
        consume(IGF, tmpOld);
        return;
      }
      
      auto *endBB = llvm::BasicBlock::Create(C);
      
      // Sanity-check whether the source and destination alias.
      llvm::Value *alias = IGF.Builder.CreateICmpEQ(dest.getAddress(),
                                                    src.getAddress());
      auto *noAliasBB = llvm::BasicBlock::Create(C);
      IGF.Builder.CreateCondBr(alias, endBB, noAliasBB);
      IGF.Builder.emitBlock(noAliasBB);
      
      // Destroy the old value.
      destroy(IGF, dest);
      
      // Reinitialize with the new value.
      emitIndirectInitialize(IGF, dest, src, isTake);
      
      IGF.Builder.CreateBr(endBB);
      IGF.Builder.emitBlock(endBB);
    }
    
    void emitIndirectInitialize(IRGenFunction &IGF,
                              Address dest, Address src, IsTake_t isTake) const{
      auto &C = IGF.IGM.getLLVMContext();

      if (isPOD(ResilienceScope::Local))
        return emitPrimitiveCopy(IGF, dest, src);
      
      // If the enum is loadable, it's better to do this directly using values,
      // so we don't need to RMW tag bits in place.
      if (TI->isLoadable()) {
        Explosion tmpSrc(ExplosionKind::Minimal);
        if (isTake)
          loadAsTake(IGF, src, tmpSrc);
        else
          loadAsCopy(IGF, src, tmpSrc);
        initialize(IGF, tmpSrc, dest);
        return;
      }

      llvm::Value *payload, *extraTagBits;
      std::tie(payload, extraTagBits)
        = emitPrimitiveLoadPayloadAndExtraTag(IGF, src);

      auto *endBB = llvm::BasicBlock::Create(C);

      /// Switch out nontrivial payloads.
      auto *trivialBB = llvm::BasicBlock::Create(C);
      llvm::Value *tag = extractPayloadTag(IGF, payload, extraTagBits);
      auto *swi = IGF.Builder.CreateSwitch(tag, trivialBB);
      auto *tagTy = cast<llvm::IntegerType>(tag->getType());
      
      unsigned tagIndex = 0;
      for (auto &payloadCasePair : ElementsWithPayload) {
        auto &payloadTI = *payloadCasePair.ti;
        // Trivial payloads can all share the default path.
        if (payloadTI.isPOD(ResilienceScope::Local)) {
          ++tagIndex;
          continue;
        }
        
        // For nontrivial payloads, we need to copy/take the payload using its
        // value semantics.
        auto *caseBB = llvm::BasicBlock::Create(C);
        swi->addCase(llvm::ConstantInt::get(tagTy, tagIndex), caseBB);
        IGF.Builder.emitBlock(caseBB);

        // Temporarily clear the tag bits from the source so we can use the
        // data.
        preparePayloadForLoad(IGF, src, tagIndex);
        
        // Do the take/copy of the payload.
        Address srcData = IGF.Builder.CreateBitCast(src,
                                payloadTI.getStorageType()->getPointerTo());
        Address destData = IGF.Builder.CreateBitCast(dest,
                                  payloadTI.getStorageType()->getPointerTo());

        if (isTake) {
          payloadTI.initializeWithTake(IGF, destData, srcData);
          // We don't need to preserve the old value.
        } else {
          payloadTI.initializeWithCopy(IGF, destData, srcData);
          // Replant the tag bits, if any, in the source.
          storePayloadTag(IGF, src, tagIndex);
        }
        
        // Plant spare bit tag bits, if any, into the new value.
        storePayloadTag(IGF, dest, tagIndex);
        IGF.Builder.CreateBr(endBB);
        
        ++tagIndex;
      }
      
      // For trivial payloads (including no-payload cases), we can just
      // primitive-store to the destination.
      IGF.Builder.emitBlock(trivialBB);
      emitPrimitiveStorePayloadAndExtraTag(IGF, dest, payload, extraTagBits);
      IGF.Builder.CreateBr(endBB);
      
      IGF.Builder.emitBlock(endBB);
    }
    
  public:
    void assignWithCopy(IRGenFunction &IGF, Address dest, Address src)
    const override {
      emitIndirectAssign(IGF, dest, src, IsNotTake);
    }
    
    void assignWithTake(IRGenFunction &IGF, Address dest, Address src)
    const override {
      emitIndirectAssign(IGF, dest, src, IsTake);
    }
    
    void initializeWithCopy(IRGenFunction &IGF, Address dest, Address src)
    const override {
      emitIndirectInitialize(IGF, dest, src, IsNotTake);
    }
    
    void initializeWithTake(IRGenFunction &IGF, Address dest, Address src)
    const override {
      emitIndirectInitialize(IGF, dest, src, IsTake);
    }
    
  private:
    /// Clear any tag bits stored in the payload area of the given address.
    void preparePayloadForLoad(IRGenFunction &IGF, Address enumAddr,
                               unsigned tagIndex) const {
      // If the case has non-zero tag bits stored in spare bits, we need to
      // mask them out before the data can be read.
      unsigned numSpareBits = CommonSpareBits.count();
      if (numSpareBits > 0) {
        unsigned spareTagBits = numSpareBits >= 32
          ? tagIndex : tagIndex & ((1U << numSpareBits) - 1U);
        
        if (spareTagBits != 0) {
          assert(payloadTy && "spare bits with empty payload?!");
          Address payloadAddr = projectPayload(IGF, enumAddr);
          llvm::Value *payloadBits = IGF.Builder.CreateLoad(payloadAddr);
          auto *spareBitMask = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(),
                                                      ~getAPIntFromBitVector(CommonSpareBits));
          payloadBits = IGF.Builder.CreateAnd(payloadBits, spareBitMask);
          IGF.Builder.CreateStore(payloadBits, payloadAddr);
        }
      }
    }
    
    void destroy(IRGenFunction &IGF, Address addr) const override {
      if (isPOD(ResilienceScope::Local))
        return;
      
      // If loadable, it's better to do this directly to the value than
      // in place, so we don't need to RMW out the tag bits in memory.
      if (TI->isLoadable()) {
        Explosion tmp(ExplosionKind::Minimal);
        loadAsTake(IGF, addr, tmp);
        consume(IGF, tmp);
        return;
      }
      
      llvm::Value *payload, *extraTagBits;
      std::tie(payload, extraTagBits)
        = emitPrimitiveLoadPayloadAndExtraTag(IGF, addr);
      
      forNontrivialPayloads(IGF, payload, extraTagBits,
        [&](unsigned tagIndex, const TypeInfo &ti) {
          // Clear tag bits out of the payload area, if any.
          preparePayloadForLoad(IGF, addr, tagIndex);
          // Destroy the data.
          Address dataAddr = IGF.Builder.CreateBitCast(addr,
                                        ti.getStorageType()->getPointerTo());
          ti.destroy(IGF, dataAddr);
        });
    }
    
    Address projectDataForStore(IRGenFunction &IGF,
                                EnumElementDecl *elt,
                                Address enumAddr) const override {
      auto payloadI = std::find_if(ElementsWithPayload.begin(),
                                   ElementsWithPayload.end(),
         [&](const Element &e) { return e.decl == elt; });

      assert(payloadI != ElementsWithPayload.end() &&
             "cannot project a no-payload case");
      
      // Payloads are all placed at the beginning of the value.
      return IGF.Builder.CreateBitCast(enumAddr,
                               payloadI->ti->getStorageType()->getPointerTo());
    }

  private:
    void storePayloadTag(IRGenFunction &IGF,
                         Address enumAddr, unsigned index) const {
      // If the tag has spare bits, we need to mask them into the
      // payload area.
      unsigned numSpareBits = CommonSpareBits.count();
      if (numSpareBits > 0) {
        unsigned spareTagBits = numSpareBits >= 32
          ? index : index & ((1U << numSpareBits) - 1U);

        // Mask the spare bits into the payload area.
        assert(payloadTy && "spare bits with empty payload?!");
        Address payloadAddr = projectPayload(IGF, enumAddr);
        llvm::Value *payloadBits = IGF.Builder.CreateLoad(payloadAddr);
        auto *spareBitMask = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(),
                                      ~getAPIntFromBitVector(CommonSpareBits));
        llvm::Value *tagBitMask
          = interleaveSpareBits(IGF.IGM, CommonSpareBits, CommonSpareBits.size(),
                                spareTagBits, 0);
        payloadBits = IGF.Builder.CreateAnd(payloadBits, spareBitMask);
        if (spareTagBits != 0)
          payloadBits = IGF.Builder.CreateOr(payloadBits, tagBitMask);
        IGF.Builder.CreateStore(payloadBits, payloadAddr);
      }

      // Initialize the extra tag bits, if we have them.
      if (ExtraTagBitCount > 0) {
        unsigned extraTagBits = index >> numSpareBits;
        auto *extraTagValue = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(),
                                        APInt(ExtraTagBitCount, extraTagBits));
        IGF.Builder.CreateStore(extraTagValue,
                                projectExtraTagBits(IGF, enumAddr));
      }
    }
    
    void storeNoPayloadTag(IRGenFunction &IGF, Address enumAddr,
                           unsigned index) const {
      // We can just primitive-store the representation for the empty case.
      llvm::Value *payload, *extraTag;
      std::tie(payload, extraTag) = getNoPayloadCaseValue(IGF, index);
      if (payloadTy)
        IGF.Builder.CreateStore(payload, projectPayload(IGF, enumAddr));
      if (ExtraTagBitCount > 0) {
        assert(extraTag);
        IGF.Builder.CreateStore(extraTag, projectExtraTagBits(IGF, enumAddr));
      }
    }
    
  public:
    void storeTag(IRGenFunction &IGF,
                  EnumElementDecl *elt,
                  Address enumAddr) const override {
      // See whether this is a payload or empty case we're emitting.
      auto payloadI = std::find_if(ElementsWithPayload.begin(),
                                   ElementsWithPayload.end(),
                               [&](const Element &e) { return e.decl == elt; });
      if (payloadI != ElementsWithPayload.end())
        return storePayloadTag(IGF,
                               enumAddr,
                               payloadI - ElementsWithPayload.begin());
      
      auto emptyI = std::find_if(ElementsWithNoPayload.begin(),
                                 ElementsWithNoPayload.end(),
                               [&](const Element &e) { return e.decl == elt; });
      assert(emptyI != ElementsWithNoPayload.end() && "case not in enum");
      storeNoPayloadTag(IGF, enumAddr, emptyI - ElementsWithNoPayload.begin());
    }
    
    Address destructiveProjectDataForLoad(IRGenFunction &IGF,
                                          EnumElementDecl *elt,
                                          Address enumAddr) const override {
      auto payloadI = std::find_if(ElementsWithPayload.begin(),
                               ElementsWithPayload.end(),
                               [&](const Element &e) { return e.decl == elt; });
      
      assert(payloadI != ElementsWithPayload.end() &&
             "cannot project a no-payload case");
      
      unsigned index = payloadI - ElementsWithPayload.begin();
      
      preparePayloadForLoad(IGF, enumAddr, index);
      
      // Payloads are all placed at the beginning of the value.
      return IGF.Builder.CreateBitCast(enumAddr,
                               payloadI->ti->getStorageType()->getPointerTo());
    }

    void initializeMetadata(IRGenFunction &IGF,
                                     llvm::Value *metadata,
                                     llvm::Value *vwtable) const override {
      // FIXME
    }
    
    /// \group Extra inhabitants
    
    // TODO
    
    bool mayHaveExtraInhabitants(IRGenModule &) const override { return false; }
    
    llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF,
                                         Address src) const override {
      llvm_unreachable("extra inhabitants for multi-payload enums not implemented");
    }
    
    void storeExtraInhabitant(IRGenFunction &IGF,
                              llvm::Value *index,
                              Address dest) const override {
      llvm_unreachable("extra inhabitants for multi-payload enums not implemented");
    }
    
    unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
      return 0;
    }
    
    llvm::ConstantInt *
    getFixedExtraInhabitantValue(IRGenModule &IGM,
                                 unsigned bits,
                                 unsigned index) const override {
      llvm_unreachable("extra inhabitants for multi-payload enums not implemented");
    }
  };
} // end anonymous namespace

EnumImplStrategy *EnumImplStrategy::get(TypeConverter &TC,
                                          CanType type,
                                          EnumDecl *theEnum)
{
  unsigned numElements = 0;
  TypeInfoKind tik = Loadable;
  std::vector<Element> elementsWithPayload;
  std::vector<Element> elementsWithRecursivePayload;
  std::vector<Element> elementsWithNoPayload;
  
  for (auto elt : theEnum->getAllElements()) {
    numElements++;
    
    // Compute whether this gives us an apparent payload or dynamic layout.
    // Note that we do *not* apply substitutions from a bound generic instance
    // yet. We want all instances of a generic enum to share an implementation
    // strategy.
    Type argType = elt->getArgumentType();
    if (argType.isNull()) {
      elementsWithNoPayload.push_back({elt, nullptr});
      continue;
    }
    auto *argTI = TC.tryGetCompleteTypeInfo(argType->getCanonicalType());
    if (!argTI) {
      elementsWithRecursivePayload.push_back({elt, nullptr});
      continue;
    }

    auto loadableArgTI = dyn_cast<LoadableTypeInfo>(argTI);
    if (loadableArgTI
        && loadableArgTI->getExplosionSize(ExplosionKind::Minimal) == 0) {
      elementsWithNoPayload.push_back({elt, nullptr});
    } else {
      // *Now* apply the substitutions and get the type info for the instance's
      // payload type, since we know this case carries an apparent payload in
      // the generic case.
      auto *substArgTI = argTI;
      if (type->is<BoundGenericType>()) {
        Type substArgTy = type->getTypeOfMember(theEnum->getModuleContext(),
                                                elt, nullptr,
                                                elt->getArgumentType());
        substArgTI = &TC.getCompleteTypeInfo(substArgTy->getCanonicalType());
      }
      
      elementsWithPayload.push_back({elt, substArgTI});
      if (!substArgTI->isFixedSize())
        tik = Opaque;
      else if (!substArgTI->isLoadable() && tik > Fixed)
        tik = Fixed;
    }
  }

  // FIXME recursive enums
  if (!elementsWithRecursivePayload.empty()) {
    TC.IGM.unimplemented(theEnum->getLoc(), "recursive enum layout");
    exit(1);
  }
  
  assert(numElements == elementsWithPayload.size()
           + elementsWithRecursivePayload.size()
           + elementsWithNoPayload.size()
         && "not all elements accounted for");
  
  // Enums from Clang use C-compatible layout.
  if (theEnum->hasClangNode()) {
    assert(elementsWithPayload.size() == 0 && "C enum with payload?!");
    return new CCompatibleEnumImplStrategy(TC.IGM, tik, numElements,
                                       std::move(elementsWithPayload),
                                       std::move(elementsWithRecursivePayload),
                                       std::move(elementsWithNoPayload));
  }
  
  if (numElements <= 1)
    return new SingletonEnumImplStrategy(TC.IGM, tik, numElements,
                                    std::move(elementsWithPayload),
                                    std::move(elementsWithRecursivePayload),
                                    std::move(elementsWithNoPayload));
  if (elementsWithPayload.size() > 1)
    return new MultiPayloadEnumImplStrategy(TC.IGM, tik, numElements,
                                    std::move(elementsWithPayload),
                                    std::move(elementsWithRecursivePayload),
                                    std::move(elementsWithNoPayload));
  if (elementsWithPayload.size() == 1)
    return new SinglePayloadEnumImplStrategy(TC.IGM, tik, numElements,
                                    std::move(elementsWithPayload),
                                    std::move(elementsWithRecursivePayload),
                                    std::move(elementsWithNoPayload));

  return new NoPayloadEnumImplStrategy(TC.IGM, tik, numElements,
                                      std::move(elementsWithPayload),
                                      std::move(elementsWithRecursivePayload),
                                      std::move(elementsWithNoPayload));
}

namespace {
  /// Common base template for enum type infos.
  template<typename BaseTypeInfo>
  class EnumTypeInfoBase : public BaseTypeInfo {
  public:
    EnumImplStrategy &Strategy;
    
    template<typename...AA>
    EnumTypeInfoBase(EnumImplStrategy &strategy, AA &&...args)
      : BaseTypeInfo(std::forward<AA>(args)...), Strategy(strategy) {}
    
    llvm::StructType *getStorageType() const {
      return cast<llvm::StructType>(TypeInfo::getStorageType());
    }
    
    /// \group Methods delegated to the EnumImplStrategy
    
    void getSchema(ExplosionSchema &s) const override {
      return Strategy.getSchema(s);
    }
    void destroy(IRGenFunction &IGF, Address addr) const override {
      return Strategy.destroy(IGF, addr);
    }
    bool isIndirectArgument(ExplosionKind kind) const override {
      return Strategy.isIndirectArgument(kind);
    }
    void initializeFromParams(IRGenFunction &IGF, Explosion &params,
                              Address dest) const override {
      return Strategy.initializeFromParams(IGF, params, dest);
    }
    void initializeWithCopy(IRGenFunction &IGF, Address dest,
                            Address src) const override {
      return Strategy.initializeWithCopy(IGF, dest, src);
    }
    void initializeWithTake(IRGenFunction &IGF, Address dest,
                            Address src) const override {
      return Strategy.initializeWithTake(IGF, dest, src);
    }
    void assignWithCopy(IRGenFunction &IGF, Address dest,
                        Address src) const override {
      return Strategy.assignWithCopy(IGF, dest, src);
    }
    void assignWithTake(IRGenFunction &IGF, Address dest,
                        Address src) const override {
      return Strategy.assignWithTake(IGF, dest, src);
    }
    virtual void initializeMetadata(IRGenFunction &IGF,
                                         llvm::Value *metadata,
                                         llvm::Value *vwtable) const override {
      return Strategy.initializeMetadata(IGF, metadata, vwtable);
    }
    bool mayHaveExtraInhabitants(IRGenModule &IGM) const override {
      return Strategy.mayHaveExtraInhabitants(IGM);
    }
    llvm::Value *getExtraInhabitantIndex(IRGenFunction &IGF,
                                         Address src) const override {
      return Strategy.getExtraInhabitantIndex(IGF, src);
    }
    void storeExtraInhabitant(IRGenFunction &IGF,
                              llvm::Value *index,
                              Address dest) const override {
      return Strategy.storeExtraInhabitant(IGF, index, dest);
    }
  };
  
  /// TypeInfo for fixed-layout, address-only enum types.
  class FixedEnumTypeInfo : public EnumTypeInfoBase<FixedTypeInfo> {
  public:
    FixedEnumTypeInfo(EnumImplStrategy &strategy,
                       llvm::StructType *T, Size S, llvm::BitVector SB,
                       Alignment A, IsPOD_t isPOD)
      : EnumTypeInfoBase(strategy, T, S, std::move(SB), A, isPOD) {}
    
    /// \group Methods delegated to the EnumImplStrategy

    unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
      return Strategy.getFixedExtraInhabitantCount(IGM);
    }

    llvm::ConstantInt *getFixedExtraInhabitantValue(IRGenModule &IGM,
                                                    unsigned bits,
                                                    unsigned index)
    const override {
      return Strategy.getFixedExtraInhabitantValue(IGM, bits, index);
    }
  };

  /// TypeInfo for loadable enum types.
  class LoadableEnumTypeInfo : public EnumTypeInfoBase<LoadableTypeInfo> {
  public:
    // FIXME: Derive spare bits from element layout.
    LoadableEnumTypeInfo(EnumImplStrategy &strategy,
                          llvm::StructType *T, Size S, llvm::BitVector SB,
                          Alignment A, IsPOD_t isPOD)
      : EnumTypeInfoBase(strategy, T, S, std::move(SB), A, isPOD) {}

    unsigned getExplosionSize(ExplosionKind kind) const override {
      return Strategy.getExplosionSize(kind);
    }
    void loadAsCopy(IRGenFunction &IGF, Address addr,
                    Explosion &e) const override {
      return Strategy.loadAsCopy(IGF, addr, e);
    }
    void loadAsTake(IRGenFunction &IGF, Address addr,
                    Explosion &e) const override {
      return Strategy.loadAsTake(IGF, addr, e);
    }
    void assign(IRGenFunction &IGF, Explosion &e,
                Address addr) const override {
      return Strategy.assign(IGF, e, addr);
    }
    void initialize(IRGenFunction &IGF, Explosion &e,
                    Address addr) const override {
      return Strategy.initialize(IGF, e, addr);
    }
    void reexplode(IRGenFunction &IGF, Explosion &src,
                   Explosion &dest) const override {
      return Strategy.reexplode(IGF, src, dest);
    }
    void copy(IRGenFunction &IGF, Explosion &src,
              Explosion &dest) const override {
      return Strategy.copy(IGF, src, dest);
    }
    void consume(IRGenFunction &IGF, Explosion &src) const override {
      return Strategy.consume(IGF, src);
    }
    llvm::Value *packEnumPayload(IRGenFunction &IGF,
                                  Explosion &in,
                                  unsigned bitWidth,
                                  unsigned offset) const override {
      return Strategy.packEnumPayload(IGF, in, bitWidth, offset);
    }
    void unpackEnumPayload(IRGenFunction &IGF,
                            llvm::Value *payload,
                            Explosion &dest,
                            unsigned offset) const override {
      return Strategy.unpackEnumPayload(IGF, payload, dest, offset);
    }
    unsigned getFixedExtraInhabitantCount(IRGenModule &IGM) const override {
      return Strategy.getFixedExtraInhabitantCount(IGM);
    }
    
    llvm::ConstantInt *getFixedExtraInhabitantValue(IRGenModule &IGM,
                                                    unsigned bits,
                                                    unsigned index)
    const override {
      return Strategy.getFixedExtraInhabitantValue(IGM, bits, index);
    }
  };
  
  /// TypeInfo for dynamically-sized enum types.
  class NonFixedEnumTypeInfo
    : public EnumTypeInfoBase<WitnessSizedTypeInfo<NonFixedEnumTypeInfo>>
  {
    CanType EnumType;
  public:
    NonFixedEnumTypeInfo(EnumImplStrategy &strategy,
                          llvm::Type *irTy, CanType enumTy,
                          Alignment align, IsPOD_t pod)
      : EnumTypeInfoBase(strategy, irTy, align, pod), EnumType(enumTy) {}
    
    llvm::Value *getMetadataRef(IRGenFunction &IGF) const {
      return IGF.emitTypeMetadataRef(EnumType);
    }
    
    llvm::Value *getValueWitnessTable(IRGenFunction &IGF) const {
      auto metadata = getMetadataRef(IGF);
      return IGF.emitValueWitnessTableRefForMetadata(metadata);
    }

  };
  
  static const EnumImplStrategy &getEnumImplStrategy(IRGenModule &IGM,
                                                       SILType ty) {
    assert(ty.getEnumOrBoundGenericEnum() && "not an enum");
    auto *ti = &IGM.getTypeInfo(ty);
    if (auto *loadableTI = dyn_cast<LoadableTypeInfo>(ti))
      return loadableTI->as<LoadableEnumTypeInfo>().Strategy;
    if (auto *fti = dyn_cast<FixedTypeInfo>(ti))
      return fti->as<FixedEnumTypeInfo>().Strategy;
    return ti->as<NonFixedEnumTypeInfo>().Strategy;
  }
} // end anonymous namespace

TypeInfo *
EnumImplStrategy::getFixedEnumTypeInfo(llvm::StructType *T, Size S,
                                    llvm::BitVector SB,
                                    Alignment A, IsPOD_t isPOD) {
  TypeInfo *mutableTI;
  switch (TIK) {
  case Opaque:
    llvm_unreachable("not valid");
  case Fixed:
    mutableTI = new FixedEnumTypeInfo(*this, T, S, SB, A, isPOD);
    break;
  case Loadable:
    mutableTI = new LoadableEnumTypeInfo(*this, T, S, SB, A, isPOD);
    break;
  }
  TI = mutableTI;
  return mutableTI;
}

namespace {
  TypeInfo *
  SingletonEnumImplStrategy::completeEnumTypeLayout(TypeConverter &TC,
                                                   CanType type,
                                                   EnumDecl *theEnum,
                                                   llvm::StructType *enumTy) {
    if (ElementsWithPayload.empty()) {
      enumTy->setBody(ArrayRef<llvm::Type*>{}, /*isPacked*/ true);
      return registerEnumTypeInfo(new LoadableEnumTypeInfo(*this, enumTy,
                                                             Size(0), {},
                                                             Alignment(1),
                                                             IsPOD));
    } else {
      const TypeInfo &eltTI = *getSingleton();

      // Use the singleton element's storage type if fixed-size.
      if (eltTI.isFixedSize()) {
        llvm::Type *body[] = { eltTI.StorageType };
        enumTy->setBody(body, /*isPacked*/ true);
      } else {
        enumTy->setBody(ArrayRef<llvm::Type*>{}, /*isPacked*/ true);
      }
      
      if (TIK <= Opaque) {
        return registerEnumTypeInfo(new NonFixedEnumTypeInfo(*this, enumTy,
                                         type,
                                         eltTI.getBestKnownAlignment(),
                                         eltTI.isPOD(ResilienceScope::Local)));
      } else {
        auto &fixedEltTI = cast<FixedTypeInfo>(eltTI);
        return getFixedEnumTypeInfo(enumTy,
                                fixedEltTI.getFixedSize(),
                                fixedEltTI.getSpareBits(),
                                fixedEltTI.getFixedAlignment(),
                                fixedEltTI.isPOD(ResilienceScope::Local));
      }
    }
  }
  
  TypeInfo *
  NoPayloadEnumImplStrategy::completeEnumTypeLayout(TypeConverter &TC,
                                                    CanType type,
                                                    EnumDecl *theEnum,
                                                    llvm::StructType *enumTy) {
    // Since there are no payloads, we need just enough bits to hold a
    // discriminator.
    unsigned tagBits = llvm::Log2_32(ElementsWithNoPayload.size() - 1) + 1;
    auto tagTy = llvm::IntegerType::get(TC.IGM.getLLVMContext(), tagBits);
    // Round the physical size up to the next power of two.
    unsigned tagBytes = (tagBits + 7U)/8U;
    if (!llvm::isPowerOf2_32(tagBytes))
      tagBytes = llvm::NextPowerOf2(tagBytes);
    Size tagSize(tagBytes);
    
    llvm::Type *body[] = { tagTy };
    enumTy->setBody(body, /*isPacked*/true);
    
    // Unused tag bits in the physical size can be used as spare bits.
    // TODO: We can use all values greater than the largest discriminator as
    // extra inhabitants, not just those made available by spare bits.
    llvm::BitVector spareBits(tagBits, false);
    spareBits.resize(tagSize.getValueInBits(), true);
    
    return registerEnumTypeInfo(new LoadableEnumTypeInfo(*this,
                                     enumTy, tagSize, std::move(spareBits),
                                     Alignment(tagBytes), IsPOD));
  }
  
  TypeInfo *
  CCompatibleEnumImplStrategy::completeEnumTypeLayout(TypeConverter &TC,
                                                      CanType type,
                                                      EnumDecl *theEnum,
                                                      llvm::StructType *enumTy){
    // The type should have come from Clang and should have a raw type.
    assert(theEnum->hasClangNode()
           && "c-compatible enum didn't come from clang!");
    assert(theEnum->hasRawType()
           && "c-compatible enum doesn't have raw type!");
    assert(!theEnum->getDeclaredTypeInContext()->is<BoundGenericType>()
           && "c-compatible enum is generic!");
    
    // The raw type should be a C integer type, which should have a single
    // scalar representation as a Swift struct. We'll use that same
    // representation type for the enum so that it's ABI-compatible.
    auto &rawTI = TC.getCompleteTypeInfo(
                                     theEnum->getRawType()->getCanonicalType());
    auto &rawFixedTI = cast<FixedTypeInfo>(rawTI);
    assert(rawFixedTI.isPOD(ResilienceScope::Component)
           && "c-compatible raw type isn't POD?!");
    ExplosionSchema rawSchema = rawTI.getSchema(ExplosionKind::Minimal);
    assert(rawSchema.size() == 1
           && "c-compatible raw type has non-single-scalar representation?!");
    assert(rawSchema.begin()[0].isScalar()
           && "c-compatible raw type has non-single-scalar representation?!");
    llvm::Type *tagTy = rawSchema.begin()[0].getScalarType();
    
    llvm::Type *body[] = { tagTy };
    enumTy->setBody(body, /*isPacked*/ false);
    
    return registerEnumTypeInfo(new LoadableEnumTypeInfo(*this, enumTy,
                                                 rawFixedTI.getFixedSize(),
                                                 rawFixedTI.getSpareBits(),
                                                 rawFixedTI.getFixedAlignment(),
                                                 IsPOD));
  }
  
  TypeInfo *SinglePayloadEnumImplStrategy::completeFixedLayout(
                                      TypeConverter &TC,
                                      CanType type,
                                      EnumDecl *theEnum,
                                      llvm::StructType *enumTy) {
    // See whether the payload case's type has extra inhabitants.
    unsigned fixedExtraInhabitants = 0;
    unsigned numTags = ElementsWithNoPayload.size();
    
    auto &payloadTI = getFixedPayloadTypeInfo(); // FIXME non-fixed payload
    fixedExtraInhabitants = payloadTI.getFixedExtraInhabitantCount(TC.IGM);
    
    // Determine how many tag bits we need. Given N extra inhabitants, we
    // represent the first N tags using those inhabitants. For additional tags,
    // we use discriminator bit(s) to inhabit the full bit size of the payload.
    unsigned tagsWithoutInhabitants = numTags <= fixedExtraInhabitants
      ? 0 : numTags - fixedExtraInhabitants;
    
    if (tagsWithoutInhabitants == 0) {
      ExtraTagBitCount = 0;
      NumExtraTagValues = 0;
    // If the payload size is greater than 32 bits, the calculation would
    // overflow, but one tag bit should suffice. if you have more than 2^32
    // enum discriminators you have other problems.
    } else if (payloadTI.getFixedSize().getValue() >= 4) {
      ExtraTagBitCount = 1;
      NumExtraTagValues = 2;
    } else {
      unsigned tagsPerTagBitValue =
        1 << payloadTI.getFixedSize().getValueInBits();
      NumExtraTagValues
        = (tagsWithoutInhabitants+(tagsPerTagBitValue-1))/tagsPerTagBitValue+1;
      ExtraTagBitCount = llvm::Log2_32(NumExtraTagValues-1) + 1;
    }

    // Create the body type.
    setTaggedEnumBody(TC.IGM, enumTy,
                       payloadTI.getFixedSize().getValueInBits(),
                       ExtraTagBitCount);
    
    // The enum has the alignment of the payload. The size includes the added
    // tag bits.
    auto sizeWithTag = payloadTI.getFixedSize().getValue();
    sizeWithTag += (ExtraTagBitCount+7U)/8U;
    
    /// FIXME: Spare bits.
    return getFixedEnumTypeInfo(enumTy, Size(sizeWithTag), {},
                            payloadTI.getFixedAlignment(),
                            payloadTI.isPOD(ResilienceScope::Component));
  }
  
  TypeInfo *SinglePayloadEnumImplStrategy::completeDynamicLayout(
                                                  TypeConverter &TC,
                                                  CanType type,
                                                  EnumDecl *theEnum,
                                                  llvm::StructType *enumTy) {
    // The body is runtime-dependent, so we can't put anything useful here
    // statically.
    enumTy->setBody(ArrayRef<llvm::Type*>{}, /*isPacked*/true);

    // Layout has to be done when the value witness table is instantiated,
    // during initializeMetadata.
    return registerEnumTypeInfo(new NonFixedEnumTypeInfo(*this, enumTy, type,
                       getPayloadTypeInfo().getBestKnownAlignment(),
                       getPayloadTypeInfo().isPOD(ResilienceScope::Component)));
  }
  
  TypeInfo *
  SinglePayloadEnumImplStrategy::completeEnumTypeLayout(TypeConverter &TC,
                                                  CanType type,
                                                  EnumDecl *theEnum,
                                                  llvm::StructType *enumTy) {
    PayloadTy = type->getTypeOfMember(theEnum->getModuleContext(),
                                      getPayloadElement(),
                                      nullptr,
                                      getPayloadElement()->getArgumentType())
      ->getCanonicalType();
    
    if (TIK >= Fixed)
      return completeFixedLayout(TC, type, theEnum, enumTy);
    return completeDynamicLayout(TC, type, theEnum, enumTy);
  }
  
  TypeInfo *
  MultiPayloadEnumImplStrategy::completeEnumTypeLayout(TypeConverter &TC,
                                                  CanType type,
                                                  EnumDecl *theEnum,
                                                  llvm::StructType *enumTy) {
    // TODO Dynamic layout for multi-payload enums.
    if (!TC.IGM.Opts.EnableDynamicValueTypeLayout && TIK < Fixed) {
      TC.IGM.unimplemented(theEnum->getLoc(),
                           "non-fixed multi-payload enum layout");
      exit(1);
    }

    // We need tags for each of the payload types, which we may be able to form
    // using spare bits, plus a minimal number of tags with which we can
    // represent the empty cases.
    unsigned numPayloadTags = ElementsWithPayload.size();
    unsigned numEmptyElements = ElementsWithNoPayload.size();
    
    // See if the payload types have any spare bits in common.
    // At the end of the loop CommonSpareBits.size() will be the size (in bits)
    // of the largest payload.
    CommonSpareBits = {};
    Alignment worstAlignment(1);
    IsPOD_t isPOD = IsPOD;
    for (auto &elt : ElementsWithPayload) {
      auto &fixedPayloadTI = cast<FixedTypeInfo>(*elt.ti); // FIXME
      fixedPayloadTI.applyFixedSpareBitsMask(CommonSpareBits);
      if (fixedPayloadTI.getFixedAlignment() > worstAlignment)
        worstAlignment = fixedPayloadTI.getFixedAlignment();
      if (!fixedPayloadTI.isPOD(ResilienceScope::Component))
        isPOD = IsNotPOD;
    }
    
    unsigned commonSpareBitCount = CommonSpareBits.count();
    unsigned usedBitCount = CommonSpareBits.size() - commonSpareBitCount;
    
    // Determine how many tags we need to accommodate the empty cases, if any.
    if (ElementsWithNoPayload.empty()) {
      NumEmptyElementTags = 0;
    } else {
      // We can store tags for the empty elements using the inhabited bits with
      // their own tag(s).
      if (usedBitCount >= 32) {
        NumEmptyElementTags = 1;
      } else {
        unsigned emptyElementsPerTag = 1 << usedBitCount;
        NumEmptyElementTags
          = (numEmptyElements + (emptyElementsPerTag-1))/emptyElementsPerTag;
      }
    }
    
    unsigned numTags = numPayloadTags + NumEmptyElementTags;
    unsigned numTagBits = llvm::Log2_32(numTags-1) + 1;
    ExtraTagBitCount = numTagBits <= commonSpareBitCount
      ? 0 : numTagBits - commonSpareBitCount;
    NumExtraTagValues = numTags >> commonSpareBitCount;
    
    // Create the type. We need enough bits to store the largest payload plus
    // extra tag bits we need.
    setTaggedEnumBody(TC.IGM, enumTy,
                       CommonSpareBits.size(),
                       ExtraTagBitCount);
    
    // The enum has the worst alignment of its payloads. The size includes the
    // added tag bits.
    auto sizeWithTag = Size((CommonSpareBits.size() + 7U)/8U)
      .roundUpToAlignment(worstAlignment)
      .getValue();
    sizeWithTag += (ExtraTagBitCount+7U)/8U;
    
    return getFixedEnumTypeInfo(enumTy, Size(sizeWithTag), {},
                            worstAlignment, isPOD);
  }
}

const TypeInfo *TypeConverter::convertEnumType(TypeBase *key, CanType type,
                                               EnumDecl *theEnum) {
  llvm::StructType *convertedStruct = IGM.createNominalType(theEnum);

  // Create a forward declaration for that type.
  addForwardDecl(key, convertedStruct);

  // Determine the implementation strategy.
  EnumImplStrategy *strategy = EnumImplStrategy::get(*this, type, theEnum);

  // Create the TI.
  return strategy->completeEnumTypeLayout(*this, type,
                                           theEnum, convertedStruct);
}

/// emitEnumDecl - Emit all the declarations associated with this enum type.
void IRGenModule::emitEnumDecl(EnumDecl *theEnum) {
  emitEnumMetadata(*this, theEnum);
  
  // FIXME: This is mostly copy-paste from emitExtension;
  // figure out how to refactor! 
  for (Decl *member : theEnum->getMembers()) {
    switch (member->getKind()) {
    case DeclKind::Import:
    case DeclKind::TopLevelCode:
    case DeclKind::Protocol:
    case DeclKind::Extension:
    case DeclKind::Destructor:
    case DeclKind::InfixOperator:
    case DeclKind::PrefixOperator:
    case DeclKind::PostfixOperator:
      llvm_unreachable("decl not allowed in enum!");

    // We can't have meaningful initializers for variables; these just show
    // up as part of parsing properties.
    case DeclKind::PatternBinding:
      continue;

    case DeclKind::Subscript:
      // Getter/setter will be handled separately.
      continue;
    case DeclKind::TypeAlias:
    case DeclKind::AssociatedType:
    case DeclKind::GenericTypeParam:
      continue;
    case DeclKind::Enum:
      emitEnumDecl(cast<EnumDecl>(member));
      continue;
    case DeclKind::Struct:
      emitStructDecl(cast<StructDecl>(member));
      continue;
    case DeclKind::Class:
      emitClassDecl(cast<ClassDecl>(member));
      continue;
    case DeclKind::Var:
      if (cast<VarDecl>(member)->isComputed())
        // Getter/setter will be handled separately.
        continue;
      continue;
    case DeclKind::Func:
      emitLocalDecls(cast<FuncDecl>(member));
      continue;
    case DeclKind::Constructor:
      emitLocalDecls(cast<ConstructorDecl>(member));
      continue;
        
    case DeclKind::EnumCase:
    case DeclKind::EnumElement:
      // Lowered in SIL.
      continue;
    }
    llvm_unreachable("bad extension member kind");
  }
}

// FIXME: PackEnumPayload and UnpackEnumPayload need to be endian-aware.

PackEnumPayload::PackEnumPayload(IRGenFunction &IGF, unsigned bitSize)
  : IGF(IGF), bitSize(bitSize)
{}

void PackEnumPayload::add(llvm::Value *v) {
  // First, bitcast to an integer type.
  if (isa<llvm::PointerType>(v->getType())) {
    v = IGF.Builder.CreatePtrToInt(v, IGF.IGM.SizeTy);
  } else if (!isa<llvm::IntegerType>(v->getType())) {
    unsigned bitSize = IGF.IGM.DataLayout.getTypeSizeInBits(v->getType());
    auto intTy = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), bitSize);
    v = IGF.Builder.CreateBitCast(v, intTy);
  }
  auto fromTy = cast<llvm::IntegerType>(v->getType());
  
  // If this was the first added value, use it to start our packed value.
  if (!packedValue) {
    // Zero-extend the integer value out to the value size.
    // FIXME: On big-endian, shift out to the value size.
    if (fromTy->getBitWidth() < bitSize) {
      auto toTy = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), bitSize);
      v = IGF.Builder.CreateZExt(v, toTy);
    }
    if (packedBits != 0)
      v = IGF.Builder.CreateShl(v, packedBits);
    packedBits += fromTy->getBitWidth();
    packedValue = v;
    return;
  }
  
  // Otherwise, shift and bitor the value into the existing value.
  v = IGF.Builder.CreateZExt(v, packedValue->getType());
  v = IGF.Builder.CreateShl(v, packedBits);
  packedBits += fromTy->getBitWidth();
  packedValue = IGF.Builder.CreateOr(packedValue, v);
}

void PackEnumPayload::addAtOffset(llvm::Value *v, unsigned bitOffset) {
  packedBits = bitOffset;
  add(v);
}

void PackEnumPayload::combine(llvm::Value *v) {
  if (!packedValue)
    packedValue = v;
  else
    packedValue = IGF.Builder.CreateOr(packedValue, v);
}

llvm::Value *PackEnumPayload::get() {
  if (!packedValue)
    packedValue = getEmpty(IGF.IGM, bitSize);
  return packedValue;
}

llvm::Value *PackEnumPayload::getEmpty(IRGenModule &IGM, unsigned bitSize) {
  return llvm::ConstantInt::get(IGM.getLLVMContext(), APInt(bitSize, 0));
}

UnpackEnumPayload::UnpackEnumPayload(IRGenFunction &IGF,
                                       llvm::Value *packedValue)
  : IGF(IGF), packedValue(packedValue)
{}

llvm::Value *UnpackEnumPayload::claim(llvm::Type *ty) {
  // Mask out the bits for the value.
  unsigned bitSize = IGF.IGM.DataLayout.getTypeSizeInBits(ty);
  auto bitTy = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), bitSize);
  llvm::Value *unpacked = unpackedBits == 0
    ? packedValue
    : IGF.Builder.CreateLShr(packedValue, unpackedBits);
  if (bitSize < cast<llvm::IntegerType>(packedValue->getType())->getBitWidth())
    unpacked = IGF.Builder.CreateTrunc(unpacked, bitTy);

  unpackedBits += bitSize;

  // Bitcast to the destination type.
  if (isa<llvm::PointerType>(ty))
    return IGF.Builder.CreateIntToPtr(unpacked, ty);
  return IGF.Builder.CreateBitCast(unpacked, ty);
}

llvm::Value *UnpackEnumPayload::claimAtOffset(llvm::Type *ty,
                                               unsigned bitOffset) {
  unpackedBits = bitOffset;
  return claim(ty);
}

void irgen::emitSwitchLoadableEnumDispatch(IRGenFunction &IGF,
                                  SILType enumTy,
                                  Explosion &enumValue,
                                  ArrayRef<std::pair<EnumElementDecl *,
                                                     llvm::BasicBlock *>> dests,
                                  llvm::BasicBlock *defaultDest) {
  getEnumImplStrategy(IGF.IGM, enumTy)
    .emitValueSwitch(IGF, enumValue, dests, defaultDest);
}

void irgen::emitSwitchAddressOnlyEnumDispatch(IRGenFunction &IGF,
                                  SILType enumTy,
                                  Address enumAddr,
                                  ArrayRef<std::pair<EnumElementDecl *,
                                                     llvm::BasicBlock *>> dests,
                                  llvm::BasicBlock *defaultDest) {
  auto &strategy = getEnumImplStrategy(IGF.IGM, enumTy);
  strategy.emitIndirectSwitch(IGF, enumAddr, dests, defaultDest);
}

void irgen::emitInjectLoadableEnum(IRGenFunction &IGF, SILType enumTy,
                                    EnumElementDecl *theCase,
                                    Explosion &data,
                                    Explosion &out) {
  getEnumImplStrategy(IGF.IGM, enumTy)
    .emitValueInjection(IGF, theCase, data, out);
}

void irgen::emitProjectLoadableEnum(IRGenFunction &IGF, SILType enumTy,
                                     Explosion &inEnumValue,
                                     EnumElementDecl *theCase,
                                     Explosion &out) {
  getEnumImplStrategy(IGF.IGM, enumTy)
    .emitValueProject(IGF, inEnumValue, theCase, out);
}

Address irgen::emitProjectEnumAddressForStore(IRGenFunction &IGF,
                                               SILType enumTy,
                                               Address enumAddr,
                                               EnumElementDecl *theCase) {
  return getEnumImplStrategy(IGF.IGM, enumTy)
    .projectDataForStore(IGF, theCase, enumAddr);
}

Address irgen::emitDestructiveProjectEnumAddressForLoad(IRGenFunction &IGF,
                                                   SILType enumTy,
                                                   Address enumAddr,
                                                   EnumElementDecl *theCase) {
  return getEnumImplStrategy(IGF.IGM, enumTy)
    .destructiveProjectDataForLoad(IGF, theCase, enumAddr);
}

void irgen::emitStoreEnumTagToAddress(IRGenFunction &IGF,
                                       SILType enumTy,
                                       Address enumAddr,
                                       EnumElementDecl *theCase) {
  getEnumImplStrategy(IGF.IGM, enumTy)
    .storeTag(IGF, theCase, enumAddr);
}

APInt irgen::getAPIntFromBitVector(const llvm::BitVector &bits) {
  SmallVector<llvm::integerPart, 2> parts;
  
  for (unsigned i = 0; i < bits.size();) {
    llvm::integerPart part = 0UL;
    for (llvm::integerPart bit = 1; bit != 0 && i < bits.size();
         ++i, bit <<= 1) {
      if (bits[i])
        part |= bit;
    }
    parts.push_back(part);
  }
  
  return APInt(bits.size(), parts);
}

/// Gather spare bits into the low bits of a smaller integer value.
llvm::Value *irgen::emitGatherSpareBits(IRGenFunction &IGF,
                                        const llvm::BitVector &spareBitMask,
                                        llvm::Value *spareBits,
                                        unsigned resultLowBit,
                                        unsigned resultBitWidth) {
  auto destTy
    = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), resultBitWidth);
  unsigned usedBits = resultLowBit;
  llvm::Value *result = nullptr;
  
  for (int i = spareBitMask.find_first(); i != -1;
       i = spareBitMask.find_next(i)) {
    assert(i >= 0);
    unsigned u = i;
    assert(u >= (usedBits - resultLowBit) &&
           "used more bits than we've processed?!");
    
    // Shift the bits into place.
    llvm::Value *newBits;
    if (u > usedBits)
      newBits = IGF.Builder.CreateLShr(spareBits, u - usedBits);
    else if (u < usedBits)
      newBits = IGF.Builder.CreateShl(spareBits, usedBits - u);
    else
      newBits = spareBits;
    newBits = IGF.Builder.CreateZExtOrTrunc(newBits, destTy);
    
    // See how many consecutive bits we have.
    unsigned numBits = 1;
    ++u;
    for (unsigned e = spareBitMask.size(); u < e && spareBitMask[u]; ++u)
      ++numBits;
    
    // Mask out the selected bits.
    auto val = APInt::getAllOnesValue(numBits);
    if (numBits < resultBitWidth)
      val = val.zext(resultBitWidth);
    val = val.shl(usedBits);
    auto *mask = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(), val);
    newBits = IGF.Builder.CreateAnd(newBits, mask);
    
    // Accumulate the result.
    if (result)
      result = IGF.Builder.CreateOr(result, newBits);
    else
      result = newBits;
    
    usedBits += numBits;
    i = u;
  }
  
  return result;
}

/// Scatter spare bits from the low bits of an integer value.
llvm::Value *irgen::emitScatterSpareBits(IRGenFunction &IGF,
                                         const llvm::BitVector &spareBitMask,
                                         llvm::Value *packedBits,
                                         unsigned packedLowBit) {
  auto destTy
    = llvm::IntegerType::get(IGF.IGM.getLLVMContext(), spareBitMask.size());
  llvm::Value *result = nullptr;
  unsigned usedBits = packedLowBit;
  
  // Expand the packed bits to the destination type.
  packedBits = IGF.Builder.CreateZExtOrTrunc(packedBits, destTy);
  
  for (int i = spareBitMask.find_first(); i != -1;
       i = spareBitMask.find_next(i)) {
    assert(i >= 0);
    unsigned u = i, startBit = u;
    assert(u >= usedBits - packedLowBit
           && "used more bits than we've processed?!");
    
    // Shift the selected bits into place.
    llvm::Value *newBits;
    if (u > usedBits)
      newBits = IGF.Builder.CreateShl(packedBits, u - usedBits);
    else if (u < usedBits)
      newBits = IGF.Builder.CreateLShr(packedBits, usedBits - u);
    else
      newBits = packedBits;
    
    // See how many consecutive bits we have.
    unsigned numBits = 1;
    ++u;
    for (unsigned e = spareBitMask.size(); u < e && spareBitMask[u]; ++u)
      ++numBits;
    
    // Mask out the selected bits.
    auto val = APInt::getAllOnesValue(numBits);
    if (numBits < spareBitMask.size())
      val = val.zext(spareBitMask.size());
    val = val.shl(startBit);
    auto mask = llvm::ConstantInt::get(IGF.IGM.getLLVMContext(), val);
    newBits = IGF.Builder.CreateAnd(newBits, mask);
    
    // Accumulate the result.
    if (result)
      result = IGF.Builder.CreateOr(result, newBits);
    else
      result = newBits;
    
    usedBits += numBits;
    i = u;
  }
  
  return result;
}

/// Interleave the occupiedValue and spareValue bits, taking a bit from one
/// or the other at each position based on the spareBits mask.
llvm::ConstantInt *
irgen::interleaveSpareBits(IRGenModule &IGM, const llvm::BitVector &spareBits,
                           unsigned bits,
                           unsigned spareValue, unsigned occupiedValue) {
  // FIXME: endianness.
  SmallVector<llvm::integerPart, 2> valueParts;
  valueParts.push_back(0);
  
  llvm::integerPart valueBit = 1;
  auto advanceValueBit = [&]{
    valueBit <<= 1;
    if (valueBit == 0) {
      valueParts.push_back(0);
      valueBit = 1;
    }
  };
  
  for (unsigned i = 0, e = spareBits.size();
       (occupiedValue || spareValue) && i < e;
       ++i, advanceValueBit()) {
    if (spareBits[i]) {
      if (spareValue & 1)
        valueParts.back() |= valueBit;
      spareValue >>= 1;
    } else {
      if (occupiedValue & 1)
        valueParts.back() |= valueBit;
      occupiedValue >>= 1;
    }
  }
  
  // Create the value.
  llvm::APInt value(bits, valueParts);
  return llvm::ConstantInt::get(IGM.getLLVMContext(), value);
}

static void setAlignmentBits(llvm::BitVector &v, Alignment align) {
  switch (align.getValue()) {
  case 16:
    v[3] = true;
    SWIFT_FALLTHROUGH;
  case 8:
    v[2] = true;
    SWIFT_FALLTHROUGH;
  case 4:
    v[1] = true;
    SWIFT_FALLTHROUGH;
  case 2:
    v[0] = true;
    SWIFT_FALLTHROUGH;
  case 1:
  case 0:
    break;
  default:
    llvm_unreachable("unexpected heap object alignment");
  }
}

const llvm::BitVector &
IRGenModule::getHeapObjectSpareBits() const {
  return HeapPointerSpareBits.cache([&]{
    // Start with the spare bit mask for all pointers.
    llvm::BitVector r = TargetInfo.PointerSpareBits;

    // Low bits are made available by heap object alignment.
    setAlignmentBits(r, TargetInfo.HeapObjectAlignment);
    
    // Mask out bits reserved by the Objective-C runtime.
    llvm::BitVector objcMask = TargetInfo.ObjCPointerReservedBits;
    objcMask.flip();
    r &= objcMask;
    
    return r;
  });
}