[serialization] Add cross-module decl references.

This is currently implemented in terms of an access path of identifiers,
with a type check at the very end to pick the correct overload for a
function. Of course there are plenty of things missing here:
- function overloading on parameter names (selector pieces)
- access to values inside extensions
- generics (as everywhere)

This allows the serialization tests to be built against the standard
library instead of requiring -parse-stdlib.

Swift SVN r5797

lib/Serialization/ModuleFile.cpp

@@ -203,6 +203,49 @@ DeclContext *ModuleFile::getDeclContext(DeclID DID) {
   llvm_unreachable("unknown DeclContext kind");
 }
 
+
+/// Finds all value decls with the given name in the given context,
+/// visible or not.
+static void lookupImmediateDecls(DeclContext *DC, Identifier name,
+                                 SmallVectorImpl<ValueDecl *> &results) {
+  ArrayRef<Decl *> members;
+
+  switch (DC->getContextKind()) {
+  case DeclContextKind::TranslationUnit:
+  case DeclContextKind::BuiltinModule:
+  case DeclContextKind::SerializedModule:
+  case DeclContextKind::ClangModule:
+    llvm_unreachable("should not be used for module contexts");
+
+  case DeclContextKind::CapturingExpr:
+  case DeclContextKind::ConstructorDecl:
+  case DeclContextKind::DestructorDecl:
+    // Don't look at params or local variables.
+    return;
+
+  case DeclContextKind::NominalTypeDecl: {
+    members = cast<NominalTypeDecl>(DC)->getMembers();
+    break;
+  }
+
+  case DeclContextKind::ExtensionDecl: {
+    members = cast<ExtensionDecl>(DC)->getMembers();
+    break;
+  }
+
+  case DeclContextKind::TopLevelCodeDecl:
+    llvm_unreachable("libraries do not expose their top-level code");
+  }
+
+  for (Decl *member : members) {
+    if (auto value = dyn_cast<ValueDecl>(member)) {
+      if (value->getName() == name)
+        results.push_back(value);
+    }
+  }
+}
+
+
 Decl *ModuleFile::getDecl(DeclID DID, DeclDeserializationOptions opts) {
   if (DID == 0)
     return nullptr;
@@ -504,6 +547,115 @@ Decl *ModuleFile::getDecl(DeclID DID, DeclDeserializationOptions opts) {
     break;
   }
 
+  case decls_block::XREF: {
+    uint8_t kind;
+    TypeID expectedTypeID;
+    ArrayRef<uint64_t> rawAccessPath;
+
+    decls_block::XRefLayout::readRecord(scratch, kind, expectedTypeID,
+                                        rawAccessPath);
+
+    // First, find the module this reference is referring to.
+    Identifier moduleName = getIdentifier(rawAccessPath.front());
+    rawAccessPath = rawAccessPath.slice(1);
+    // FIXME: provide a real source location.
+    Module *M = ctx.getModule(std::make_pair(moduleName, SourceLoc()));
+    assert(M && "missing dependency");
+
+    switch (kind) {
+    case XRefKind::SwiftValue: {
+      // Start by looking up the top-level decl in the module.
+      SmallVector<ValueDecl *, 8> values;
+      M->lookupValue(Module::AccessPathTy(),
+                     getIdentifier(rawAccessPath.front()),
+                     NLKind::QualifiedLookup,
+                     values);
+      rawAccessPath = rawAccessPath.slice(1);
+
+      // Then, follow the chain of nested ValueDecls until we run out of
+      // identifiers in the access path.
+      SmallVector<ValueDecl *, 8> baseValues;
+      while (!rawAccessPath.empty()) {
+        baseValues.swap(values);
+        values.clear();
+        for (auto base : baseValues) {
+          // FIXME: extensions?
+          if (auto nominal = dyn_cast<NominalTypeDecl>(base))
+            lookupImmediateDecls(nominal,
+                                 getIdentifier(rawAccessPath.front()),
+                                 values);
+        }
+        rawAccessPath = rawAccessPath.slice(1);
+      }
+
+      // If we have a type to validate against, filter out any ValueDecls that
+      // don't match that type.
+      CanType expectedTy;
+      Type maybeExpectedTy = getType(expectedTypeID);
+      if (maybeExpectedTy)
+        expectedTy = maybeExpectedTy->getCanonicalType();
+
+      ValueDecl *result = nullptr;
+      for (auto value : values) {
+        if (!expectedTy || value->getType()->getCanonicalType() == expectedTy) {
+          // It's an error if more than one value has the same type.
+          // FIXME: Functions and constructors can overload based on parameter
+          // names.
+          if (result) {
+            error();
+            return nullptr;
+          }
+          result = value;
+        }
+      }
+
+      // It's an error if lookup doesn't actually find anything -- that means
+      // the module's out of date.
+      if (!result) {
+        error();
+        return nullptr;
+      }
+
+      declOrOffset = result;
+      break;
+    }
+    case XRefKind::SwiftOperator: {
+      assert(rawAccessPath.size() == 1 &&
+             "can't import operators not at module scope");
+      Identifier opName = getIdentifier(rawAccessPath.back());
+
+      switch (expectedTypeID) {
+      case OperatorKind::Infix: {
+        auto op = M->lookupInfixOperator(opName);
+        declOrOffset = op.hasValue() ? op.getValue() : nullptr;
+        break;
+      }
+      case OperatorKind::Prefix: {
+        auto op = M->lookupPrefixOperator(opName);
+        declOrOffset = op.hasValue() ? op.getValue() : nullptr;
+        break;
+      }
+      case OperatorKind::Postfix: {
+        auto op = M->lookupPostfixOperator(opName);
+        declOrOffset = op.hasValue() ? op.getValue() : nullptr;
+        break;
+      }
+      default:
+        // Unknown operator kind.
+        error();
+        return nullptr;
+      }
+      break;
+    }
+    default:
+      // Unknown cross-reference kind.
+      error();
+      return nullptr;
+    }
+
+    break;
+  }
+      
   default:
     // We don't know how to deserialize this kind of decl.
     error();
@@ -521,7 +673,7 @@ Decl *ModuleFile::getDecl(DeclID DID, DeclDeserializationOptions opts) {
 static Optional<swift::AbstractCC> getActualCC(uint8_t cc) {
   switch (cc) {
 #define CASE(THE_CC) \
-  case static_cast<uint8_t>(serialization::AbstractCC::THE_CC): \
+  case serialization::AbstractCC::THE_CC: \
     return swift::AbstractCC::THE_CC;
   CASE(C)
   CASE(ObjCMethod)

lib/Serialization/ModuleFormat.h

@@ -60,14 +60,31 @@ using CharOffsetField = BitOffsetField;
 
 // These IDs must \em not be renumbered or reordered without incrementing
 // VERSION_MAJOR.
-enum class AbstractCC : uint8_t {
+enum AbstractCC : uint8_t {
   C = 0,
   ObjCMethod,
   Freestanding,
   Method
 };
-using AbstractCCField = BCVBR<4>;
+using AbstractCCField = BCFixed<2>;
 
+// These IDs must \em not be renumbered or reordered without incrementing
+// VERSION_MAJOR.
+enum XRefKind : uint8_t {
+  SwiftValue = 0,
+  SwiftOperator
+};
+using XRefKindField = BCFixed<1>;
+
+// These IDs must \em not be renumbered or reordered without incrementing
+// VERSION_MAJOR.
+enum OperatorKind : uint8_t {
+  Infix = 0,
+  Prefix,
+  Postfix
+};
+static_assert(sizeof(OperatorKind) <= sizeof(TypeID),
+              "too many operator kinds");
 
 /// The various types of blocks that can occur within a serialized Swift
 /// module.
@@ -190,6 +207,7 @@ namespace decls_block {
     ISA_PATTERN,
     NOMINAL_TYPE_PATTERN,
 
+    XREF = 254,
     DECL_CONTEXT = 255
   };
 
@@ -356,6 +374,13 @@ namespace decls_block {
     // The sub-pattern trails the record.
   >;
 
+  using XRefLayout = BCRecordLayout<
+    XREF,
+    XRefKindField, // reference kind
+    TypeIDField,   // type if value, operator kind if operator
+    BCArray<IdentifierIDField> // access path
+  >;
+
   using DeclContextLayout = BCRecordLayout<
     DECL_CONTEXT,
     BCArray<DeclIDField>

lib/Serialization/Serialization.cpp

@@ -37,6 +37,9 @@ namespace {
     /// A reusable buffer for emitting records.
     SmallVector<uint64_t, 64> ScratchRecord;
 
+    /// The TranslationUnit currently being serialized.
+    const TranslationUnit *TU;
+
   public:    
     /// Stores a declaration or a type to be written to the AST file.
     ///
@@ -161,6 +164,12 @@ namespace {
     /// Writes the given pattern, recursively.
     void writePattern(const Pattern *pattern);
 
+    /// Writes a reference to a decl in another module.
+    ///
+    /// Returns false if the decl cannot be serialized without losing
+    /// information.
+    bool writeCrossReference(const Decl *D);
+
     /// Writes the given decl.
     ///
     /// Returns false if the decl cannot be serialized without losing
@@ -203,7 +212,7 @@ namespace {
 
   public:
     Serializer()
-      : Out(Buffer), LastDeclID(0), LastTypeID(0),
+      : Out(Buffer), TU(nullptr), LastDeclID(0), LastTypeID(0),
         ShouldFallBackToTranslationUnit(false) {
     }
 
@@ -364,6 +373,8 @@ void Serializer::writeBlockInfoBlock() {
   RECORD(decls_block, NAMED_PATTERN);
   RECORD(decls_block, ANY_PATTERN);
   RECORD(decls_block, TYPED_PATTERN);
+
+  RECORD(decls_block, XREF);
   RECORD(decls_block, DECL_CONTEXT);
 
   BLOCK(IDENTIFIER_DATA_BLOCK);
@@ -512,6 +523,66 @@ void Serializer::writePattern(const Pattern *pattern) {
   }
 }
 
+bool Serializer::writeCrossReference(const Decl *D) {
+  using namespace decls_block;
+
+  SmallVector<IdentifierID, 4> accessPath;
+  XRefKind kind;
+  TypeID typeID;
+
+  if (auto value = dyn_cast<ValueDecl>(D)) {
+    kind = XRefKind::SwiftValue;
+    accessPath.push_back(addIdentifierRef(value->getName()));
+
+    // Make sure we don't create a self-referential type.
+    Type ty = value->getType();
+    if (ty->is<MetaTypeType>())
+      ty = nullptr;
+    typeID = addTypeRef(ty);
+
+  } else if (auto op = dyn_cast<OperatorDecl>(D)) {
+    kind = XRefKind::SwiftOperator;
+    accessPath.push_back(addIdentifierRef(op->getName()));
+
+    switch (op->getKind()) {
+      case DeclKind::InfixOperator:
+        typeID = OperatorKind::Infix;
+        break;
+      case DeclKind::PrefixOperator:
+        typeID = OperatorKind::Prefix;
+        break;
+      case DeclKind::PostfixOperator:
+        typeID = OperatorKind::Postfix;
+        break;
+      default:
+        llvm_unreachable("unknown operator kind");
+    }
+  } else {
+    llvm_unreachable("cannot cross-reference this kind of decl");
+  }
+
+  // Build up the access path by walking through parent DeclContexts.
+  const DeclContext *DC;
+  for (DC = D->getDeclContext(); !DC->isModuleContext(); DC = DC->getParent()) {
+    // FIXME: Handle references to things in extensions.
+    if (isa<ExtensionDecl>(D))
+      return false;
+
+    auto value = cast<ValueDecl>(getDeclForContext(DC));
+    accessPath.push_back(addIdentifierRef(value->getName()));
+  }
+
+  accessPath.push_back(addIdentifierRef(cast<Module>(DC)->Name));
+  // Store the access path in forward order.
+  std::reverse(accessPath.begin(), accessPath.end());
+
+  unsigned abbrCode = DeclTypeAbbrCodes[XRefLayout::Code];
+  XRefLayout::emitRecord(Out, ScratchRecord, abbrCode,
+                         kind, typeID, accessPath);
+
+  return true;
+}
+
 bool Serializer::writeDecl(const Decl *D) {
   using namespace decls_block;
 
@@ -519,6 +590,10 @@ bool Serializer::writeDecl(const Decl *D) {
   if (D->hasClangNode())
     return false;
 
+  Module *M = D->getModuleContext();
+  if (M != TU)
+    return writeCrossReference(D);
+
   switch (D->getKind()) {
   case DeclKind::Import:
     // FIXME: Do imported module names appear in the DeclContext of the
@@ -726,7 +801,7 @@ static uint8_t getRawStableCC(swift::AbstractCC cc) {
   switch (cc) {
 #define CASE(THE_CC) \
   case swift::AbstractCC::THE_CC: \
-    return static_cast<uint8_t>(serialization::AbstractCC::THE_CC);
+    return serialization::AbstractCC::THE_CC;
   CASE(C)
   CASE(ObjCMethod)
   CASE(Freestanding)
@@ -921,6 +996,8 @@ void Serializer::writeAllDeclsAndTypes() {
     registerDeclTypeAbbr<NamedPatternLayout>();
     registerDeclTypeAbbr<AnyPatternLayout>();
     registerDeclTypeAbbr<TypedPatternLayout>();
+
+    registerDeclTypeAbbr<XRefLayout>();
     registerDeclTypeAbbr<DeclContextLayout>();
   }
 
@@ -969,6 +1046,9 @@ void Serializer::writeOffsets(const index_block::OffsetsLayout &Offsets,
 }
 
 void Serializer::writeTranslationUnit(const TranslationUnit *TU) {
+  assert(!this->TU && "already serializing a translation unit");
+  this->TU = TU;
+
   SmallVector<DeclID, 32> topLevelIDs;
   for (auto D : TU->Decls) {
     if (isa<ImportDecl>(D))
@@ -990,6 +1070,10 @@ void Serializer::writeTranslationUnit(const TranslationUnit *TU) {
     index_block::TopLevelDeclsLayout TopLevelDecls(Out);
     TopLevelDecls.emit(ScratchRecord, topLevelIDs);
   }
+
+#ifndef NDEBUG
+  this->TU = nullptr;
+#endif
 }
 
 void Serializer::writeToStream(raw_ostream &os, const TranslationUnit *TU,