averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-21 12:14:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

[serialization] Lazily load top-level decls, extensions, and operators.

Browse Source 
This switches from simple lists of decls to name-based on-disk hash tables,
which allows decls to be loaded lazily when doing simple lookup (but not
code completion, at least not yet).

The on-disk hash table implementation is borrowed from Clang; eventually
it will be pushed down to LLVM's Support library. (Fortunately the
implementation is header-only.)

This breaks a few tests that rely on magic protocols like
IntegerLiteralConvertible, because the type checker won't have seen the
types that conform to those protocols yet. This will be fixed by doing
an additional "hey, modules, got any of these?" lookup.

Swift SVN r7259
This commit is contained in:
Jordan Rose
2013-08-15 17:31:44 +00:00
parent 49f1d53e50
commit bca05dab59
6 changed files with 302 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
include/swift/Serialization/SerializedModuleLoader.h
View File

@@ -26,7 +26,9 @@ class SerializedModuleLoader : public ModuleLoader {
private:
ASTContext &Ctx;
std::vector<llvm::OwningPtr<ModuleFile>> LoadedModuleFiles;
/// A { module, generation # } pair.
using LoadedModulePair = std::pair<std::unique_ptr<ModuleFile>, unsigned>;
std::vector<LoadedModulePair> LoadedModuleFiles;
explicit SerializedModuleLoader(ASTContext &ctx);
@@ -89,6 +91,9 @@ public:
Module::AccessPathTy accessPath,
VisibleDeclConsumer &consumer,
NLKind lookupKind) override;
virtual void loadExtensions(NominalTypeDecl *nominal,
unsigned previousGeneration) override;
};
} // end namespace swift

171
lib/Serialization/ModuleFile.cpp
View File

@@ -15,8 +15,14 @@
#include "swift/AST/AST.h"
#include "swift/AST/ModuleLoader.h"
#include "swift/AST/NameLookup.h"
#include "swift/Basic/Range.h"
#include "swift/Basic/STLExtras.h"
#include "swift/Serialization/BCReadingExtras.h"
// This is a template-only header; eventually it should move to llvm/Support.
#include "clang/Basic/OnDiskHashTable.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Support/MemoryBuffer.h"
#include <functional>
@@ -1936,6 +1942,52 @@ Type ModuleFile::getType(TypeID TID) {
return typeOrOffset;
}
/// Used to deserialize entries in the on-disk decl hash table.
class ModuleFile::DeclTableInfo {
public:
using internal_key_type = StringRef;
using external_key_type = Identifier;
using data_type = SmallVector<std::pair<uint8_t, DeclID>, 8>;
internal_key_type GetInternalKey(external_key_type ID) {
return ID.str();
}
uint32_t ComputeHash(internal_key_type key) {
return llvm::HashString(key);
}
static bool EqualKey(internal_key_type lhs, internal_key_type rhs) {
return lhs == rhs;
}
static std::pair<unsigned, unsigned> ReadKeyDataLength(const uint8_t *&data) {
using namespace clang::io;
unsigned keyLength = ReadUnalignedLE16(data);
unsigned dataLength = ReadUnalignedLE16(data);
return { keyLength, dataLength };
}
static internal_key_type ReadKey(const uint8_t *data, unsigned length) {
return StringRef(reinterpret_cast<const char *>(data), length);
}
static data_type ReadData(internal_key_type key, const uint8_t *data,
unsigned length) {
using namespace clang::io;
data_type result;
while (length > 0) {
uint8_t kind = *data++;
DeclID offset = ReadUnalignedLE32(data);
result.push_back({ kind, offset });
length -= 5;
}
return result;
}
};
ModuleFile::ModuleFile(llvm::OwningPtr<llvm::MemoryBuffer> &&input)
: ModuleContext(nullptr),
InputFile(std::move(input)),
@@ -2085,14 +2137,22 @@ ModuleFile::ModuleFile(llvm::OwningPtr<llvm::MemoryBuffer> &&input)
assert(blobData.empty());
Identifiers.assign(scratch.begin(), scratch.end());
break;
case index_block::TOP_LEVEL_DECLS:
assert(blobData.empty());
RawTopLevelIDs.assign(scratch.begin(), scratch.end());
case index_block::TOP_LEVEL_DECLS: {
uint32_t tableOffset;
index_block::DeclListLayout::readRecord(scratch, tableOffset);
auto base = reinterpret_cast<const uint8_t *>(blobData.data());
TopLevelDecls.reset(SerializedDeclTable::Create(base + tableOffset,
base));
break;
case index_block::OPERATORS:
assert(blobData.empty());
RawOperatorIDs.assign(scratch.begin(), scratch.end());
}
case index_block::OPERATORS: {
uint32_t tableOffset;
index_block::DeclListLayout::readRecord(scratch, tableOffset);
auto base = reinterpret_cast<const uint8_t *>(blobData.data());
OperatorDecls.reset(SerializedDeclTable::Create(base + tableOffset,
base));
break;
}
default:
// Unknown index kind, which this version of the compiler won't use.
break;
@@ -2165,58 +2225,42 @@ bool ModuleFile::associateWithModule(Module *module) {
return false;
}
// FIXME: The only reason we're deserializing eagerly is to force extensions
// to load, which they shouldn't yet anyway.
buildTopLevelDeclMap();
return Status == ModuleStatus::Valid;
}
void ModuleFile::buildTopLevelDeclMap() {
// FIXME: be more lazy about deserialization by encoding this some other way.
for (DeclID ID : RawTopLevelIDs) {
// FIXME: Right now ExtensionDecls are in here as well.
if (auto value = dyn_cast<ValueDecl>(getDecl(ID)))
TopLevelDecls[value->getName()].push_back(value);
}
RawTopLevelIDs.clear();
}
// This is here so that other clients don't
ModuleFile::~ModuleFile() = default;
void ModuleFile::lookupValue(Identifier name,
SmallVectorImpl<ValueDecl*> &results) {
auto iter = TopLevelDecls.find(name);
if (iter == TopLevelDecls.end())
auto iter = TopLevelDecls->find(name);
if (iter == TopLevelDecls->end())
return;
results.append(iter->second.begin(), iter->second.end());
for (auto item : *iter) {
// FIXME: Once extensions are removed from the top-level decls, this can
// be a cast<>.
if (auto VD = dyn_cast<ValueDecl>(getDecl(item.second))) {
// Force load our own extensions, which may contain conformances.
if (auto TD = dyn_cast<TypeDecl>(VD))
if (auto nominal = TD->getDeclaredType()->getAnyNominal())
loadExtensions(nominal);
results.push_back(VD);
}
OperatorKind getOperatorKind(DeclKind kind) {
switch (kind) {
case DeclKind::PrefixOperator:
return Prefix;
case DeclKind::PostfixOperator:
return Postfix;
case DeclKind::InfixOperator:
return Infix;
default:
llvm_unreachable("unknown operator fixity");
}
}
OperatorDecl *ModuleFile::lookupOperator(Identifier name, DeclKind fixity) {
if (!RawOperatorIDs.empty()) {
for (DeclID ID : RawOperatorIDs) {
auto op = cast<OperatorDecl>(getDecl(ID));
OperatorKey key(op->getName(), getOperatorKind(op->getKind()));
Operators[key] = op;
auto iter = OperatorDecls->find(name);
if (iter == OperatorDecls->end())
return nullptr;
for (auto item : *iter) {
if (getRawStableFixity(fixity) == item.first)
return cast<OperatorDecl>(getDecl(item.second));
}
RawOperatorIDs = {};
}
return Operators.lookup(OperatorKey(name, getOperatorKind(fixity)));
return nullptr;
}
void ModuleFile::getReexportedModules(
@@ -2235,17 +2279,40 @@ void ModuleFile::lookupVisibleDecls(Module::AccessPathTy accessPath,
assert(accessPath.size() <= 1 && "can only refer to top-level decls");
if (!accessPath.empty()) {
auto I = TopLevelDecls.find(accessPath.front().first);
if (I == TopLevelDecls.end()) return;
for (auto vd : I->second)
consumer.foundDecl(vd);
auto iter = TopLevelDecls->find(accessPath.front().first);
if (iter == TopLevelDecls->end())
return;
}
for (auto &topLevelEntry : TopLevelDecls) {
for (auto &value : topLevelEntry.second)
consumer.foundDecl(value);
for (auto item : *iter) {
// FIXME: Once extensions are removed from the top-level decls, this can
// be a cast<>.
if (auto VD = dyn_cast<ValueDecl>(getDecl(item.second)))
consumer.foundDecl(VD);
}
}
for (auto entry : make_range(TopLevelDecls->data_begin(),
TopLevelDecls->data_end())) {
for (auto item : entry) {
// FIXME: Once extensions are removed from the top-level decls, this can
// be a cast<>.
if (auto VD = dyn_cast<ValueDecl>(getDecl(item.second)))
consumer.foundDecl(VD);
}
}
}
void ModuleFile::loadExtensions(NominalTypeDecl *nominal) {
// FIXME: Need dedicated extension table.
auto iter = TopLevelDecls->find(nominal->getName());
if (iter == TopLevelDecls->end())
return;
for (auto item : *iter) {
// FIXME: Once extensions are removed from the top-level decls, this can
// be a cast<>.
if (item.first == decls_block::EXTENSION_DECL)
(void)getDecl(item.second);
}
}

58
lib/Serialization/ModuleFile.h
View File

@@ -33,6 +33,12 @@ namespace llvm {
class MemoryBuffer;
}
// This template should eventually move to llvm/Support.
namespace clang {
template <typename Info>
class OnDiskChainedHashTable;
}
namespace swift {
class Pattern;
class ProtocolConformance;
@@ -163,22 +169,13 @@ private:
/// Identifiers referenced by this module.
std::vector<SerializedIdentifier> Identifiers;
/// All top-level decls in this module.
llvm::DenseMap<Identifier, TinyPtrVector<ValueDecl *>> TopLevelDecls;
private:
class DeclTableInfo;
using SerializedDeclTable = clang::OnDiskChainedHashTable<DeclTableInfo>;
/// An array of the top-level decl IDs.
// FIXME: We don't really want to deserialize all of these at once.
std::vector<serialization::DeclID> RawTopLevelIDs;
using OperatorKey = std::pair<Identifier, serialization::OperatorKind>;
friend struct llvm::DenseMapInfo<OperatorKey>;
/// All the operators in the module.
llvm::DenseMap<OperatorKey, OperatorDecl *> Operators;
/// An array of the top-level decl IDs.
// FIXME: We don't really want to deserialize all of these at once.
std::vector<serialization::DeclID> RawOperatorIDs;
public:
std::unique_ptr<SerializedDeclTable> TopLevelDecls;
std::unique_ptr<SerializedDeclTable> OperatorDecls;
/// Whether this module file can be used.
ModuleStatus Status;
@@ -186,6 +183,9 @@ private:
/// Constructs an new module and validates it.
ModuleFile(llvm::OwningPtr<llvm::MemoryBuffer> &&input);
// Out of line to avoid instantiation OnDiskChainedHashTable here.
~ModuleFile();
/// Convenience function for module loading.
void error(ModuleStatus issue = ModuleStatus::Malformed) {
assert(issue != ModuleStatus::Valid &&
@@ -277,7 +277,7 @@ public:
/// \returns Whether the module was successfully loaded, or what went wrong
/// if it was not.
static ModuleStatus load(llvm::OwningPtr<llvm::MemoryBuffer> &&input,
llvm::OwningPtr<ModuleFile> &module) {
std::unique_ptr<ModuleFile> &module) {
module.reset(new ModuleFile(std::move(input)));
return module->getStatus();
}
@@ -317,6 +317,11 @@ public:
void lookupVisibleDecls(Module::AccessPathTy accessPath,
VisibleDeclConsumer &consumer,
NLKind lookupKind);
/// Loads extensions for the given decl.
///
/// Note that this may cause other extensions to load as well.
void loadExtensions(NominalTypeDecl *nominal);
};
class SerializedModule : public LoadedModule {
@@ -337,25 +342,4 @@ public:
} // end namespace swift
namespace llvm {
template<> struct DenseMapInfo<swift::ModuleFile::OperatorKey> {
using OperatorKey = swift::ModuleFile::OperatorKey;
using Identifier = swift::Identifier;
static OperatorKey getEmptyKey() {
return OperatorKey(Identifier(), swift::serialization::Prefix);
}
static OperatorKey getTombstoneKey() {
return OperatorKey(Identifier(), swift::serialization::Postfix);
}
static unsigned getHashValue(OperatorKey Val) {
using RawPair = std::pair<Identifier, unsigned>;
return DenseMapInfo<RawPair>::getHashValue(RawPair(Val));
}
static bool isEqual(OperatorKey LHS, OperatorKey RHS) {
return LHS == RHS;
}
};
}
#endif

22
lib/Serialization/ModuleFormat.h
View File

@@ -18,6 +18,7 @@
#ifndef SWIFT_SERIALIZATION_MODULEFORMAT_H
#define SWIFT_SERIALIZATION_MODULEFORMAT_H
#include "swift/AST/Decl.h"
#include "swift/Serialization/BCRecordLayout.h"
#include "llvm/Bitcode/BitCodes.h"
@@ -87,6 +88,20 @@ enum OperatorKind : uint8_t {
static_assert(sizeof(OperatorKind) <= sizeof(TypeID),
"too many operator kinds");
/// Translates an operator DeclKind to a fixity.
static inline OperatorKind getRawStableFixity(DeclKind kind) {
switch (kind) {
case DeclKind::PrefixOperator:
return Prefix;
case DeclKind::PostfixOperator:
return Postfix;
case DeclKind::InfixOperator:
return Infix;
default:
llvm_unreachable("unknown operator fixity");
}
}
// These IDs must \em not be renumbered or reordered without incrementing
// VERSION_MAJOR.
enum GenericRequirementKind : uint8_t {
@@ -218,7 +233,7 @@ namespace input_block {
namespace decls_block {
// These IDs must \em not be renumbered or reordered without incrementing
// VERSION_MAJOR.
enum : uint8_t {
enum RecordKind : uint8_t {
NAME_ALIAS_TYPE = 1,
NOMINAL_TYPE,
PAREN_TYPE,
@@ -730,7 +745,7 @@ namespace identifier_block {
namespace index_block {
// These IDs must \em not be renumbered or reordered without incrementing
// VERSION_MAJOR.
enum {
enum RecordKind {
TYPE_OFFSETS = 1,
DECL_OFFSETS,
IDENTIFIER_OFFSETS,
@@ -745,7 +760,8 @@ namespace index_block {
using DeclListLayout = BCGenericRecordLayout<
BCFixed<3>, // record ID
BCArray<DeclIDField>
BCVBR<16>, // table offset within the blob (see below)
BCBlob // map from identifier strings to decl kinds / decl IDs
>;
}

150
lib/Serialization/Serialization.cpp
View File

@@ -17,6 +17,11 @@
#include "swift/Basic/STLExtras.h"
#include "swift/Basic/SourceManager.h"
#include "swift/Serialization/BCRecordLayout.h"
// This is a template-only header; eventually it should move to llvm/Support.
#include "clang/Basic/OnDiskHashTable.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Bitcode/BitstreamWriter.h"
#include "llvm/Config/config.h"
#include "llvm/Support/FileSystem.h"
@@ -27,6 +32,7 @@
using namespace swift;
using namespace swift::serialization;
using clang::OnDiskChainedHashTableGenerator;
namespace {
typedef ArrayRef<unsigned> FileBufferIDs;
@@ -946,20 +952,7 @@ bool Serializer::writeCrossReference(const Decl *D) {
} else if (auto op = dyn_cast<OperatorDecl>(D)) {
kind = XRefKind::SwiftOperator;
accessPath.push_back(addIdentifierRef(op->getName()));
switch (op->getKind()) {
case DeclKind::InfixOperator:
typeID = OperatorKind::Infix;
break;
case DeclKind::PrefixOperator:
typeID = OperatorKind::Prefix;
break;
case DeclKind::PostfixOperator:
typeID = OperatorKind::Postfix;
break;
default:
llvm_unreachable("unknown operator kind");
}
typeID = getRawStableFixity(op->getKind());
} else {
llvm_unreachable("cannot cross-reference this kind of decl");
}
@@ -1810,22 +1803,127 @@ void Serializer::writeOffsets(const index_block::OffsetsLayout &Offsets,
Offsets.emit(ScratchRecord, getOffsetRecordCode(values), values);
}
namespace {
/// Used to serialize the on-disk decl hash table.
class DeclTableInfo {
public:
using key_type = Identifier;
using key_type_ref = key_type;
using data_type = SmallVector<std::pair<uint8_t, DeclID>, 8>;
using data_type_ref = const data_type &;
uint32_t ComputeHash(key_type_ref key) {
assert(!key.empty());
return llvm::HashString(key.str());
}
std::pair<unsigned, unsigned> EmitKeyDataLength(raw_ostream &out,
key_type_ref key,
data_type_ref data) {
using namespace clang::io;
uint32_t keyLength = key.str().size();
uint32_t dataLength = (sizeof(DeclID) + 1) * data.size();
Emit16(out, keyLength);
Emit16(out, dataLength);
return { keyLength, dataLength };
}
void EmitKey(raw_ostream &out, key_type_ref key, unsigned len) {
out << key.str();
}
void EmitData(raw_ostream &out, key_type_ref key, data_type_ref data,
unsigned len) {
static_assert(sizeof(DeclID) <= 32, "DeclID too large");
using namespace clang::io;
for (auto entry : data) {
Emit8(out, entry.first);
Emit32(out, entry.second);
}
}
};
}
/// Returns the encoding kind for the given decl.
///
/// Note that this does not work for all encodable decls, only those designed
/// to be stored in a hash table.
static decls_block::RecordKind getDeclKind(const Decl *D) {
using namespace decls_block;
switch (D->getKind()) {
case DeclKind::TypeAlias:
return decls_block::TYPE_ALIAS_DECL;
case DeclKind::Union:
return decls_block::UNION_DECL;
case DeclKind::Struct:
return decls_block::STRUCT_DECL;
case DeclKind::Class:
return decls_block::CLASS_DECL;
case DeclKind::Protocol:
return decls_block::PROTOCOL_DECL;
case DeclKind::Func:
return decls_block::FUNC_DECL;
case DeclKind::Var:
return decls_block::VAR_DECL;
case DeclKind::Extension:
return decls_block::EXTENSION_DECL;
default:
llvm_unreachable("cannot store this kind of decl in a hash table");
}
}
/// The in-memory representation of what will eventually be an on-disk hash
/// table.
using DeclTable = llvm::DenseMap<Identifier, DeclTableInfo::data_type>;
/// Writes an in-memory decl table to an on-disk representation, using the
/// given layout.
static void writeDeclTable(const index_block::DeclListLayout &DeclList,
index_block::RecordKind kind,
const DeclTable &table) {
SmallVector<uint64_t, 8> scratch;
llvm::SmallString<4096> hashTableBlob;
uint32_t tableOffset;
{
OnDiskChainedHashTableGenerator<DeclTableInfo> generator;
for (auto &entry : table)
generator.insert(entry.first, entry.second);
llvm::raw_svector_ostream blobStream(hashTableBlob);
// Make sure that no bucket is at offset 0
clang::io::Emit32(blobStream, 0);
tableOffset = generator.Emit(blobStream);
}
DeclList.emit(scratch, kind, tableOffset, hashTableBlob);
}
void Serializer::writeTranslationUnit(const TranslationUnit *TU) {
assert(!this->TU && "already serializing a translation unit");
this->TU = TU;
SmallVector<DeclID, 32> topLevelIDs;
SmallVector<DeclID, 8> operatorIDs;
DeclTable topLevelDecls;
DeclTable operatorDecls;
for (auto D : TU->Decls) {
if (isa<ImportDecl>(D))
continue;
else if (isa<ValueDecl>(D))
topLevelIDs.push_back(addDeclRef(D));
else if (isa<ExtensionDecl>(D))
// FIXME: should have a lazy extension table
topLevelIDs.push_back(addDeclRef(D));
else if (isa<OperatorDecl>(D))
operatorIDs.push_back(addDeclRef(D));
else if (auto VD = dyn_cast<ValueDecl>(D)) {
if (VD->getName().empty())
continue;
topLevelDecls[VD->getName()]
.push_back({getDeclKind(D), addDeclRef(D)});
} else if (auto ED = dyn_cast<ExtensionDecl>(D)) {
// FIXME: should have a separate extension table.
topLevelDecls[ED->getExtendedType()->getAnyNominal()->getName()]
.push_back({getDeclKind(D), addDeclRef(D)});
} else if (auto OD = dyn_cast<OperatorDecl>(D)) {
operatorDecls[OD->getName()]
.push_back({getRawStableFixity(OD->getKind()), addDeclRef(D)});
}
}
writeAllDeclsAndTypes();
@@ -1833,15 +1931,15 @@ void Serializer::writeTranslationUnit(const TranslationUnit *TU) {
{
BCBlockRAII restoreBlock(Out, INDEX_BLOCK_ID, 3);
index_block::OffsetsLayout Offsets(Out);
index_block::OffsetsLayout Offsets(Out);
writeOffsets(Offsets, DeclOffsets);
writeOffsets(Offsets, TypeOffsets);
writeOffsets(Offsets, IdentifierOffsets);
index_block::DeclListLayout DeclList(Out);
DeclList.emit(ScratchRecord, index_block::TOP_LEVEL_DECLS, topLevelIDs);
DeclList.emit(ScratchRecord, index_block::OPERATORS, operatorIDs);
writeDeclTable(DeclList, index_block::TOP_LEVEL_DECLS, topLevelDecls);
writeDeclTable(DeclList, index_block::OPERATORS, operatorDecls);
}
#ifndef NDEBUG

15
lib/Serialization/SerializedModuleLoader.cpp
View File

@@ -138,7 +138,7 @@ Module *SerializedModuleLoader::loadModule(SourceLoc importLoc,
assert(inputFile);
std::string DebugModuleName = inputFile->getBufferIdentifier();
llvm::OwningPtr<ModuleFile> loadedModuleFile;
std::unique_ptr<ModuleFile> loadedModuleFile;
ModuleStatus err = ModuleFile::load(std::move(inputFile), loadedModuleFile);
switch (err) {
case ModuleStatus::FallBackToTranslationUnit:
@@ -169,7 +169,8 @@ Module *SerializedModuleLoader::loadModule(SourceLoc importLoc,
if (loadedModuleFile) {
bool success = loadedModuleFile->associateWithModule(module);
if (success) {
LoadedModuleFiles.push_back(std::move(loadedModuleFile));
LoadedModuleFiles.emplace_back(std::move(loadedModuleFile),
Ctx.getCurrentGeneration());
} else {
assert(loadedModuleFile->getStatus() == ModuleStatus::MissingDependency);
@@ -260,3 +261,13 @@ SerializedModuleLoader::lookupVisibleDecls(const Module *module,
moduleFile->lookupVisibleDecls(accessPath, consumer, lookupKind);
}
void SerializedModuleLoader::loadExtensions(NominalTypeDecl *nominal,
unsigned previousGeneration) {
for (auto &modulePair : LoadedModuleFiles) {
if (modulePair.second <= previousGeneration)
continue;
modulePair.first->loadExtensions(nominal);
}
}