averello/swift-mirror
1
0
Fork 0
mirror of https://github.com/apple/swift.git synced 2025-12-14 20:36:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

[Index] Add an option to compress the record and unit files

Browse Source 
Companion of https://github.com/swiftlang/llvm-project/pull/10977.
This commit is contained in:
Alex Hoppen
2025-07-10 15:09:41 +02:00
parent ba715db935
commit ed3b64af2c
10 changed files with 46 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
include/swift/Frontend/FrontendOptions.h
View File

@@ -136,6 +136,9 @@ public:
/// Include local definitions/references in the index data. /// Include local definitions/references in the index data.
bool IndexIncludeLocals = false; bool IndexIncludeLocals = false;
/// Whether to compress the record and unit files in the index store.
bool IndexStoreCompress;
bool SerializeDebugInfoSIL = false; bool SerializeDebugInfoSIL = false;
/// If building a module from interface, ignore compiler flags /// If building a module from interface, ignore compiler flags
/// specified in the swiftinterface. /// specified in the swiftinterface.

4
include/swift/Index/IndexRecord.h
View File

@@ -56,7 +56,7 @@ namespace index {
bool indexAndRecord(SourceFile *primarySourceFile, StringRef indexUnitToken, bool indexAndRecord(SourceFile *primarySourceFile, StringRef indexUnitToken,
StringRef indexStorePath, bool indexClangModules, StringRef indexStorePath, bool indexClangModules,
bool indexSystemModules, bool skipStdlib, bool indexSystemModules, bool skipStdlib,
bool includeLocals, bool isDebugCompilation, bool includeLocals, bool compress, bool isDebugCompilation,
bool isExplicitModuleBuild, StringRef targetTriple, bool isExplicitModuleBuild, StringRef targetTriple,
const DependencyTracker &dependencyTracker, const DependencyTracker &dependencyTracker,
const PathRemapper &pathRemapper); const PathRemapper &pathRemapper);
@@ -98,7 +98,7 @@ bool indexAndRecord(SourceFile *primarySourceFile, StringRef indexUnitToken,
bool indexAndRecord(ModuleDecl *module, ArrayRef<std::string> indexUnitTokens, bool indexAndRecord(ModuleDecl *module, ArrayRef<std::string> indexUnitTokens,
StringRef moduleUnitToken, StringRef indexStorePath, StringRef moduleUnitToken, StringRef indexStorePath,
bool indexClangModules, bool indexSystemModules, bool indexClangModules, bool indexSystemModules,
bool skipStdlib, bool includeLocals, bool skipStdlib, bool includeLocals, bool compress,
bool isDebugCompilation, bool isExplicitModuleBuild, bool isDebugCompilation, bool isExplicitModuleBuild,
StringRef targetTriple, StringRef targetTriple,
const DependencyTracker &dependencyTracker, const DependencyTracker &dependencyTracker,

4
include/swift/Option/Options.td
View File

@@ -1686,6 +1686,10 @@ def index_store_path : Separate<["-"], "index-store-path">,
Flags<[FrontendOption, ArgumentIsPath, CacheInvariant]>, MetaVarName<"<path>">, Flags<[FrontendOption, ArgumentIsPath, CacheInvariant]>, MetaVarName<"<path>">,
HelpText<"Store indexing data to <path>">; HelpText<"Store indexing data to <path>">;
def index_store_compress : Flag<["-"], "index-store-compress">,
Flags<[FrontendOption]>,
HelpText<"Compress the unit and record files in the index store">;
def index_unit_output_path : Separate<["-"], "index-unit-output-path">, def index_unit_output_path : Separate<["-"], "index-unit-output-path">,
Flags<[FrontendOption, ArgumentIsPath, CacheInvariant]>, MetaVarName<"<path>">, Flags<[FrontendOption, ArgumentIsPath, CacheInvariant]>, MetaVarName<"<path>">,
HelpText<"Use <path> as the output path in the produced index data.">; HelpText<"Use <path> as the output path in the produced index data.">;

2
lib/FrontendTool/FrontendTool.cpp
View File

@@ -1884,6 +1884,7 @@ static void emitIndexDataForSourceFile(SourceFile *PrimarySourceFile,
opts.IndexSystemModules, opts.IndexSystemModules,
opts.IndexIgnoreStdlib, opts.IndexIgnoreStdlib,
opts.IndexIncludeLocals, opts.IndexIncludeLocals,
opts.IndexStoreCompress,
isDebugCompilation, isDebugCompilation,
opts.DisableImplicitModules, opts.DisableImplicitModules,
Invocation.getTargetTriple(), Invocation.getTargetTriple(),
@@ -1903,6 +1904,7 @@ static void emitIndexDataForSourceFile(SourceFile *PrimarySourceFile,
opts.IndexSystemModules, opts.IndexSystemModules,
opts.IndexIgnoreStdlib, opts.IndexIgnoreStdlib,
opts.IndexIncludeLocals, opts.IndexIncludeLocals,
opts.IndexStoreCompress,
isDebugCompilation, isDebugCompilation,
opts.DisableImplicitModules, opts.DisableImplicitModules,
Invocation.getTargetTriple(), Invocation.getTargetTriple(),

38
lib/Index/IndexRecord.cpp
View File

@@ -311,9 +311,9 @@ StringRef StdlibGroupsIndexRecordingConsumer::findGroupForSymbol(const IndexSymb
} }
static bool writeRecord(SymbolTracker &record, std::string Filename, static bool writeRecord(SymbolTracker &record, std::string Filename,
std::string indexStorePath, DiagnosticEngine *diags, std::string indexStorePath, bool compress, DiagnosticEngine *diags,
std::string &outRecordFile) { std::string &outRecordFile) {
IndexRecordWriter recordWriter(indexStorePath); IndexRecordWriter recordWriter(indexStorePath, compress);
std::string error; std::string error;
auto result = recordWriter.beginRecord( auto result = recordWriter.beginRecord(
Filename, record.hashRecord(), error, &outRecordFile); Filename, record.hashRecord(), error, &outRecordFile);
@@ -360,25 +360,25 @@ static bool writeRecord(SymbolTracker &record, std::string Filename,
static std::unique_ptr<IndexRecordingConsumer> static std::unique_ptr<IndexRecordingConsumer>
makeRecordingConsumer(std::string Filename, std::string indexStorePath, makeRecordingConsumer(std::string Filename, std::string indexStorePath,
bool includeLocals, DiagnosticEngine *diags, bool includeLocals, bool compress, DiagnosticEngine *diags,
std::string *outRecordFile, std::string *outRecordFile,
bool *outFailed) { bool *outFailed) {
return std::make_unique<IndexRecordingConsumer>(includeLocals, return std::make_unique<IndexRecordingConsumer>(includeLocals,
[=](SymbolTracker &record) { [=](SymbolTracker &record) {
*outFailed = writeRecord(record, Filename, indexStorePath, diags, *outFailed = writeRecord(record, Filename, indexStorePath, compress, diags,
*outRecordFile); *outRecordFile);
}); });
} }
static bool static bool
recordSourceFile(SourceFile *SF, StringRef indexStorePath, recordSourceFile(SourceFile *SF, StringRef indexStorePath,
bool includeLocals, DiagnosticEngine &diags, bool includeLocals, bool compress, DiagnosticEngine &diags,
llvm::function_ref<void(StringRef, StringRef)> callback) { llvm::function_ref<void(StringRef, StringRef)> callback) {
std::string recordFile; std::string recordFile;
bool failed = false; bool failed = false;
auto consumer = auto consumer =
makeRecordingConsumer(SF->getFilename().str(), indexStorePath.str(), makeRecordingConsumer(SF->getFilename().str(), indexStorePath.str(),
includeLocals, &diags, &recordFile, &failed); includeLocals, compress, &diags, &recordFile, &failed);
indexSourceFile(SF, *consumer); indexSourceFile(SF, *consumer);
if (!failed && !recordFile.empty()) if (!failed && !recordFile.empty())
@@ -418,6 +418,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
bool indexSystemModules, bool indexSystemModules,
bool skipStdlib, bool skipStdlib,
bool includeLocals, bool includeLocals,
bool compress,
bool explicitModulebuild, bool explicitModulebuild,
StringRef targetTriple, StringRef targetTriple,
const clang::CompilerInstance &clangCI, const clang::CompilerInstance &clangCI,
@@ -432,6 +433,7 @@ static void addModuleDependencies(ArrayRef<ImportedModule> imports,
bool indexSystemModules, bool indexSystemModules,
bool skipStdlib, bool skipStdlib,
bool includeLocals, bool includeLocals,
bool compress,
bool explicitModuleBuild, bool explicitModuleBuild,
StringRef targetTriple, StringRef targetTriple,
const clang::CompilerInstance &clangCI, const clang::CompilerInstance &clangCI,
@@ -508,6 +510,7 @@ static void addModuleDependencies(ArrayRef<ImportedModule> imports,
indexClangModules, indexClangModules,
indexSystemModules, skipStdlib, indexSystemModules, skipStdlib,
includeLocals, includeLocals,
compress,
explicitModuleBuild, explicitModuleBuild,
targetTriple, targetTriple,
clangCI, diags, clangCI, diags,
@@ -551,6 +554,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
bool indexSystemModules, bool indexSystemModules,
bool skipStdlib, bool skipStdlib,
bool includeLocals, bool includeLocals,
bool compress,
bool explicitModuleBuild, bool explicitModuleBuild,
StringRef targetTriple, StringRef targetTriple,
const clang::CompilerInstance &clangCI, const clang::CompilerInstance &clangCI,
@@ -632,7 +636,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
std::string recordFile; std::string recordFile;
bool failed = false; bool failed = false;
auto consumer = makeRecordingConsumer(filename.str(), indexStorePath.str(), auto consumer = makeRecordingConsumer(filename.str(), indexStorePath.str(),
includeLocals, &diags, &recordFile, &failed); includeLocals, compress, &diags, &recordFile, &failed);
indexModule(module, *consumer); indexModule(module, *consumer);
if (failed) if (failed)
@@ -676,7 +680,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
std::string outRecordFile; std::string outRecordFile;
failed = failed =
failed || writeRecord(tracker, std::string(fileNameWithGroup.str()), failed || writeRecord(tracker, std::string(fileNameWithGroup.str()),
indexStorePath.str(), &diags, outRecordFile); indexStorePath.str(), compress, &diags, outRecordFile);
if (failed) if (failed)
return false; return false;
records.emplace_back(outRecordFile, moduleName.str().str()); records.emplace_back(outRecordFile, moduleName.str().str());
@@ -698,7 +702,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
auto clangRemapper = pathRemapper.asClangPathRemapper(); auto clangRemapper = pathRemapper.asClangPathRemapper();
IndexUnitWriter unitWriter( IndexUnitWriter unitWriter(
fileMgr, indexStorePath, "swift", swiftVersion, filename, moduleName, fileMgr, indexStorePath, "swift", swiftVersion, compress, filename, moduleName,
/*MainFile=*/{}, isSystem, /*IsModuleUnit=*/true, isDebugCompilation, /*MainFile=*/{}, isSystem, /*IsModuleUnit=*/true, isDebugCompilation,
targetTriple, sysrootPath, clangRemapper, getModuleInfoFromOpaqueModule); targetTriple, sysrootPath, clangRemapper, getModuleInfoFromOpaqueModule);
@@ -718,7 +722,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
ModuleDecl::ImportFilterKind::Default}); ModuleDecl::ImportFilterKind::Default});
StringScratchSpace moduleNameScratch; StringScratchSpace moduleNameScratch;
addModuleDependencies(imports, indexStorePath, indexClangModules, addModuleDependencies(imports, indexStorePath, indexClangModules,
indexSystemModules, skipStdlib, includeLocals, indexSystemModules, skipStdlib, includeLocals, compress,
explicitModuleBuild, explicitModuleBuild,
targetTriple, clangCI, diags, unitWriter, targetTriple, clangCI, diags, unitWriter,
moduleNameScratch, pathRemapper, initialFile); moduleNameScratch, pathRemapper, initialFile);
@@ -735,7 +739,7 @@ static bool
recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken, recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken,
StringRef indexStorePath, bool indexClangModules, StringRef indexStorePath, bool indexClangModules,
bool indexSystemModules, bool skipStdlib, bool indexSystemModules, bool skipStdlib,
bool includeLocals, bool isDebugCompilation, bool includeLocals, bool compress, bool isDebugCompilation,
bool isExplicitModuleBuild, StringRef targetTriple, bool isExplicitModuleBuild, StringRef targetTriple,
ArrayRef<clang::FileEntryRef> fileDependencies, ArrayRef<clang::FileEntryRef> fileDependencies,
const clang::CompilerInstance &clangCI, const clang::CompilerInstance &clangCI,
@@ -754,7 +758,7 @@ recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken,
StringRef swiftVersion; StringRef swiftVersion;
StringRef sysrootPath = clangCI.getHeaderSearchOpts().Sysroot; StringRef sysrootPath = clangCI.getHeaderSearchOpts().Sysroot;
IndexUnitWriter unitWriter( IndexUnitWriter unitWriter(
fileMgr, indexStorePath, "swift", swiftVersion, indexUnitToken, fileMgr, indexStorePath, "swift", swiftVersion, compress, indexUnitToken,
module->getNameStr(), *mainFile, isSystem, module->getNameStr(), *mainFile, isSystem,
/*isModuleUnit=*/false, isDebugCompilation, targetTriple, sysrootPath, /*isModuleUnit=*/false, isDebugCompilation, targetTriple, sysrootPath,
clangRemapper, getModuleInfoFromOpaqueModule); clangRemapper, getModuleInfoFromOpaqueModule);
@@ -765,7 +769,7 @@ recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken,
ModuleDecl::getImportFilterLocal()); ModuleDecl::getImportFilterLocal());
StringScratchSpace moduleNameScratch; StringScratchSpace moduleNameScratch;
addModuleDependencies(imports, indexStorePath, indexClangModules, addModuleDependencies(imports, indexStorePath, indexClangModules,
indexSystemModules, skipStdlib, includeLocals, indexSystemModules, skipStdlib, includeLocals, compress,
isExplicitModuleBuild, targetTriple, clangCI, diags, isExplicitModuleBuild, targetTriple, clangCI, diags,
unitWriter, moduleNameScratch, pathRemapper, unitWriter, moduleNameScratch, pathRemapper,
primarySourceFile); primarySourceFile);
@@ -774,7 +778,7 @@ recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken,
for (auto F : fileDependencies) for (auto F : fileDependencies)
unitWriter.addFileDependency(F, /*FIXME:isSystem=*/false, /*Module=*/nullptr); unitWriter.addFileDependency(F, /*FIXME:isSystem=*/false, /*Module=*/nullptr);
recordSourceFile(primarySourceFile, indexStorePath, includeLocals, diags, recordSourceFile(primarySourceFile, indexStorePath, includeLocals, compress, diags,
[&](StringRef recordFile, StringRef filename) { [&](StringRef recordFile, StringRef filename) {
if (auto file = fileMgr.getFileRef(filename)) { if (auto file = fileMgr.getFileRef(filename)) {
unitWriter.addRecordFile( unitWriter.addRecordFile(
@@ -823,6 +827,7 @@ bool index::indexAndRecord(SourceFile *primarySourceFile,
bool indexSystemModules, bool indexSystemModules,
bool skipStdlib, bool skipStdlib,
bool includeLocals, bool includeLocals,
bool compress,
bool isDebugCompilation, bool isDebugCompilation,
bool isExplicitModuleBuild, bool isExplicitModuleBuild,
StringRef targetTriple, StringRef targetTriple,
@@ -840,7 +845,7 @@ bool index::indexAndRecord(SourceFile *primarySourceFile,
return recordSourceFileUnit(primarySourceFile, indexUnitToken, return recordSourceFileUnit(primarySourceFile, indexUnitToken,
indexStorePath, indexClangModules, indexStorePath, indexClangModules,
indexSystemModules, skipStdlib, includeLocals, indexSystemModules, skipStdlib, includeLocals, compress,
isDebugCompilation, isExplicitModuleBuild, isDebugCompilation, isExplicitModuleBuild,
targetTriple, {}, targetTriple, {},
clangCI, pathRemapper, diags); clangCI, pathRemapper, diags);
@@ -854,6 +859,7 @@ bool index::indexAndRecord(ModuleDecl *module,
bool indexSystemModules, bool indexSystemModules,
bool skipStdlib, bool skipStdlib,
bool includeLocals, bool includeLocals,
bool compress,
bool isDebugCompilation, bool isDebugCompilation,
bool isExplicitModuleBuild, bool isExplicitModuleBuild,
StringRef targetTriple, StringRef targetTriple,
@@ -879,7 +885,7 @@ bool index::indexAndRecord(ModuleDecl *module,
} }
if (recordSourceFileUnit(SF, indexUnitTokens[unitIndex], if (recordSourceFileUnit(SF, indexUnitTokens[unitIndex],
indexStorePath, indexClangModules, indexStorePath, indexClangModules,
indexSystemModules, skipStdlib, includeLocals, indexSystemModules, skipStdlib, includeLocals, compress,
isDebugCompilation, isExplicitModuleBuild, isDebugCompilation, isExplicitModuleBuild,
targetTriple, {}, targetTriple, {},
clangCI, pathRemapper, diags)) clangCI, pathRemapper, diags))

6
test/Index/Store/index_compress.swift Normal file
View File

@@ -0,0 +1,6 @@
// RUN: rm -rf %t
// RUN: %target-swift-frontend -index-store-path %t/idx -index-store-compress -o %t.o -typecheck %s
// RUN: c-index-test core -print-record %t/idx | %FileCheck %s
func foo() {}
// CHECK: [[@LINE-1]]:6 | function/Swift | s:4main3fooyyF | Def | rel: 0

1
tools/SourceKit/include/SourceKit/Core/LangSupport.h
View File

@@ -922,6 +922,7 @@ struct IndexStoreOptions {
bool IgnoreStdlib = false; bool IgnoreStdlib = false;
bool DisableImplicitModules = false; bool DisableImplicitModules = false;
bool IncludeLocals = false; bool IncludeLocals = false;
bool Compress = false;
}; };
struct IndexStoreInfo{}; struct IndexStoreInfo{};

1
tools/SourceKit/lib/SwiftLang/SwiftIndexing.cpp
View File

@@ -403,6 +403,7 @@ static void emitIndexDataForSourceFile(SourceFile &PrimarySourceFile,
IndexOpts.IncludeSystemModules, IndexOpts.IncludeSystemModules,
IndexOpts.IgnoreStdlib, IndexOpts.IgnoreStdlib,
IndexOpts.IncludeLocals, IndexOpts.IncludeLocals,
IndexOpts.Compress,
isDebugCompilation, isDebugCompilation,
IndexOpts.DisableImplicitModules, IndexOpts.DisableImplicitModules,
Invocation.getTargetTriple(), Invocation.getTargetTriple(),

4
tools/SourceKit/tools/sourcekitd/lib/Service/Requests.cpp
View File

@@ -1512,6 +1512,10 @@ getIndexStoreOpts(const RequestDict &Req, ResponseReceiver Rec) {
Opts.IncludeLocals = IncludeLocals.value() > 0; Opts.IncludeLocals = IncludeLocals.value() > 0;
} }
if (auto Compress = Req.getOptionalInt64(KeyCompress)) {
Opts.Compress = Compress.value() > 0;
}
if (auto IgnoreClangModules = Req.getOptionalInt64(KeyIgnoreClangModules)) { if (auto IgnoreClangModules = Req.getOptionalInt64(KeyIgnoreClangModules)) {
Opts.IgnoreClangModules = IgnoreClangModules.value() > 0; Opts.IgnoreClangModules = IgnoreClangModules.value() > 0;
} }

1
utils/gyb_sourcekit_support/UIDs.py
View File

@@ -219,6 +219,7 @@ UID_KEYS = [
KEY('IndexStorePath', 'key.index_store_path'), KEY('IndexStorePath', 'key.index_store_path'),
KEY('IndexUnitOutputPath', 'key.index_unit_output_path'), KEY('IndexUnitOutputPath', 'key.index_unit_output_path'),
KEY('IncludeLocals', 'key.include_locals'), KEY('IncludeLocals', 'key.include_locals'),
KEY('Compress', 'key.compress'),
KEY('IgnoreClangModules', 'key.ignore_clang_modules'), KEY('IgnoreClangModules', 'key.ignore_clang_modules'),
KEY('IncludeSystemModules', 'key.include_system_modules'), KEY('IncludeSystemModules', 'key.include_system_modules'),
KEY('IgnoreStdlib', 'key.ignore_stdlib'), KEY('IgnoreStdlib', 'key.ignore_stdlib'),