Merge remote-tracking branch 'origin/master' into master-next

This commit is contained in:
swift_jenkins
2020-07-01 14:59:55 -07:00
15 changed files with 234 additions and 118 deletions

View File

@@ -284,6 +284,9 @@ public:
/// Whether to disable using mangled names for accessing concrete type metadata.
unsigned DisableConcreteTypeMetadataMangledNameAccessors : 1;
/// The number of threads for multi-threaded code generation.
unsigned NumThreads = 0;
/// Path to the profdata file to be used for PGO, or the empty string.
std::string UseProfile = "";
@@ -383,6 +386,10 @@ public:
llvm::hash_code getPCHHashComponents() const {
return llvm::hash_value(0);
}
bool hasMultipleIRGenThreads() const { return NumThreads > 1; }
bool shouldPerformIRGenerationInParallel() const { return NumThreads != 0; }
bool hasMultipleIGMs() const { return hasMultipleIRGenThreads(); }
};
} // end namespace swift

View File

@@ -172,6 +172,13 @@ public:
outModuleHash,
LinkerDirectives};
}
/// Retrieves the files to perform IR generation for.
TinyPtrVector<FileUnit *> getFiles() const;
/// For a single file, returns its parent module, otherwise returns the module
/// itself.
ModuleDecl *getParentModule() const;
};
/// Report that a request of the given kind is being evaluated, so it
@@ -180,10 +187,10 @@ template<typename Request>
void reportEvaluatedRequest(UnifiedStatsReporter &stats,
const Request &request);
class IRGenSourceFileRequest
: public SimpleRequest<IRGenSourceFileRequest,
GeneratedModule(IRGenDescriptor),
RequestFlags::Uncached|RequestFlags::DependencySource> {
class IRGenRequest
: public SimpleRequest<IRGenRequest, GeneratedModule(IRGenDescriptor),
RequestFlags::Uncached |
RequestFlags::DependencySource> {
public:
using SimpleRequest::SimpleRequest;
@@ -194,33 +201,12 @@ private:
GeneratedModule
evaluate(Evaluator &evaluator, IRGenDescriptor desc) const;
public:
bool isCached() const { return true; }
public:
// Incremental dependencies.
evaluator::DependencySource
readDependencySource(const evaluator::DependencyRecorder &) const;
};
class IRGenWholeModuleRequest
: public SimpleRequest<IRGenWholeModuleRequest,
GeneratedModule(IRGenDescriptor),
RequestFlags::Uncached> {
public:
using SimpleRequest::SimpleRequest;
private:
friend SimpleRequest;
// Evaluation.
GeneratedModule
evaluate(Evaluator &evaluator, IRGenDescriptor desc) const;
public:
bool isCached() const { return true; }
};
void simple_display(llvm::raw_ostream &out, const IRGenDescriptor &d);
SourceLoc extractNearestSourceLoc(const IRGenDescriptor &desc);

View File

@@ -14,10 +14,6 @@
//
//===----------------------------------------------------------------------===//
SWIFT_REQUEST(IRGen, IRGenSourceFileRequest,
SWIFT_REQUEST(IRGen, IRGenRequest,
GeneratedModule(IRGenDescriptor),
Uncached, NoLocationInfo)
SWIFT_REQUEST(IRGen, IRGenWholeModuleRequest,
GeneratedModule(IRGenDescriptor),
Uncached, NoLocationInfo)

View File

@@ -40,9 +40,6 @@ public:
/// Controls the aggressiveness of the loop unroller.
int UnrollThreshold = 250;
/// The number of threads for multi-threaded code generation.
unsigned NumThreads = 0;
/// Controls whether to pull in SIL from partial modules during the
/// merge modules step. Could perhaps be merged with the link mode
/// above but the interactions between all the flags are tricky.
@@ -183,10 +180,6 @@ public:
bool shouldOptimize() const {
return OptMode > OptimizationMode::NoOptimization;
}
bool hasMultipleIRGenThreads() const { return NumThreads > 1; }
bool shouldPerformIRGenerationInParallel() const { return NumThreads != 0; }
bool hasMultipleIGMs() const { return hasMultipleIRGenThreads(); }
};
} // end namespace swift

View File

@@ -1043,18 +1043,6 @@ static bool ParseSILArgs(SILOptions &Opts, ArgList &Args,
return true;
}
}
if (const Arg *A = Args.getLastArg(OPT_num_threads)) {
if (StringRef(A->getValue()).getAsInteger(10, Opts.NumThreads)) {
Diags.diagnose(SourceLoc(), diag::error_invalid_arg_value,
A->getAsString(Args), A->getValue());
return true;
}
if (environmentVariableRequestedMaximumDeterminism()) {
Opts.NumThreads = 1;
Diags.diagnose(SourceLoc(), diag::remark_max_determinism_overriding,
"-num-threads");
}
}
// If we're only emitting a module, stop optimizations once we've serialized
// the SIL for the module.
@@ -1259,7 +1247,7 @@ static bool ParseTBDGenArgs(TBDGenOptions &Opts, ArgList &Args,
CompilerInvocation &Invocation) {
using namespace options;
Opts.HasMultipleIGMs = Invocation.getSILOptions().hasMultipleIGMs();
Opts.HasMultipleIGMs = Invocation.getIRGenOptions().hasMultipleIGMs();
if (const Arg *A = Args.getLastArg(OPT_module_link_name)) {
Opts.ModuleLinkName = A->getValue();
@@ -1593,6 +1581,19 @@ static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args,
getRuntimeCompatVersion();
}
if (const Arg *A = Args.getLastArg(OPT_num_threads)) {
if (StringRef(A->getValue()).getAsInteger(10, Opts.NumThreads)) {
Diags.diagnose(SourceLoc(), diag::error_invalid_arg_value,
A->getAsString(Args), A->getValue());
return true;
}
if (environmentVariableRequestedMaximumDeterminism()) {
Opts.NumThreads = 1;
Diags.diagnose(SourceLoc(), diag::remark_max_determinism_overriding,
"-num-threads");
}
}
return false;
}

View File

@@ -443,6 +443,7 @@ public:
/// Emit all the top-level code in the source file.
void IRGenModule::emitSourceFile(SourceFile &SF) {
assert(SF.ASTStage == SourceFile::TypeChecked);
PrettySourceFileEmission StackEntry(SF);
// Emit types and other global decls.

View File

@@ -889,14 +889,16 @@ static void runIRGenPreparePasses(SILModule &Module,
/// Generates LLVM IR, runs the LLVM passes and produces the output file.
/// All this is done in a single thread.
static GeneratedModule
performIRGeneration(const IRGenOptions &Opts, ModuleDecl *M,
std::unique_ptr<SILModule> SILMod, StringRef ModuleName,
const PrimarySpecificPaths &PSPs,
StringRef PrivateDiscriminator,
SourceFile *SF = nullptr,
llvm::GlobalVariable **outModuleHash = nullptr,
llvm::StringSet<> *linkerDirectives = nullptr) {
GeneratedModule IRGenRequest::evaluate(Evaluator &evaluator,
IRGenDescriptor desc) const {
const auto &Opts = desc.Opts;
const auto &PSPs = desc.PSPs;
auto SILMod = std::unique_ptr<SILModule>(desc.SILMod);
auto *M = desc.getParentModule();
auto filesToEmit = desc.getFiles();
auto *primaryFile = desc.Ctx.dyn_cast<SourceFile *>();
auto &Ctx = M->getASTContext();
assert(!Ctx.hadError());
@@ -906,9 +908,9 @@ performIRGeneration(const IRGenOptions &Opts, ModuleDecl *M,
if (!targetMachine) return GeneratedModule::null();
// Create the IR emitter.
IRGenModule IGM(irgen, std::move(targetMachine), SF, ModuleName,
IRGenModule IGM(irgen, std::move(targetMachine), primaryFile, desc.ModuleName,
PSPs.OutputFilename, PSPs.MainInputFilenameForDebugInfo,
PrivateDiscriminator);
desc.PrivateDiscriminator);
initLLVMModule(IGM, *SILMod);
@@ -919,27 +921,19 @@ performIRGeneration(const IRGenOptions &Opts, ModuleDecl *M,
FrontendStatsTracer tracer(Ctx.Stats, "IRGen");
// Emit the module contents.
irgen.emitGlobalTopLevel(linkerDirectives);
irgen.emitGlobalTopLevel(desc.LinkerDirectives);
if (SF) {
IGM.emitSourceFile(*SF);
// Emit synthesized file unit, if it exists.
if (auto *synthesizedFile = SF->getSynthesizedFile())
IGM.emitSynthesizedFileUnit(*synthesizedFile);
} else {
for (auto *File : M->getFiles()) {
if (auto *nextSF = dyn_cast<SourceFile>(File)) {
if (nextSF->ASTStage >= SourceFile::TypeChecked)
for (auto *file : filesToEmit) {
if (auto *nextSF = dyn_cast<SourceFile>(file)) {
IGM.emitSourceFile(*nextSF);
} else if (auto *nextSFU = dyn_cast<SynthesizedFileUnit>(File)) {
} else if (auto *nextSFU = dyn_cast<SynthesizedFileUnit>(file)) {
IGM.emitSynthesizedFileUnit(*nextSFU);
} else {
File->collectLinkLibraries([&IGM](LinkLibrary LinkLib) {
file->collectLinkLibraries([&IGM](LinkLibrary LinkLib) {
IGM.addLinkLibrary(LinkLib);
});
}
}
}
// Okay, emit any definitions that we suddenly need.
irgen.emitLazyDefinitions();
@@ -990,7 +984,7 @@ performIRGeneration(const IRGenOptions &Opts, ModuleDecl *M,
embedBitcode(IGM.getModule(), Opts);
if (outModuleHash) {
if (auto **outModuleHash = desc.outModuleHash) {
*outModuleHash = IGM.ModuleHash;
} else {
FrontendStatsTracer tracer(Ctx.Stats, "LLVM pipeline");
@@ -1138,7 +1132,7 @@ static void performParallelIRGeneration(
bool DidRunSILCodeGenPreparePasses = false;
for (auto *File : M->getFiles()) {
auto nextSF = dyn_cast<SourceFile>(File);
if (!nextSF || nextSF->ASTStage < SourceFile::TypeChecked)
if (!nextSF)
continue;
// There must be an output filename for each source file.
@@ -1286,7 +1280,7 @@ static void performParallelIRGeneration(
// Start all the threads and do the LLVM compilation.
LLVMCodeGenThreads codeGenThreads(&irgen, &DiagMutex, SILMod->getOptions().NumThreads - 1);
LLVMCodeGenThreads codeGenThreads(&irgen, &DiagMutex, Opts.NumThreads - 1);
codeGenThreads.startThreads();
// Free the memory occupied by the SILModule.
@@ -1307,30 +1301,19 @@ GeneratedModule swift::performIRGeneration(
const PrimarySpecificPaths &PSPs,
ArrayRef<std::string> parallelOutputFilenames,
llvm::GlobalVariable **outModuleHash, llvm::StringSet<> *LinkerDirectives) {
auto desc = IRGenDescriptor::forWholeModule(
Opts, M, std::move(SILMod), ModuleName, PSPs,
parallelOutputFilenames, outModuleHash, LinkerDirectives);
return llvm::cantFail(
M->getASTContext().evaluator(IRGenWholeModuleRequest{desc}));
}
GeneratedModule
IRGenWholeModuleRequest::evaluate(Evaluator &evaluator,
IRGenDescriptor desc) const {
auto *M = desc.Ctx.get<ModuleDecl *>();
if (desc.SILMod->getOptions().shouldPerformIRGenerationInParallel() &&
!desc.parallelOutputFilenames.empty()) {
::performParallelIRGeneration(
desc.Opts, M, std::unique_ptr<SILModule>(desc.SILMod), desc.ModuleName,
desc.parallelOutputFilenames, desc.LinkerDirectives);
if (Opts.shouldPerformIRGenerationInParallel() &&
!parallelOutputFilenames.empty()) {
::performParallelIRGeneration(Opts, M, std::move(SILMod), ModuleName,
parallelOutputFilenames, LinkerDirectives);
// TODO: Parallel LLVM compilation cannot be used if a (single) module is
// needed as return value.
return GeneratedModule::null();
}
return ::performIRGeneration(
desc.Opts, M, std::unique_ptr<SILModule>(desc.SILMod), desc.ModuleName,
desc.PSPs, "", nullptr, desc.outModuleHash,
desc.LinkerDirectives);
auto desc = IRGenDescriptor::forWholeModule(
Opts, M, std::move(SILMod), ModuleName, PSPs, parallelOutputFilenames,
outModuleHash, LinkerDirectives);
return llvm::cantFail(M->getASTContext().evaluator(IRGenRequest{desc}));
}
GeneratedModule swift::
@@ -1343,18 +1326,7 @@ performIRGeneration(const IRGenOptions &Opts, SourceFile &SF,
auto desc = IRGenDescriptor::forFile(Opts, SF, std::move(SILMod), ModuleName,
PSPs, PrivateDiscriminator,
outModuleHash, LinkerDirectives);
return llvm::cantFail(
SF.getASTContext().evaluator(IRGenSourceFileRequest{desc}));
}
GeneratedModule
IRGenSourceFileRequest::evaluate(Evaluator &evaluator,
IRGenDescriptor desc) const {
auto *SF = desc.Ctx.get<SourceFile *>();
return ::performIRGeneration(
desc.Opts, SF->getParentModule(), std::unique_ptr<SILModule>(desc.SILMod),
desc.ModuleName, desc.PSPs, desc.PrivateDiscriminator,
SF, desc.outModuleHash, desc.LinkerDirectives);
return llvm::cantFail(SF.getASTContext().evaluator(IRGenRequest{desc}));
}
void

View File

@@ -52,13 +52,40 @@ SourceLoc swift::extractNearestSourceLoc(const IRGenDescriptor &desc) {
return SourceLoc();
}
evaluator::DependencySource IRGenSourceFileRequest::readDependencySource(
TinyPtrVector<FileUnit *> IRGenDescriptor::getFiles() const {
// For a whole module, we emit IR for all files.
if (auto *mod = Ctx.dyn_cast<ModuleDecl *>())
return TinyPtrVector<FileUnit *>(mod->getFiles());
// For a primary source file, we emit IR for both it and potentially its
// SynthesizedFileUnit.
auto *SF = Ctx.get<SourceFile *>();
TinyPtrVector<FileUnit *> files;
files.push_back(SF);
if (auto *synthesizedFile = SF->getSynthesizedFile())
files.push_back(synthesizedFile);
return files;
}
ModuleDecl *IRGenDescriptor::getParentModule() const {
if (auto *SF = Ctx.dyn_cast<SourceFile *>())
return SF->getParentModule();
return Ctx.get<ModuleDecl *>();
}
evaluator::DependencySource IRGenRequest::readDependencySource(
const evaluator::DependencyRecorder &e) const {
auto &desc = std::get<0>(getStorage());
return {
desc.Ctx.dyn_cast<SourceFile *>(),
evaluator::DependencyScope::Cascading
};
// We don't track dependencies in whole-module mode.
if (auto *mod = desc.Ctx.dyn_cast<ModuleDecl *>()) {
return {nullptr, e.getActiveSourceScope()};
}
auto *SF = desc.Ctx.get<SourceFile *>();
return {SF, evaluator::DependencyScope::Cascading};
}
// Define request evaluation functions for each of the IRGen requests.

View File

@@ -1706,6 +1706,22 @@ assertions
swiftsyntax
swiftsyntax-verify-generated-files
#===------------------------------------------------------------------------===#
# Test Swift Format
#===------------------------------------------------------------------------===#
[preset: buildbot_swiftformat_macos]
mixin-preset=mixin_swiftpm_package_macos_platform
release
assertions
swiftformat
[preset: buildbot_swiftformat_linux]
mixin-preset=mixin_swiftpm_package_linux_platform
release
assertions
swiftformat
#===------------------------------------------------------------------------===#
# Test Swift Stress Tester
#===------------------------------------------------------------------------===#

View File

@@ -868,6 +868,8 @@ class BuildScriptInvocation(object):
product_classes.append(products.SwiftSyntax)
if self.args.build_skstresstester:
product_classes.append(products.SKStressTester)
if self.args.build_swiftformat:
product_classes.append(products.SwiftFormat)
if self.args.build_swiftevolve:
product_classes.append(products.SwiftEvolve)
if self.args.build_indexstoredb:

View File

@@ -189,6 +189,7 @@ def _apply_default_arguments(args):
args.test_indexstoredb = False
args.test_sourcekitlsp = False
args.test_skstresstester = False
args.test_swiftformat = False
args.test_swiftevolve = False
args.test_toolchainbenchmarks = False
@@ -572,6 +573,9 @@ def create_argument_parser():
option(['--skstresstester'], store_true('build_skstresstester'),
help='build the SourceKit stress tester')
option(['--swiftformat'], store_true('build_swiftformat'),
help='build swift-format')
option(['--swiftevolve'], store_true('build_swiftevolve'),
help='build the swift-evolve tool')
@@ -1028,6 +1032,8 @@ def create_argument_parser():
help='skip testing PlaygroundSupport')
option('--skip-test-skstresstester', toggle_false('test_skstresstester'),
help='skip testing the SourceKit Stress tester')
option('--skip-test-swiftformat', toggle_false('test_swiftformat'),
help='skip testing swift-format')
option('--skip-test-swiftevolve', toggle_false('test_swiftevolve'),
help='skip testing SwiftEvolve')
option('--skip-test-toolchain-benchmarks',

View File

@@ -7,6 +7,9 @@
<FileRef
location = "group:../../../../swift-stress-tester/SourceKitStressTester">
</FileRef>
<FileRef
location = "group:../../../../swift-format">
</FileRef>
<FileRef
location = "group:../../../../swift-stress-tester/SwiftEvolve">
</FileRef>

View File

@@ -94,6 +94,7 @@ EXPECTED_DEFAULTS = {
'build_tensorflow_swift_apis': False,
'build_libparser_only': False,
'build_skstresstester': False,
'build_swiftformat': False,
'build_swiftevolve': False,
'build_indexstoredb': False,
'test_indexstoredb_sanitize_all': False,
@@ -237,6 +238,7 @@ EXPECTED_DEFAULTS = {
'test_indexstoredb': False,
'test_sourcekitlsp': False,
'test_skstresstester': False,
'test_swiftformat': False,
'test_swiftevolve': False,
'test_toolchainbenchmarks': False,
'tvos': False,
@@ -480,6 +482,7 @@ EXPECTED_OPTIONS = [
SetTrueOption('--swiftsyntax', dest='build_swiftsyntax'),
SetTrueOption('--build-libparser-only', dest='build_libparser_only'),
SetTrueOption('--skstresstester', dest='build_skstresstester'),
SetTrueOption('--swiftformat', dest='build_swiftformat'),
SetTrueOption('--swiftevolve', dest='build_swiftevolve'),
SetTrueOption('-B', dest='benchmark'),
SetTrueOption('-S', dest='skip_build'),
@@ -598,6 +601,7 @@ EXPECTED_OPTIONS = [
DisableOption('--skip-test-indexstore-db', dest='test_indexstoredb'),
DisableOption('--skip-test-sourcekit-lsp', dest='test_sourcekitlsp'),
DisableOption('--skip-test-skstresstester', dest='test_skstresstester'),
DisableOption('--skip-test-swiftformat', dest='test_swiftformat'),
DisableOption('--skip-test-swiftevolve', dest='test_swiftevolve'),
DisableOption('--skip-test-toolchain-benchmarks',
dest='test_toolchainbenchmarks'),

View File

@@ -28,6 +28,7 @@ from .sourcekitlsp import SourceKitLSP
from .swift import Swift
from .swiftdriver import SwiftDriver
from .swiftevolve import SwiftEvolve
from .swiftformat import SwiftFormat
from .swiftinspect import SwiftInspect
from .swiftpm import SwiftPM
from .swiftsyntax import SwiftSyntax
@@ -49,6 +50,7 @@ __all__ = [
'PlaygroundSupport',
'PythonKit',
'Swift',
'SwiftFormat',
'SwiftInspect',
'SwiftPM',
'SwiftDriver',

View File

@@ -0,0 +1,100 @@
# swift_build_support/products/swiftformat.py -----------------*- python -*-
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ----------------------------------------------------------------------------
import os
from build_swift.build_swift.constants import MULTIROOT_DATA_FILE_PATH
from . import cmark
from . import foundation
from . import libcxx
from . import libdispatch
from . import libicu
from . import llbuild
from . import llvm
from . import product
from . import swift
from . import swiftpm
from . import swiftsyntax
from . import xctest
from .. import shell
class SwiftFormat(product.Product):
@classmethod
def product_source_name(cls):
"""product_source_name() -> str
The name of the source code directory of this product.
"""
return "swift-format"
@classmethod
def is_build_script_impl_product(cls):
return False
@classmethod
def is_swiftpm_unified_build_product(cls):
return True
def run_build_script_helper(self, action, additional_params=[]):
script_path = os.path.join(
self.source_dir, 'build-script-helper.py')
configuration = 'release' if self.is_release() else 'debug'
helper_cmd = [
script_path,
action,
'--toolchain', self.install_toolchain_path(),
'--configuration', configuration,
'--build-path', self.build_dir,
'--multiroot-data-file', MULTIROOT_DATA_FILE_PATH,
# There might have been a Package.resolved created by other builds
# or by the package being opened using Xcode. Discard that and
# reset the dependencies to be local.
'--update'
]
if self.args.verbose_build:
helper_cmd.append('--verbose')
helper_cmd.extend(additional_params)
shell.call(helper_cmd)
def should_build(self, host_target):
return True
def build(self, host_target):
self.run_build_script_helper('build')
def should_test(self, host_target):
return self.args.test_swiftformat
def test(self, host_target):
self.run_build_script_helper('test')
def should_install(self, host_target):
return False
@classmethod
def get_dependencies(cls):
return [cmark.CMark,
llvm.LLVM,
libcxx.LibCXX,
libicu.LibICU,
swift.Swift,
libdispatch.LibDispatch,
foundation.Foundation,
xctest.XCTest,
llbuild.LLBuild,
swiftpm.SwiftPM,
swiftsyntax.SwiftSyntax]