Files
swift-mirror/utils/build-script
Ben Langmuir 5619138a10 [build-script] Turn on --no-legacy-impl by default
Turns on the `--no-legacy-impl` option to build-script by default; the
old behaviour is temporarily still available as `--legacy-impl`.

This causes build-script to invoke build-script-impl for every
individual build/test/install/etc. action rather than a single global
invocation. For example, a single invocation might be for
`macosx-swift-install`. This will enable the python code in build-script
to drive the overall process and add additional steps in between actions
without the involvement of build-script-impl. It also provides a path to
refactoring the existing actions out of build-script-impl individually.

Discussed as part of https://forums.swift.org/t/rfc-building-swift-packages-in-build-script/18920

The --no-legacy-impl flag was originally disabled by default because of
concerns about the performance of null builds due to the increased
number of script invocations. There is a small optimization in this
commit to use `tr` when processing command-line options instead of
bash's builtin substitution, which eliminates most of the overhead.
After this change, a null build of llvm+swift changes from 1.6 s to
2.1 s on Linux, and from 5 s to 6 s on macOS.  Non-null builds and
builds that involve more build products than just llvm+swift (e.g.
corelibs) are basically unaffected since they are not correctly
incremental to begin with.

The changes to build-script-impl in this commit are to fix the behaviour
of --no-legacy-impl, which had bitrotted since it was introduced. These
changes are to make various parts of the script not rely on variables
defined in "earlier" parts of the script, which is good hygiene in
general.
2019-01-22 15:37:50 -08:00

1192 lines
48 KiB
Python
Executable File

#!/usr/bin/env python
# utils/build-script - The ultimate tool for building Swift -*- python -*-
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
from __future__ import print_function
import argparse
import os
import pipes
import platform
import sys
import time
from build_swift import defaults
from build_swift import driver_arguments
from build_swift import presets
from build_swift.migration import migrate_swift_sdks
from swift_build_support.swift_build_support import (
arguments,
debug,
diagnostics,
migration,
products,
shell,
tar,
targets,
workspace
)
from swift_build_support.swift_build_support.SwiftBuildSupport import (
HOME,
SWIFT_BUILD_ROOT,
SWIFT_REPO_NAME,
SWIFT_SOURCE_ROOT,
)
from swift_build_support.swift_build_support.cmake import CMake
from swift_build_support.swift_build_support.targets import \
StdlibDeploymentTarget
from swift_build_support.swift_build_support.toolchain import host_toolchain
build_script_impl = os.path.join(
SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils", "build-script-impl")
def exit_rejecting_arguments(message, parser=None):
print(message, file=sys.stderr)
if parser:
parser.print_usage(sys.stderr)
sys.exit(2) # 2 is the same as `argparse` error exit code.
def call_without_sleeping(command, env=None, dry_run=False, echo=False):
"""
Execute a command during which system sleep is disabled.
By default, this ignores the state of the `shell.dry_run` flag.
"""
# Disable system sleep, if possible.
if platform.system() == 'Darwin':
# Don't mutate the caller's copy of the arguments.
command = ["caffeinate"] + list(command)
shell.call(command, env=env, dry_run=dry_run, echo=echo)
class HostSpecificConfiguration(object):
"""Configuration information for an individual host."""
def __init__(self, host_target, invocation):
"""Initialize for the given `host_target`."""
# Compute the set of deployment targets to configure/build.
args = invocation.args
if host_target == args.host_target:
# This host is the user's desired product, so honor the requested
# set of targets to configure/build.
stdlib_targets_to_configure = args.stdlib_deployment_targets
if "all" in args.build_stdlib_deployment_targets:
stdlib_targets_to_build = set(stdlib_targets_to_configure)
else:
stdlib_targets_to_build = set(
args.build_stdlib_deployment_targets).intersection(
set(args.stdlib_deployment_targets))
else:
# Otherwise, this is a host we are building as part of
# cross-compiling, so we only need the target itself.
stdlib_targets_to_configure = [host_target]
stdlib_targets_to_build = set(stdlib_targets_to_configure)
# Compute the lists of **CMake** targets for each use case (configure
# vs. build vs. run) and the SDKs to configure with.
self.sdks_to_configure = set()
self.swift_stdlib_build_targets = []
self.swift_test_run_targets = []
self.swift_benchmark_build_targets = []
self.swift_benchmark_run_targets = []
for deployment_target_name in stdlib_targets_to_configure:
# Get the target object.
deployment_target = StdlibDeploymentTarget.get_target_for_name(
deployment_target_name)
if deployment_target is None:
diagnostics.fatal("unknown target: %r" % (
deployment_target_name,))
# Add the SDK to use.
deployment_platform = deployment_target.platform
self.sdks_to_configure.add(deployment_platform.sdk_name)
# If we aren't actually building this target (only configuring
# it), do nothing else.
if deployment_target_name not in stdlib_targets_to_build:
continue
# Compute which actions are desired.
build = (
deployment_platform not in invocation.platforms_to_skip_build)
test = (
deployment_platform not in invocation.platforms_to_skip_test)
test_host_only = None
dt_supports_benchmark = deployment_target.supports_benchmark
build_benchmarks = build and dt_supports_benchmark
build_external_benchmarks = all([build, dt_supports_benchmark,
args.build_external_benchmarks])
# FIXME: Note, `build-script-impl` computed a property here
# w.r.t. testing, but it was actually unused.
# For platforms which normally require a connected device to
# test, the default behavior is to run tests that only require
# the host (i.e., they do not attempt to execute).
if deployment_platform.is_darwin and \
deployment_platform.is_embedded and \
not deployment_platform.is_simulator:
if deployment_platform not in \
invocation.platforms_to_skip_test_host:
test_host_only = True
test = True
else:
test = False
name = deployment_target.name
for skip_test_arch in invocation.platforms_archs_to_skip_test:
if deployment_target.name == skip_test_arch.name:
test = False
if build:
# Validation, long, and stress tests require building the full
# standard library, whereas the other targets can build a
# slightly smaller subset which is faster to build.
if args.build_swift_stdlib_unittest_extra or \
args.validation_test or args.long_test or \
args.stress_test:
self.swift_stdlib_build_targets.append(
"swift-stdlib-" + name)
else:
self.swift_stdlib_build_targets.append(
"swift-test-stdlib-" + name)
if build_benchmarks:
self.swift_benchmark_build_targets.append(
"swift-benchmark-" + name)
if args.benchmark:
self.swift_benchmark_run_targets.append(
"check-swift-benchmark-" + name)
if build_external_benchmarks:
# Add support for the external benchmarks.
self.swift_benchmark_build_targets.append(
"swift-benchmark-{}-external".format(name))
if args.benchmark:
self.swift_benchmark_run_targets.append(
"check-swift-benchmark-{}-external".format(name))
if test:
if test_host_only:
suffix = "-non-executable"
else:
suffix = ""
subset_suffix = ""
if args.validation_test and args.long_test and \
args.stress_test:
subset_suffix = "-all"
elif args.validation_test:
subset_suffix = "-validation"
elif args.long_test:
subset_suffix = "-only_long"
elif args.stress_test:
subset_suffix = "-only_stress"
else:
subset_suffix = ""
self.swift_test_run_targets.append("check-swift{}{}-{}".format(
subset_suffix, suffix, name))
if args.test_optimized and not test_host_only:
self.swift_test_run_targets.append(
"check-swift{}-optimize-{}".format(
subset_suffix, name))
if args.test_optimize_for_size and not test_host_only:
self.swift_test_run_targets.append(
"check-swift{}-optimize_size-{}".format(
subset_suffix, name))
class BuildScriptInvocation(object):
"""Represent a single build script invocation."""
@staticmethod
def validate_arguments(toolchain, args):
if toolchain.cc is None or toolchain.cxx is None:
diagnostics.fatal(
"can't find clang (please install clang-3.5 or a "
"later version)")
if toolchain.cmake is None:
diagnostics.fatal("can't find CMake (please install CMake)")
if args.distcc:
if toolchain.distcc is None:
diagnostics.fatal(
"can't find distcc (please install distcc)")
if toolchain.distcc_pump is None:
diagnostics.fatal(
"can't find distcc-pump (please install distcc-pump)")
if args.host_target is None or args.stdlib_deployment_targets is None:
diagnostics.fatal("unknown operating system")
if args.symbols_package:
if not os.path.isabs(args.symbols_package):
print(
'--symbols-package must be an absolute path '
'(was \'{}\')'.format(args.symbols_package))
return 1
if not args.install_symroot:
diagnostics.fatal(
"--install-symroot is required when specifying "
"--symbols-package.")
if args.android:
if args.android_ndk is None or \
args.android_api_level is None or \
args.android_icu_uc is None or \
args.android_icu_uc_include is None or \
args.android_icu_i18n is None or \
args.android_icu_i18n_include is None or \
args.android_icu_data is None:
diagnostics.fatal(
"when building for Android, --android-ndk, "
"--android-api-level, --android-icu-uc, "
"--android-icu-uc-include, --android-icu-i18n, "
"--android-icu-i18n-include, and --android-icu-data "
"must be specified")
@staticmethod
def apply_default_arguments(toolchain, args):
# Infer if ninja is required
ninja_required = (
args.cmake_generator == 'Ninja' or args.build_foundation)
if ninja_required and toolchain.ninja is None:
args.build_ninja = True
# Set the default stdlib-deployment-targets, if none were provided.
if args.stdlib_deployment_targets is None:
stdlib_targets = \
StdlibDeploymentTarget.default_stdlib_deployment_targets()
args.stdlib_deployment_targets = [
target.name for target in stdlib_targets]
# SwiftPM and XCTest have a dependency on Foundation.
# On OS X, Foundation is built automatically using xcodebuild.
# On Linux, we must ensure that it is built manually.
if ((args.build_swiftpm or args.build_xctest) and
platform.system() != "Darwin"):
args.build_foundation = True
# Foundation has a dependency on libdispatch.
# On OS X, libdispatch is provided by the OS.
# On Linux, we must ensure that it is built manually.
if (args.build_foundation and
platform.system() != "Darwin"):
args.build_libdispatch = True
if args.build_subdir is None:
args.build_subdir = \
workspace.compute_build_subdir(args)
# Add optional stdlib-deployment-targets
if args.android:
if args.android_arch == "armv7":
args.stdlib_deployment_targets.append(
StdlibDeploymentTarget.Android.armv7.name)
elif args.android_arch == "aarch64":
args.stdlib_deployment_targets.append(
StdlibDeploymentTarget.Android.aarch64.name)
# Infer platform flags from manually-specified configure targets.
# This doesn't apply to Darwin platforms, as they are
# already configured. No building without the platform flag, though.
android_tgts = [tgt for tgt in args.stdlib_deployment_targets
if StdlibDeploymentTarget.Android.contains(tgt)]
if not args.android and len(android_tgts) > 0:
args.android = True
args.build_android = False
# ---
def __init__(self, toolchain, args):
self.toolchain = toolchain
self.args = args
self.workspace = workspace.Workspace(
source_root=SWIFT_SOURCE_ROOT,
build_root=os.path.join(SWIFT_BUILD_ROOT, args.build_subdir))
# Compute derived information from the arguments.
#
# FIXME: We should move the platform-derived arguments to be entirely
# data driven, so that we can eliminate this code duplication and just
# iterate over all supported platforms.
self.platforms_to_skip_build = set()
if not args.build_linux:
self.platforms_to_skip_build.add(StdlibDeploymentTarget.Linux)
if not args.build_freebsd:
self.platforms_to_skip_build.add(StdlibDeploymentTarget.FreeBSD)
if not args.build_cygwin:
self.platforms_to_skip_build.add(StdlibDeploymentTarget.Cygwin)
if not args.build_osx:
self.platforms_to_skip_build.add(StdlibDeploymentTarget.OSX)
if not args.build_ios_device:
self.platforms_to_skip_build.add(StdlibDeploymentTarget.iOS)
if not args.build_ios_simulator:
self.platforms_to_skip_build.add(
StdlibDeploymentTarget.iOSSimulator)
if not args.build_tvos_device:
self.platforms_to_skip_build.add(StdlibDeploymentTarget.AppleTV)
if not args.build_tvos_simulator:
self.platforms_to_skip_build.add(
StdlibDeploymentTarget.AppleTVSimulator)
if not args.build_watchos_device:
self.platforms_to_skip_build.add(StdlibDeploymentTarget.AppleWatch)
if not args.build_watchos_simulator:
self.platforms_to_skip_build.add(
StdlibDeploymentTarget.AppleWatchSimulator)
if not args.build_android:
self.platforms_to_skip_build.add(StdlibDeploymentTarget.Android)
self.platforms_to_skip_test = set()
self.platforms_archs_to_skip_test = set()
if not args.test_linux:
self.platforms_to_skip_test.add(StdlibDeploymentTarget.Linux)
if not args.test_freebsd:
self.platforms_to_skip_test.add(StdlibDeploymentTarget.FreeBSD)
if not args.test_cygwin:
self.platforms_to_skip_test.add(StdlibDeploymentTarget.Cygwin)
if not args.test_osx:
self.platforms_to_skip_test.add(StdlibDeploymentTarget.OSX)
if not args.test_ios_host:
self.platforms_to_skip_test.add(StdlibDeploymentTarget.iOS)
else:
exit_rejecting_arguments("error: iOS device tests are not " +
"supported in open-source Swift.")
if not args.test_ios_simulator:
self.platforms_to_skip_test.add(
StdlibDeploymentTarget.iOSSimulator)
if not args.test_ios_32bit_simulator:
self.platforms_archs_to_skip_test.add(
StdlibDeploymentTarget.iOSSimulator.i386)
if not args.test_tvos_host:
self.platforms_to_skip_test.add(StdlibDeploymentTarget.AppleTV)
else:
exit_rejecting_arguments("error: tvOS device tests are not " +
"supported in open-source Swift.")
if not args.test_tvos_simulator:
self.platforms_to_skip_test.add(
StdlibDeploymentTarget.AppleTVSimulator)
if not args.test_watchos_host:
self.platforms_to_skip_test.add(StdlibDeploymentTarget.AppleWatch)
else:
exit_rejecting_arguments("error: watchOS device tests are not " +
"supported in open-source Swift.")
if not args.test_watchos_simulator:
self.platforms_to_skip_test.add(
StdlibDeploymentTarget.AppleWatchSimulator)
if not args.test_android_host:
self.platforms_to_skip_test.add(StdlibDeploymentTarget.Android)
self.platforms_to_skip_test_host = set()
if not args.test_ios_host:
self.platforms_to_skip_test_host.add(StdlibDeploymentTarget.iOS)
if not args.test_tvos_host:
self.platforms_to_skip_test_host.add(
StdlibDeploymentTarget.AppleTV)
if not args.test_watchos_host:
self.platforms_to_skip_test_host.add(
StdlibDeploymentTarget.AppleWatch)
self.build_libparser_only = args.build_libparser_only
def initialize_runtime_environment(self):
"""Change the program environment for building."""
# Set an appropriate default umask.
os.umask(0o022)
# Unset environment variables that might affect how tools behave.
for v in [
'MAKEFLAGS',
'SDKROOT',
'MACOSX_DEPLOYMENT_TARGET',
'IPHONEOS_DEPLOYMENT_TARGET',
'TVOS_DEPLOYMENT_TARGET',
'WATCHOS_DEPLOYMENT_TARGET']:
os.environ.pop(v, None)
# Set NINJA_STATUS to format ninja output
os.environ['NINJA_STATUS'] = '[%f/%t][%p][%es] '
def build_ninja(self):
if not os.path.exists(self.workspace.source_dir("ninja")):
diagnostics.fatal(
"can't find source directory for ninja "
"(tried %s)" % (self.workspace.source_dir("ninja")))
ninja_build = products.Ninja(
args=self.args,
toolchain=self.toolchain,
source_dir=self.workspace.source_dir("ninja"),
build_dir=self.workspace.build_dir("build", "ninja"))
ninja_build.do_build()
self.toolchain.ninja = ninja_build.ninja_bin_path
def convert_to_impl_arguments(self):
"""convert_to_impl_arguments() -> (env, args)
Convert the invocation to an environment and list of arguments suitable
for invoking `build-script-impl`.
"""
# Create local shadows, for convenience.
args = self.args
toolchain = self.toolchain
cmake = CMake(args=args,
toolchain=self.toolchain)
impl_args = [
"--workspace", self.workspace.source_root,
"--build-dir", self.workspace.build_root,
"--install-prefix", args.install_prefix,
"--host-target", args.host_target,
"--stdlib-deployment-targets",
" ".join(args.stdlib_deployment_targets),
"--host-cc", toolchain.cc,
"--host-cxx", toolchain.cxx,
"--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain,
"--darwin-deployment-version-osx=%s" % (
args.darwin_deployment_version_osx),
"--darwin-deployment-version-ios=%s" % (
args.darwin_deployment_version_ios),
"--darwin-deployment-version-tvos=%s" % (
args.darwin_deployment_version_tvos),
"--darwin-deployment-version-watchos=%s" % (
args.darwin_deployment_version_watchos),
"--cmake", toolchain.cmake,
"--cmark-build-type", args.cmark_build_variant,
"--llvm-build-type", args.llvm_build_variant,
"--swift-build-type", args.swift_build_variant,
"--swift-stdlib-build-type", args.swift_stdlib_build_variant,
"--lldb-build-type", args.lldb_build_variant,
"--lldb-build-with-xcode", args.lldb_build_with_xcode,
"--foundation-build-type", args.foundation_build_variant,
"--libdispatch-build-type", args.libdispatch_build_variant,
"--libicu-build-type", args.libicu_build_variant,
"--xctest-build-type", args.build_variant,
"--swiftpm-build-type", args.build_variant,
"--swiftsyntax-build-type", args.build_variant,
"--skstresstester-build-type", args.build_variant,
"--swiftevolve-build-type", args.build_variant,
"--llbuild-build-type", args.build_variant,
"--swift-enable-assertions", str(args.swift_assertions).lower(),
"--swift-stdlib-enable-assertions", str(
args.swift_stdlib_assertions).lower(),
"--swift-analyze-code-coverage", str(
args.swift_analyze_code_coverage).lower(),
"--llbuild-enable-assertions", str(
args.llbuild_assertions).lower(),
"--lldb-assertions", str(
args.lldb_assertions).lower(),
"--cmake-generator", args.cmake_generator,
"--build-jobs", str(args.build_jobs),
"--common-cmake-options=%s" % ' '.join(
pipes.quote(opt) for opt in cmake.common_options()),
"--build-args=%s" % ' '.join(
pipes.quote(arg) for arg in cmake.build_args()),
]
# Compute any product specific cmake arguments.
for product_class in self.compute_product_classes():
product_name = product_class.product_name()
product_source_name = product_class.product_source_name()
source_dir = self.workspace.source_dir(product_source_name)
if not os.path.exists(source_dir):
diagnostics.fatal(
"can't find source directory for %s "
"(tried %s)" % (product_name, source_dir))
product = product_class(
args=args,
toolchain=self.toolchain,
source_dir=source_dir,
# FIXME: This is incorrect since it always assumes the host
# target I think?
build_dir=self.workspace.build_dir(
args.host_target, product_name))
cmake_opts = product.cmake_options
# FIXME: We should be using pipes.quote here but we run into issues
# with build-script-impl/cmake not being happy with all of the
# extra "'" in the strings. To fix this easily, we really need to
# just invoke cmake from build-script directly rather than futzing
# with build-script-impl. This makes even more sense since there
# really isn't a security issue here.
if len(cmake_opts) > 0:
impl_args += [
"--%s-cmake-options=%s" %
(product_name, ' '.join(cmake_opts))
]
if args.build_stdlib_deployment_targets:
impl_args += [
"--build-stdlib-deployment-targets", " ".join(
args.build_stdlib_deployment_targets)]
if args.cross_compile_hosts:
impl_args += [
"--cross-compile-hosts", " ".join(args.cross_compile_hosts)]
if args.test_paths:
impl_args += ["--test-paths", " ".join(args.test_paths)]
if toolchain.ninja:
impl_args += ["--ninja-bin=%s" % toolchain.ninja]
if args.distcc:
impl_args += [
"--distcc",
"--distcc-pump=%s" % toolchain.distcc_pump
]
# *NOTE* We use normal cmake to pass through tsan/ubsan options. We do
# NOT pass them to build-script-impl.
if args.enable_asan:
impl_args += ["--enable-asan"]
# If we are on linux, disable leak detection when running ASAN. We
# have a separate bot that checks for leaks.
if platform.system() == 'Linux':
os.environ['ASAN_OPTIONS'] = 'detect_leaks=0'
if args.enable_ubsan:
impl_args += ["--enable-ubsan"]
# If we have lsan, we need to export our suppression list. The actual
# passing in of the LSAN flag is done via the normal cmake method. We
# do not pass the flag to build-script-impl.
if args.enable_lsan:
supp_file = os.path.join(SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME,
"utils",
"lsan_leaks_suppression_list.txt")
os.environ['LSAN_OPTIONS'] = 'suppressions={}'.format(supp_file)
if args.verbose_build:
impl_args += ["--verbose-build"]
if args.install_symroot:
impl_args += [
"--install-symroot", os.path.abspath(args.install_symroot)
]
if args.skip_build:
impl_args += ["--skip-build-cmark",
"--skip-build-llvm",
"--skip-build-swift"]
if not args.build_benchmarks:
impl_args += ["--skip-build-benchmarks"]
# Currently we do not build external benchmarks by default.
if args.build_external_benchmarks:
impl_args += ["--skip-build-external-benchmarks=0"]
if not args.build_foundation:
impl_args += ["--skip-build-foundation"]
if not args.build_xctest:
impl_args += ["--skip-build-xctest"]
if not args.build_lldb:
impl_args += ["--skip-build-lldb"]
if not args.build_llbuild:
impl_args += ["--skip-build-llbuild"]
if not args.build_libdispatch:
impl_args += ["--skip-build-libdispatch"]
if not args.build_libicu:
impl_args += ["--skip-build-libicu"]
if not args.build_swiftpm:
impl_args += ["--skip-build-swiftpm"]
if not args.build_swiftsyntax:
impl_args += ["--skip-build-swiftsyntax"]
if not args.build_skstresstester:
impl_args += ["--skip-build-skstresstester"]
if not args.build_swiftevolve:
impl_args += ["--skip-build-swiftevolve"]
if not args.build_playgroundsupport:
impl_args += ["--skip-build-playgroundsupport"]
if args.build_swift_dynamic_stdlib:
impl_args += ["--build-swift-dynamic-stdlib"]
if args.build_swift_static_stdlib:
impl_args += ["--build-swift-static-stdlib"]
if args.build_swift_stdlib_unittest_extra:
impl_args += ["--build-swift-stdlib-unittest-extra"]
if args.build_swift_dynamic_sdk_overlay:
impl_args += ["--build-swift-dynamic-sdk-overlay"]
if args.build_swift_static_sdk_overlay:
impl_args += ["--build-swift-static-sdk-overlay"]
if not args.build_linux:
impl_args += ["--skip-build-linux"]
if not args.build_freebsd:
impl_args += ["--skip-build-freebsd"]
if not args.build_cygwin:
impl_args += ["--skip-build-cygwin"]
if not args.build_osx:
impl_args += ["--skip-build-osx"]
if not args.build_ios_device:
impl_args += ["--skip-build-ios-device"]
if not args.build_ios_simulator:
impl_args += ["--skip-build-ios-simulator"]
if not args.build_tvos_device:
impl_args += ["--skip-build-tvos-device"]
if not args.build_tvos_simulator:
impl_args += ["--skip-build-tvos-simulator"]
if not args.build_watchos_device:
impl_args += ["--skip-build-watchos-device"]
if not args.build_watchos_simulator:
impl_args += ["--skip-build-watchos-simulator"]
if not args.build_android:
impl_args += ["--skip-build-android"]
if not args.test and not args.long_test and not args.stress_test:
impl_args += ["--skip-test-swift"]
if not args.test:
impl_args += ["--skip-test-cmark",
"--skip-test-lldb",
"--skip-test-llbuild",
"--skip-test-swiftpm",
"--skip-test-swiftsyntax",
"--skip-test-skstresstester",
"--skip-test-swiftevolve",
"--skip-test-xctest",
"--skip-test-foundation",
"--skip-test-libdispatch",
"--skip-test-libicu",
"--skip-test-playgroundsupport"]
if not args.test_linux:
impl_args += ["--skip-test-linux"]
if not args.test_freebsd:
impl_args += ["--skip-test-freebsd"]
if not args.test_cygwin:
impl_args += ["--skip-test-cygwin"]
if not args.test_osx:
impl_args += ["--skip-test-osx"]
if not args.test_ios_host:
impl_args += ["--skip-test-ios-host"]
if not args.test_ios_simulator:
impl_args += ["--skip-test-ios-simulator"]
if not args.test_ios_32bit_simulator:
impl_args += ["--skip-test-ios-32bit-simulator"]
if not args.test_tvos_host:
impl_args += ["--skip-test-tvos-host"]
if not args.test_tvos_simulator:
impl_args += ["--skip-test-tvos-simulator"]
if not args.test_watchos_host:
impl_args += ["--skip-test-watchos-host"]
if not args.test_watchos_simulator:
impl_args += ["--skip-test-watchos-simulator"]
if not args.test_android_host:
impl_args += ["--skip-test-android-host"]
if args.build_runtime_with_host_compiler:
impl_args += ["--build-runtime-with-host-compiler"]
if args.validation_test:
impl_args += ["--validation-test"]
if args.long_test:
impl_args += ["--long-test"]
if args.stress_test:
impl_args += ["--stress-test"]
if not args.benchmark:
impl_args += ["--skip-test-benchmarks"]
if not args.test_optimized:
impl_args += ["--skip-test-optimized"]
if not args.test_optimize_for_size:
impl_args += ["--skip-test-optimize-for-size"]
if args.build_libparser_only:
impl_args += ["--build-libparser-only"]
if args.android:
impl_args += [
"--android-arch", args.android_arch,
"--android-ndk", args.android_ndk,
"--android-api-level", args.android_api_level,
"--android-ndk-gcc-version", args.android_ndk_gcc_version,
"--android-icu-uc", args.android_icu_uc,
"--android-icu-uc-include", args.android_icu_uc_include,
"--android-icu-i18n", args.android_icu_i18n,
"--android-icu-i18n-include", args.android_icu_i18n_include,
"--android-icu-data", args.android_icu_data,
]
if args.android_deploy_device_path:
impl_args += [
"--android-deploy-device-path",
args.android_deploy_device_path,
]
if platform.system() == 'Darwin':
impl_args += [
"--toolchain-prefix",
targets.darwin_toolchain_prefix(
args.install_prefix),
"--host-lipo", toolchain.lipo,
]
if toolchain.libtool is not None:
impl_args += [
"--host-libtool", toolchain.libtool,
]
# If we have extra_swift_args, combine all of them together and then
# add them as one command.
if args.extra_swift_args:
impl_args += [
"--extra-swift-args",
";".join(args.extra_swift_args)]
# If we have extra_cmake_options, combine all of them together and then
# add them as one command.
if args.extra_cmake_options:
impl_args += [
"--extra-cmake-options=%s" % ' '.join(
pipes.quote(opt) for opt in args.extra_cmake_options)
]
if args.lto_type is not None:
impl_args += [
"--llvm-enable-lto=%s" % args.lto_type,
"--swift-tools-enable-lto=%s" % args.lto_type
]
if args.llvm_max_parallel_lto_link_jobs is not None:
impl_args += [
"--llvm-num-parallel-lto-link-jobs=%s" %
min(args.llvm_max_parallel_lto_link_jobs, args.build_jobs)
]
if args.swift_tools_max_parallel_lto_link_jobs is not None:
impl_args += [
"--swift-tools-num-parallel-lto-link-jobs=%s" %
min(args.swift_tools_max_parallel_lto_link_jobs,
args.build_jobs)
]
impl_args += args.build_script_impl_args
if args.dry_run:
impl_args += ["--dry-run"]
if args.clang_profile_instr_use:
impl_args += [
"--clang-profile-instr-use=%s" %
os.path.abspath(args.clang_profile_instr_use)
]
if args.lit_args:
impl_args += ["--llvm-lit-args=%s" % args.lit_args]
if args.coverage_db:
impl_args += [
"--coverage-db=%s" %
os.path.abspath(args.coverage_db)
]
# Compute the set of host-specific variables, which we pass through to
# the build script via environment variables.
host_specific_variables = self.compute_host_specific_variables()
impl_env = {}
for (host_target, options) in host_specific_variables.items():
for (name, value) in options.items():
# We mangle into an environment variable we can easily evaluate
# from the `build-script-impl`.
impl_env["HOST_VARIABLE_{}__{}".format(
host_target.replace("-", "_"), name)] = value
return (impl_env, impl_args)
def compute_host_specific_variables(self):
"""compute_host_specific_variables(args) -> dict
Compute the host-specific options, organized as a dictionary keyed by
host of options.
"""
args = self.args
options = {}
for host_target in [args.host_target] + args.cross_compile_hosts:
# Compute the host specific configuration.
config = HostSpecificConfiguration(host_target, self)
# Convert into `build-script-impl` style variables.
options[host_target] = {
"SWIFT_SDKS": " ".join(sorted(
config.sdks_to_configure)),
"SWIFT_STDLIB_TARGETS": " ".join(
config.swift_stdlib_build_targets),
"SWIFT_BENCHMARK_TARGETS": " ".join(
config.swift_benchmark_build_targets),
"SWIFT_RUN_BENCHMARK_TARGETS": " ".join(
config.swift_benchmark_run_targets),
"SWIFT_TEST_TARGETS": " ".join(
config.swift_test_run_targets),
}
return options
def compute_product_classes(self):
"""compute_product_classes() -> list
Compute the list of all Product classes used in this build. This list
is constructed in dependency order.
"""
# FIXME: This is a weird division (returning a list of class objects),
# but it matches the existing structure of the `build-script-impl`.
product_classes = []
product_classes.append(products.CMark)
product_classes.append(products.LLVM)
if self.args.build_libicu:
product_classes.append(products.LibICU)
product_classes.append(products.Swift)
if self.args.build_lldb:
product_classes.append(products.LLDB)
if self.args.build_libdispatch:
product_classes.append(products.LibDispatch)
if self.args.build_foundation:
product_classes.append(products.Foundation)
if self.args.build_xctest:
product_classes.append(products.XCTest)
if self.args.build_llbuild:
product_classes.append(products.LLBuild)
if self.args.build_swiftpm:
product_classes.append(products.SwiftPM)
if self.args.build_swiftsyntax:
product_classes.append(products.SwiftSyntax)
if self.args.build_skstresstester:
product_classes.append(products.SKStressTester)
return product_classes
def execute(self):
"""Execute the invocation with the configured arguments."""
# Convert to a build-script-impl invocation.
(impl_env, impl_args) = self.convert_to_impl_arguments()
# If using the legacy implementation, delegate all behavior to
# `build-script-impl`.
if self.args.legacy_impl:
# Execute the underlying build script implementation.
call_without_sleeping([build_script_impl] + impl_args,
env=impl_env, echo=True)
return
# Otherwise, we compute and execute the individual actions ourselves.
def execute_one_impl_action(host=None, product_class=None, name=None):
if host is None:
assert (product_class is None and
name == "merged-hosts-lipo"), "invalid action"
action_name = name
elif product_class is None:
assert name in ("package", "extractsymbols"), "invalid action"
action_name = "{}-{}".format(host.name, name)
else:
assert name is not None, "invalid action"
action_name = "{}-{}-{}".format(
host.name, product_class.product_name(), name)
call_without_sleeping(
[build_script_impl] + impl_args + [
"--only-execute", action_name],
env=impl_env, echo=self.args.verbose_build)
# Compute the list of hosts to operate on.
all_host_names = [
self.args.host_target] + self.args.cross_compile_hosts
all_hosts = [StdlibDeploymentTarget.get_target_for_name(name)
for name in all_host_names]
# Compute the list of product classes to operate on.
#
# FIXME: This should really be per-host, but the current structure
# matches that of `build-script-impl`.
product_classes = self.compute_product_classes()
# Execute each "pass".
# Build...
for host_target in all_hosts:
# FIXME: We should only compute these once.
config = HostSpecificConfiguration(host_target.name, self)
print("Building the standard library for: {}".format(
" ".join(config.swift_stdlib_build_targets)))
if config.swift_test_run_targets and (
self.args.test or self.args.long_test):
print("Running Swift tests for: {}".format(
" ".join(config.swift_test_run_targets)))
if config.swift_benchmark_run_targets and self.args.benchmark:
print("Running Swift benchmarks for: {}".format(
" ".join(config.swift_benchmark_run_targets)))
for product_class in product_classes:
execute_one_impl_action(host_target, product_class, "build")
# Test...
for host_target in all_hosts:
for product_class in product_classes:
execute_one_impl_action(host_target, product_class, "test")
# Install...
for host_target in all_hosts:
for product_class in product_classes:
execute_one_impl_action(host_target, product_class, "install")
# Extract symbols...
for host_target in all_hosts:
execute_one_impl_action(host_target, name="extractsymbols")
# Package...
for host_target in all_hosts:
execute_one_impl_action(host_target, name="package")
# Lipo...
execute_one_impl_action(name="merged-hosts-lipo")
# Provide a short delay so accidentally invoked clean builds can be canceled.
def clean_delay():
sys.stdout.write('Starting clean build in ')
for i in range(3, 0, -1):
sys.stdout.write('\b%d' % i)
sys.stdout.flush()
time.sleep(1)
print('\b\b\b\bnow.')
# Main entry point for the preset mode.
def main_preset():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""Builds Swift using a preset.""")
parser.add_argument(
"-n", "--dry-run",
help="print the commands that would be executed, but do not execute "
"them",
action="store_true",
default=False)
parser.add_argument(
"--preset-file",
help="load presets from the specified file",
metavar="PATH",
action="append",
dest="preset_file_names",
default=[])
parser.add_argument(
"--preset",
help="use the specified option preset",
metavar="NAME")
parser.add_argument(
"--show-presets",
help="list all presets and exit",
action=arguments.action.optional_bool)
parser.add_argument(
"--distcc",
help="use distcc",
action=arguments.action.optional_bool)
parser.add_argument(
"-j", "--jobs",
help="the number of parallel build jobs to use",
type=int,
dest="build_jobs")
parser.add_argument(
"preset_substitutions_raw",
help="'name=value' pairs that are substituted in the preset",
nargs="*",
metavar="SUBSTITUTION")
parser.add_argument(
"--expand-build-script-invocation",
help="Print the expanded build-script invocation generated "
"by the preset, but do not run the preset",
action=arguments.action.optional_bool)
parser.add_argument(
"--swiftsyntax-install-prefix",
help="specify the directory to where SwiftSyntax should be installed")
args = parser.parse_args()
if len(args.preset_file_names) == 0:
args.preset_file_names = [
os.path.join(
SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils",
"build-presets.ini")
]
user_presets_file = os.path.join(HOME, '.swift-build-presets')
if os.path.isfile(user_presets_file):
args.preset_file_names.append(user_presets_file)
preset_parser = presets.PresetParser()
try:
preset_parser.read(args.preset_file_names)
except presets.Error as e:
diagnostics.fatal(e.message)
if args.show_presets:
for name in sorted(preset_parser.preset_names(),
key=lambda name: name.lower()):
print(name)
return 0
if not args.preset:
diagnostics.fatal("missing --preset option")
args.preset_substitutions = {}
for arg in args.preset_substitutions_raw:
name, value = arg.split("=", 1)
args.preset_substitutions[name] = value
try:
preset = preset_parser.get_preset(args.preset,
vars=args.preset_substitutions)
except presets.Error as e:
diagnostics.fatal(e.message)
preset_args = migrate_swift_sdks(preset.format_args())
build_script_args = [sys.argv[0]]
if args.dry_run:
build_script_args += ["--dry-run"]
build_script_args += preset_args
if args.distcc:
build_script_args += ["--distcc"]
if args.build_jobs:
build_script_args += ["--jobs", str(args.build_jobs)]
if args.swiftsyntax_install_prefix:
build_script_args += ["--install-swiftsyntax",
"--install-destdir",
args.swiftsyntax_install_prefix]
diagnostics.note('using preset "{}", which expands to \n\n{}\n'.format(
args.preset, shell.quote_command(build_script_args)))
if args.expand_build_script_invocation:
return 0
call_without_sleeping(build_script_args)
return 0
# Main entry point for the normal mode.
def main_normal():
parser = driver_arguments.create_argument_parser()
args = migration.parse_args(parser, sys.argv[1:])
if args.build_script_impl_args:
# If we received any impl args, check if `build-script-impl` would
# accept them or not before any further processing.
try:
migration.check_impl_args(build_script_impl,
args.build_script_impl_args)
except ValueError as e:
exit_rejecting_arguments(e, parser)
if '--check-args-only' in args.build_script_impl_args:
return 0
shell.dry_run = args.dry_run
# Prepare and validate toolchain
if args.darwin_xcrun_toolchain is None:
xcrun_toolchain = os.environ.get('TOOLCHAINS',
defaults.DARWIN_XCRUN_TOOLCHAIN)
diagnostics.note('Using toolchain {}'.format(xcrun_toolchain))
args.darwin_xcrun_toolchain = xcrun_toolchain
toolchain = host_toolchain(xcrun_toolchain=args.darwin_xcrun_toolchain)
os.environ['TOOLCHAINS'] = args.darwin_xcrun_toolchain
if args.host_cc is not None:
toolchain.cc = args.host_cc
if args.host_cxx is not None:
toolchain.cxx = args.host_cxx
if args.host_lipo is not None:
toolchain.lipo = args.host_lipo
if args.host_libtool is not None:
toolchain.libtool = args.host_libtool
if args.cmake is not None:
toolchain.cmake = args.cmake
# Preprocess the arguments to apply defaults.
BuildScriptInvocation.apply_default_arguments(toolchain, args)
# Validate the arguments.
BuildScriptInvocation.validate_arguments(toolchain, args)
# Create the build script invocation.
invocation = BuildScriptInvocation(toolchain, args)
# Sanitize the runtime environment.
invocation.initialize_runtime_environment()
# Show SDKs, if requested.
if args.show_sdks:
debug.print_xcodebuild_versions()
# Clean build directory if requested.
if args.clean:
clean_delay()
shell.rmtree(invocation.workspace.build_root)
# Create build directory.
shell.makedirs(invocation.workspace.build_root)
# Build ninja if required, which will update the toolchain.
if args.build_ninja:
invocation.build_ninja()
# Execute the underlying build script implementation.
invocation.execute()
if args.symbols_package:
print('--- Creating symbols package ---')
print('-- Package file: {} --'.format(args.symbols_package))
if platform.system() == 'Darwin':
prefix = targets.darwin_toolchain_prefix(args.install_prefix)
else:
prefix = args.install_prefix
# As a security measure, `tar` normally strips leading '/' from paths
# it is archiving. To stay safe, we change working directories, then
# run `tar` without the leading '/' (we remove it ourselves to keep
# `tar` from emitting a warning).
with shell.pushd(args.install_symroot):
tar.tar(source=prefix.lstrip('/'),
destination=args.symbols_package)
return 0
def main():
if not SWIFT_SOURCE_ROOT:
diagnostics.fatal(
"could not infer source root directory " +
"(forgot to set $SWIFT_SOURCE_ROOT environment variable?)")
if not os.path.isdir(SWIFT_SOURCE_ROOT):
diagnostics.fatal(
"source root directory \'" + SWIFT_SOURCE_ROOT +
"\' does not exist " +
"(forgot to set $SWIFT_SOURCE_ROOT environment variable?)")
# Determine if we are invoked in the preset mode and dispatch accordingly.
if any([(opt.startswith("--preset") or opt == "--show-presets")
for opt in sys.argv[1:]]):
return main_preset()
else:
return main_normal()
if __name__ == "__main__":
try:
sys.exit(main())
except KeyboardInterrupt:
sys.exit(1)