#!/usr/bin/env python3 # utils/build-tooling-libs - Helper tool for building the SwiftStaticMirror # library -*- python -*- # # This source file is part of the Swift.org open source project # # Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors # Licensed under Apache License v2.0 with Runtime Library Exception # # See https://swift.org/LICENSE.txt for license information # See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors # This is a utility for building only the swift static-mirror # library as fast as possible by only building the necessary dependency libraries # and nothing extraneous. To achieve this it does a single unified CMake configuration for # llvm/clang/swift and builds only the required CMake targets for the two libraries. # This mechanism is fundamentally different from build-script, which builds llvm/clang # in a separate build directory than swift. # # Even though this bypasses build-script, it does share some underlying helper # utilities from the python infrastructure. import copy import multiprocessing import os import platform import sys from build_swift.build_swift import argparse from build_swift.build_swift import defaults from build_swift.build_swift.constants import SWIFT_BUILD_ROOT from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT from build_swift.build_swift.wrappers import xcrun from swift_build_support.swift_build_support import shell from swift_build_support.swift_build_support.toolchain import host_toolchain isDarwin = platform.system() == "Darwin" isSwiftContainedInLLVMProject = os.path.exists(os.path.join(SWIFT_SOURCE_ROOT, "llvm")) staticMirrorLibTarget = "tools/swift/tools/libStaticMirror/install" class Builder(object): def __init__( self, toolchain, args, host, arch, profile_data=None, native_build_dir=None ): self.toolchain = toolchain self.ninja_path = args.ninja_path self.build_release = args.release self.enable_assertions = not args.no_assertions self.lto_type = args.lto_type self.profile_data = profile_data self.dry_run = args.dry_run self.jobs = args.build_jobs self.verbose = args.verbose self.build_dir = args.build_dir self.install_destdir = args.install_destdir self.install_prefix = args.install_prefix self.host = host self.arch = arch self.native_build_dir = native_build_dir self.swiftc_path = args.swiftc_path def call(self, command, env=None, without_sleeping=False): if without_sleeping: shell.call_without_sleeping( command, env=env, dry_run=self.dry_run, echo=self.verbose ) else: shell.call(command, env=env, dry_run=self.dry_run, echo=self.verbose) def configure(self, enable_debuginfo, instrumentation=None, profile_data=None): environment = {} cmake_args = [self.toolchain.cmake, "-G", "Ninja"] cmake_args += ["-DCMAKE_MAKE_PROGRAM=" + self.ninja_path] isEmbeddedHost = isDarwin and self.host != "macosx" host_triple = None host_sdk = None llvm_c_flags = "-arch " + self.arch if self.host == "macosx": deployment_version = "10.12" host_triple = "%s-apple-macosx%s" % (self.arch, deployment_version) host_sdk = "OSX" cmake_args += [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_OSX=" + deployment_version, ] environment["SDKROOT"] = "macosx" elif self.host == "linux": host_triple = "%s-unknown-linux" % (self.arch) host_sdk = "LINUX" elif self.host == "iphonesimulator": deployment_version = "10.0" host_triple = "%s-apple-ios%s-simulator" % (self.arch, deployment_version) host_sdk = "IOS_SIMULATOR" cmake_args += [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS=" + deployment_version, ] llvm_c_flags += " -target " + host_triple elif self.host == "iphoneos": deployment_version = "10.0" host_triple = "%s-apple-ios%s" % (self.arch, deployment_version) host_sdk = "IOS" cmake_args += [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS=" + deployment_version, ] llvm_c_flags += " -target " + host_triple elif self.host == "appletvsimulator": deployment_version = "10.0" host_triple = "%s-apple-tvos%s-simulator" % (self.arch, deployment_version) host_sdk = "TVOS_SIMULATOR" cmake_args += [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_TVOS=" + deployment_version, ] llvm_c_flags += " -target " + host_triple elif self.host == "appletvos": deployment_version = "10.0" host_triple = "%s-apple-tvos%s" % (self.arch, deployment_version) host_sdk = "TVOS" cmake_args += [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_TVOS=" + deployment_version, ] llvm_c_flags += " -target " + host_triple elif self.host == "watchsimulator": deployment_version = "3.0" host_triple = "%s-apple-watchos%s-simulator" % ( self.arch, deployment_version, ) host_sdk = "WATCHOS_SIMULATOR" cmake_args += [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_WATCHOS=" + deployment_version, ] llvm_c_flags += " -target " + host_triple elif self.host == "watchos": deployment_version = "3.0" host_triple = "%s-apple-watchos%s" % (self.arch, deployment_version) host_sdk = "WATCHOS" cmake_args += [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_WATCHOS=" + deployment_version, ] llvm_c_flags += " -target " + host_triple elif self.host == "xrsimulator": deployment_version = "1.0" host_triple = "%s-apple-xros%s-simulator" % ( self.arch, deployment_version, ) host_sdk = "XROS_SIMULATOR" cmake_args += [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_XROS=" + deployment_version, ] llvm_c_flags += " -target " + host_triple elif self.host == "xros": deployment_version = "1.0" host_triple = "%s-apple-xros%s" % (self.arch, deployment_version) host_sdk = "XROS" cmake_args += [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version, "-DSWIFT_DARWIN_DEPLOYMENT_VERSION_XROS=" + deployment_version, ] llvm_c_flags += " -target " + host_triple assert host_triple assert host_sdk cmake_args += [ "-DLLVM_HOST_TRIPLE:STRING=" + host_triple, "-DLLVM_TARGET_ARCH=" + self.arch, "-DSWIFT_HOST_VARIANT=" + self.host, "-DSWIFT_HOST_VARIANT_SDK=" + host_sdk, "-DSWIFT_HOST_VARIANT_ARCH=" + self.arch, "-DCMAKE_Swift_COMPILER_TARGET=" + host_triple, "-DCMAKE_Swift_COMPILER=" + self.swiftc_path, "-DCMAKE_C_FLAGS=" + llvm_c_flags, "-DCMAKE_CXX_FLAGS=" + llvm_c_flags, ] if isEmbeddedHost: cmake_args += [ "-DCMAKE_OSX_SYSROOT:PATH=" + xcrun.sdk_path(self.host), # For embedded hosts CMake runs the checks and triggers crashes because # the test binary was built for embedded host. "-DHAVE_POSIX_REGEX:BOOL=TRUE", "-DHAVE_STEADY_CLOCK:BOOL=TRUE", ] if isDarwin: if self.native_build_dir is not None: cmake_args += [ "-DLLVM_TABLEGEN=" + os.path.join(self.native_build_dir, "bin", "llvm-tblgen"), "-DCLANG_TABLEGEN=" + os.path.join(self.native_build_dir, "bin", "clang-tblgen"), "-DLLVM_NATIVE_BUILD=" + self.native_build_dir, "-DSWIFT_NATIVE_LLVM_TOOLS_PATH:STRING=" + os.path.join(self.native_build_dir, "bin"), "-DSWIFT_NATIVE_CLANG_TOOLS_PATH:STRING=" + os.path.join(self.native_build_dir, "bin"), "-DSWIFT_NATIVE_SWIFT_TOOLS_PATH:STRING=" + os.path.join(self.native_build_dir, "bin"), ] else: dispatch_source_path = os.path.join( SWIFT_SOURCE_ROOT, "swift-corelibs-libdispatch" ) cmake_args += [ "-DSWIFT_PATH_TO_LIBDISPATCH_SOURCE:PATH=" + dispatch_source_path, "-DLLVM_ENABLE_LLD=ON", ] cmake_args += ["-DLLVM_TARGETS_TO_BUILD=X86"] if self.build_release: if enable_debuginfo: cmake_args += ["-DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo"] else: cmake_args += ["-DCMAKE_BUILD_TYPE:STRING=Release"] else: cmake_args += ["-DCMAKE_BUILD_TYPE:STRING=Debug"] if self.enable_assertions: cmake_args += ["-DLLVM_ENABLE_ASSERTIONS:BOOL=ON"] if instrumentation: cmake_args += ["-DLLVM_BUILD_INSTRUMENTED=" + instrumentation] if profile_data: cmake_args += ["-DLLVM_PROFDATA_FILE=" + profile_data] if self.lto_type and not instrumentation: cmake_args += ["-DLLVM_ENABLE_LTO=" + self.lto_type.upper()] if self.install_prefix: cmake_args += ["-DCMAKE_INSTALL_PREFIX:PATH=" + self.install_prefix] cmake_args += [ "-DLLVM_ENABLE_PROJECTS=clang", "-DLLVM_EXTERNAL_PROJECTS=swift", '-DLLVM_EXTERNAL_SWIFT_SOURCE_DIR=' + os.path.join(SWIFT_SOURCE_ROOT, 'swift'), ] cmake_args += ["-DSWIFT_BUILD_PERF_TESTSUITE=NO", "-DSWIFT_INCLUDE_DOCS=NO"] cmake_args += [ "-DSWIFT_BUILD_REMOTE_MIRROR=FALSE", "-DSWIFT_BUILD_DYNAMIC_STDLIB=FALSE", "-DSWIFT_BUILD_STATIC_STDLIB=FALSE", "-DSWIFT_BUILD_DYNAMIC_SDK_OVERLAY=FALSE", "-DSWIFT_BUILD_STATIC_SDK_OVERLAY=FALSE", "-DSWIFT_BUILD_STDLIB_EXTRA_TOOLCHAIN_CONTENT=FALSE", ] cmake_args += [ "-DLLVM_ENABLE_LIBXML2=FALSE", "-DLLVM_ENABLE_LIBEDIT=FALSE", "-DLLVM_ENABLE_TERMINFO=FALSE", "-DLLVM_ENABLE_ZLIB=FALSE", ] # We are not using cmark but initialize the CMARK variables to something so # that configure can succeed. cmake_args += [ "-DCMARK_MAIN_INCLUDE_DIR=" + os.path.join(SWIFT_SOURCE_ROOT, "cmark"), "-DCMARK_BUILD_INCLUDE_DIR=" + os.path.join(self.build_dir, "cmark"), ] cmake_args += [ "-DLLVM_INCLUDE_TESTS=FALSE", "-DCLANG_INCLUDE_TESTS=FALSE", "-DSWIFT_INCLUDE_TESTS=FALSE", ] cmake_args += [ "-DBOOTSTRAPPING_MODE=HOSTTOOLS", "-DSWIFT_PATH_TO_STRING_PROCESSING_SOURCE=" + os.path.join(SWIFT_SOURCE_ROOT, "swift-experimental-string-processing"), "-DSWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE=" + os.path.join(SWIFT_SOURCE_ROOT, "swift-syntax"), ] llvm_src_path = os.path.join(SWIFT_SOURCE_ROOT, "llvm") \ if isSwiftContainedInLLVMProject \ else os.path.join(SWIFT_SOURCE_ROOT, "llvm-project", "llvm") cmake_args += [llvm_src_path] self.call(cmake_args, env=environment) def build_targets(self, build_dir, targets, env=None): invocation = [self.toolchain.cmake, "--build", build_dir] invocation += ["--", "-j%d" % self.jobs] # libStaticMirror requires a DESTDIR for the build action # Ensure that if one was not provided, we still have a temporary # location for it. if env is None or (env is not None and env["DESTDIR"] is not None): if self.install_destdir: env = {"DESTDIR": self.install_destdir} else: env = {"DESTDIR": os.path.join(build_dir, "install")} if self.verbose: invocation += ["-v"] invocation += targets self.call(invocation, env=env, without_sleeping=True) def install(self): print("--- Installing ---", file=sys.stderr) env = None if self.install_destdir: env = {"DESTDIR": self.install_destdir} self.build_targets( self.build_dir, [staticMirrorLibTarget], env=env ) def run(self): shell.makedirs(self.build_dir, dry_run=self.dry_run) with shell.pushd(self.build_dir, dry_run=self.dry_run): self.configure(enable_debuginfo=True, profile_data=self.profile_data) self.build_targets(self.build_dir, [staticMirrorLibTarget]) if self.install_destdir: self.install() def extract_symbols(install_destdir, install_prefix, install_symroot, jobs): if not isDarwin: return extract_script = os.path.join( SWIFT_SOURCE_ROOT, "swift", "utils", "parser-lib", "darwin-extract-symbols" ) print("--- Extracting symbols ---", file=sys.stderr) env = { "INSTALL_DIR": install_destdir, "INSTALL_PREFIX": install_prefix, "INSTALL_SYMROOT": install_symroot, "BUILD_JOBS": str(jobs), } shell.call([extract_script], env=env) def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=""" Builds Swift Static Mirror library. Example invocations: * Building for host (macOS, linux): $ utils/build-tooling-libs --release --no-assertions --build-dir /tmp/tooling-libs-build * Building for iOS $ utils/build-tooling-libs --release --no-assertions --build-dir \ /tmp/tooling-libs-build-iossim --host iphonesimulator --architectures x86_64 $ utils/build-tooling-libs --release --no-assertions --build-dir \ /tmp/tooling-libs-build-ios --host iphoneos --architectures arm64 """, ) optbuilder = parser.to_builder() option = optbuilder.add_option store = optbuilder.actions.store store_true = optbuilder.actions.store_true store_int = optbuilder.actions.store_int store_path = optbuilder.actions.store_path toolchain = host_toolchain(xcrun_toolchain="default") default_host = "macosx" if isDarwin else "linux" default_architectures = platform.machine() default_jobs = multiprocessing.cpu_count() default_build_dir = os.path.join(SWIFT_BUILD_ROOT, "tooling-libs") default_install_prefix = ( defaults.DARWIN_INSTALL_PREFIX if isDarwin else defaults.UNIX_INSTALL_PREFIX ) default_ninja = toolchain.ninja default_swiftc = toolchain.swiftc option("--release", store_true, help="build in release mode") option( "--lto", store("lto_type"), choices=["thin", "full"], const="full", metavar="LTO_TYPE", help="use lto optimization." "Options: thin, full. If no optional arg is provided, full is " "chosen by default", ) option("--no-assertions", store_true, help="disable assertions") option( ["-v", "--verbose"], store_true, help="print the commands executed during the build", ) option( "--dry-run", store_true, help="print the commands to execute but not actually execute them", ) option( ["-j", "--jobs"], store_int("build_jobs"), default=default_jobs, help="the number of parallel build jobs to use (default = %s)" % default_jobs, ) option( "--build-dir", store_path, default=default_build_dir, help="the path where the build products will be placed. (default = %s)" % default_build_dir, ) option( "--host", store, choices=[ "macosx", "linux", "iphonesimulator", "iphoneos", "appletvsimulator", "appletvos", "watchsimulator", "watchos", ], default=default_host, help="host platform to build for (default = %s)" % default_host, ) option( "--architectures", store, default=default_architectures, help="space-separated list of architectures to build for. (default = %s)" % default_architectures, ) option("--no-install", store_true, help="disable install step") option( "--install-symroot", store_path, help="the path to install debug symbols into" ) option( "--install-destdir", store_path, help="the path to use as the filesystem root for the installation (default = " "'$(build_dir)/install')", ) option( "--install-prefix", store, default=default_install_prefix, help="the install prefix to use (default = %s)" % default_install_prefix, ) option( "--ninja-path", store_path, default=default_ninja, help="the path to ninja (default = %s)" % default_ninja, ) option( "--swiftc-path", store_path, default=default_swiftc, help=""" Override the swiftc compiler used to build SwiftCompilerSources. This is needed to build the most recent state of main if it uses C++ interop features that aren't in the host's toolchain yet. (default = %s) """ % default_swiftc ) parser = optbuilder.build() args = parser.parse_args() if not args.install_destdir and not args.no_install: args.install_destdir = os.path.join(args.build_dir, "install") if not isSwiftContainedInLLVMProject: swift_src_path = os.path.join(SWIFT_SOURCE_ROOT, "swift") swift_src_in_llvm_project_path = \ os.path.join(SWIFT_SOURCE_ROOT, "llvm-project", "swift") # Need to symlink 'swift' into 'llvm-project' since we will be doing # a unified configure with 'swift' as an external project. if not os.path.exists(swift_src_in_llvm_project_path): print("Symlinking '%s' to '%s'" % (swift_src_path, swift_src_in_llvm_project_path), file=sys.stderr) shell.symlink(swift_src_path, swift_src_in_llvm_project_path, dry_run=args.dry_run, echo=args.verbose) architectures = args.architectures.split(" ") architectures = [arch for arch in architectures if arch != ""] if platform.machine() in architectures: # Make sure the machine architecture is at the front. architectures.remove(platform.machine()) architectures = [platform.machine()] + architectures if isDarwin: objroot = args.build_dir dstroot = args.install_destdir symroot = args.install_symroot prefix = args.install_prefix native_build_dir = None profile_data = None dst_dirs = [] if args.host == "macosx" and architectures[0] == platform.machine(): # Build for the native machine. arch = architectures.pop(0) tmpargs = copy.copy(args) tmpargs.build_dir = os.path.join(objroot, arch, "obj") if not args.no_install: tmpargs.install_destdir = os.path.join(objroot, arch, "dst") tmpargs.install_prefix = "/" native_build_dir = tmpargs.build_dir dst_dirs.append(tmpargs.install_destdir) builder = Builder( toolchain, tmpargs, tmpargs.host, arch, profile_data=profile_data ) builder.run() else: tmpargs = copy.copy(args) # Build the tablegen binaries so we can use them for the cross-compile # build. native_build_dir = os.path.join(objroot, platform.machine() + "-tblgen") tmpargs.lto_type = None builder = Builder(toolchain, tmpargs, "macosx", platform.machine()) shell.makedirs(native_build_dir, dry_run=tmpargs.dry_run) with shell.pushd(native_build_dir, dry_run=tmpargs.dry_run): builder.configure(enable_debuginfo=False) builder.build_targets(native_build_dir, ["llvm-tblgen", "clang-tblgen"]) for arch in architectures: args.build_dir = os.path.join(objroot, arch, "obj") if not args.no_install: args.install_destdir = os.path.join(objroot, arch, "dst") args.install_prefix = "/" dst_dirs.append(args.install_destdir) builder = Builder( toolchain, args, args.host, arch, profile_data=profile_data, native_build_dir=native_build_dir, ) builder.run() if not args.no_install: lipo = os.path.join(SWIFT_SOURCE_ROOT, "swift", "utils", "recursive-lipo") shell.call( [lipo, "-v", "--destination", os.path.join(dstroot, "./" + prefix)] + dst_dirs ) if args.install_symroot: extract_symbols(dstroot, prefix, symroot, args.build_jobs) return 0 assert ( args.architectures == platform.machine() ), "building for non-machine architecture is not supported for non-darwin host" builder = Builder(toolchain, args, args.host, args.architectures) builder.run() return 0 if __name__ == "__main__": try: sys.exit(main()) except KeyboardInterrupt: sys.exit(1)