Files
swift-mirror/utils/build-tooling-libs
Mike Ash 4341102c92 [Tools] Add a library to build specialized generic metadata out of process.
This library uses GenericMetadataBuilder with a ReaderWriter that can read data and resolve pointers from MachO files, and emit a JSON representation of a dylib containing the built metadata.

We use LLVM's binary file readers to parse the MachO files and resolve fixups so we can follow pointers. This code is somewhat MachO specific, but could be generalized to other formats that LLVM supports.

rdar://116592577
2024-01-24 20:45:50 -05:00

563 lines
21 KiB
Python
Executable File

#!/usr/bin/env python3
# utils/build-tooling-libs - Helper tool for building the SwiftStaticMirror
# library -*- python -*-
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
# This is a utility for building only the swift static-mirror
# library as fast as possible by only building the necessary dependency libraries
# and nothing extraneous. To achieve this it does a single unified CMake configuration for
# llvm/clang/swift and builds only the required CMake targets for the two libraries.
# This mechanism is fundamentally different from build-script, which builds llvm/clang
# in a separate build directory than swift.
#
# Even though this bypasses build-script, it does share some underlying helper
# utilities from the python infrastructure.
import copy
import multiprocessing
import os
import platform
import sys
from build_swift.build_swift import argparse
from build_swift.build_swift import defaults
from build_swift.build_swift.constants import SWIFT_BUILD_ROOT
from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT
from build_swift.build_swift.wrappers import xcrun
from swift_build_support.swift_build_support import shell
from swift_build_support.swift_build_support.toolchain import host_toolchain
isDarwin = platform.system() == "Darwin"
isSwiftContainedInLLVMProject = os.path.exists(os.path.join(SWIFT_SOURCE_ROOT, "llvm"))
staticMirrorLibTarget = "tools/swift/tools/libStaticMirror/install"
class Builder(object):
def __init__(
self, toolchain, args, host, arch, profile_data=None, native_build_dir=None
):
self.toolchain = toolchain
self.ninja_path = args.ninja_path
self.build_release = args.release
self.enable_assertions = not args.no_assertions
self.lto_type = args.lto_type
self.profile_data = profile_data
self.dry_run = args.dry_run
self.jobs = args.build_jobs
self.verbose = args.verbose
self.build_dir = args.build_dir
self.install_destdir = args.install_destdir
self.install_prefix = args.install_prefix
self.host = host
self.arch = arch
self.native_build_dir = native_build_dir
self.swiftc_path = args.swiftc_path
def call(self, command, env=None, without_sleeping=False):
if without_sleeping:
shell.call_without_sleeping(
command, env=env, dry_run=self.dry_run, echo=self.verbose
)
else:
shell.call(command, env=env, dry_run=self.dry_run, echo=self.verbose)
def configure(self, enable_debuginfo, instrumentation=None, profile_data=None):
environment = {}
cmake_args = [self.toolchain.cmake, "-G", "Ninja"]
cmake_args += ["-DCMAKE_MAKE_PROGRAM=" + self.ninja_path]
isEmbeddedHost = isDarwin and self.host != "macosx"
host_triple = None
host_sdk = None
llvm_c_flags = "-arch " + self.arch
if self.host == "macosx":
deployment_version = "10.12"
host_triple = "%s-apple-macosx%s" % (self.arch, deployment_version)
host_sdk = "OSX"
cmake_args += [
"-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
"-DSWIFT_DARWIN_DEPLOYMENT_VERSION_OSX=" + deployment_version,
]
environment["SDKROOT"] = "macosx"
elif self.host == "linux":
host_triple = "%s-unknown-linux" % (self.arch)
host_sdk = "LINUX"
elif self.host == "iphonesimulator":
deployment_version = "10.0"
host_triple = "%s-apple-ios%s-simulator" % (self.arch, deployment_version)
host_sdk = "IOS_SIMULATOR"
cmake_args += [
"-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
"-DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS=" + deployment_version,
]
llvm_c_flags += " -target " + host_triple
elif self.host == "iphoneos":
deployment_version = "10.0"
host_triple = "%s-apple-ios%s" % (self.arch, deployment_version)
host_sdk = "IOS"
cmake_args += [
"-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
"-DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS=" + deployment_version,
]
llvm_c_flags += " -target " + host_triple
elif self.host == "appletvsimulator":
deployment_version = "10.0"
host_triple = "%s-apple-tvos%s-simulator" % (self.arch, deployment_version)
host_sdk = "TVOS_SIMULATOR"
cmake_args += [
"-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
"-DSWIFT_DARWIN_DEPLOYMENT_VERSION_TVOS=" + deployment_version,
]
llvm_c_flags += " -target " + host_triple
elif self.host == "appletvos":
deployment_version = "10.0"
host_triple = "%s-apple-tvos%s" % (self.arch, deployment_version)
host_sdk = "TVOS"
cmake_args += [
"-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
"-DSWIFT_DARWIN_DEPLOYMENT_VERSION_TVOS=" + deployment_version,
]
llvm_c_flags += " -target " + host_triple
elif self.host == "watchsimulator":
deployment_version = "3.0"
host_triple = "%s-apple-watchos%s-simulator" % (
self.arch,
deployment_version,
)
host_sdk = "WATCHOS_SIMULATOR"
cmake_args += [
"-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
"-DSWIFT_DARWIN_DEPLOYMENT_VERSION_WATCHOS=" + deployment_version,
]
llvm_c_flags += " -target " + host_triple
elif self.host == "watchos":
deployment_version = "3.0"
host_triple = "%s-apple-watchos%s" % (self.arch, deployment_version)
host_sdk = "WATCHOS"
cmake_args += [
"-DCMAKE_OSX_DEPLOYMENT_TARGET=" + deployment_version,
"-DSWIFT_DARWIN_DEPLOYMENT_VERSION_WATCHOS=" + deployment_version,
]
llvm_c_flags += " -target " + host_triple
assert host_triple
assert host_sdk
cmake_args += [
"-DLLVM_HOST_TRIPLE:STRING=" + host_triple,
"-DLLVM_TARGET_ARCH=" + self.arch,
"-DSWIFT_HOST_VARIANT=" + self.host,
"-DSWIFT_HOST_VARIANT_SDK=" + host_sdk,
"-DSWIFT_HOST_VARIANT_ARCH=" + self.arch,
"-DCMAKE_Swift_COMPILER_TARGET=" + host_triple,
"-DCMAKE_Swift_COMPILER=" + self.swiftc_path,
"-DCMAKE_C_FLAGS=" + llvm_c_flags,
"-DCMAKE_CXX_FLAGS=" + llvm_c_flags,
]
if isEmbeddedHost:
cmake_args += [
"-DCMAKE_OSX_SYSROOT:PATH=" + xcrun.sdk_path(self.host),
# For embedded hosts CMake runs the checks and triggers crashes because
# the test binary was built for embedded host.
"-DHAVE_POSIX_REGEX:BOOL=TRUE",
"-DHAVE_STEADY_CLOCK:BOOL=TRUE",
]
if isDarwin:
if self.native_build_dir is not None:
cmake_args += [
"-DLLVM_TABLEGEN="
+ os.path.join(self.native_build_dir, "bin", "llvm-tblgen"),
"-DCLANG_TABLEGEN="
+ os.path.join(self.native_build_dir, "bin", "clang-tblgen"),
"-DLLVM_NATIVE_BUILD=" + self.native_build_dir,
"-DSWIFT_NATIVE_LLVM_TOOLS_PATH:STRING="
+ os.path.join(self.native_build_dir, "bin"),
"-DSWIFT_NATIVE_CLANG_TOOLS_PATH:STRING="
+ os.path.join(self.native_build_dir, "bin"),
"-DSWIFT_NATIVE_SWIFT_TOOLS_PATH:STRING="
+ os.path.join(self.native_build_dir, "bin"),
]
else:
dispatch_source_path = os.path.join(
SWIFT_SOURCE_ROOT, "swift-corelibs-libdispatch"
)
cmake_args += [
"-DSWIFT_PATH_TO_LIBDISPATCH_SOURCE:PATH=" + dispatch_source_path,
"-DLLVM_ENABLE_LLD=ON",
]
cmake_args += ["-DLLVM_TARGETS_TO_BUILD=X86"]
if self.build_release:
if enable_debuginfo:
cmake_args += ["-DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo"]
else:
cmake_args += ["-DCMAKE_BUILD_TYPE:STRING=Release"]
else:
cmake_args += ["-DCMAKE_BUILD_TYPE:STRING=Debug"]
if self.enable_assertions:
cmake_args += ["-DLLVM_ENABLE_ASSERTIONS:BOOL=ON"]
if instrumentation:
cmake_args += ["-DLLVM_BUILD_INSTRUMENTED=" + instrumentation]
if profile_data:
cmake_args += ["-DLLVM_PROFDATA_FILE=" + profile_data]
if self.lto_type and not instrumentation:
cmake_args += ["-DLLVM_ENABLE_LTO=" + self.lto_type.upper()]
if self.install_prefix:
cmake_args += ["-DCMAKE_INSTALL_PREFIX:PATH=" + self.install_prefix]
cmake_args += [
"-DLLVM_ENABLE_PROJECTS=clang",
"-DLLVM_EXTERNAL_PROJECTS=swift",
'-DLLVM_EXTERNAL_SWIFT_SOURCE_DIR=' +
os.path.join(SWIFT_SOURCE_ROOT, 'swift'),
]
cmake_args += ["-DSWIFT_BUILD_PERF_TESTSUITE=NO", "-DSWIFT_INCLUDE_DOCS=NO"]
cmake_args += [
"-DSWIFT_BUILD_REMOTE_MIRROR=FALSE",
"-DSWIFT_BUILD_EXTERNAL_GENERIC_METADATA_BUILDER=FALSE",
"-DSWIFT_BUILD_DYNAMIC_STDLIB=FALSE",
"-DSWIFT_BUILD_STATIC_STDLIB=FALSE",
"-DSWIFT_BUILD_DYNAMIC_SDK_OVERLAY=FALSE",
"-DSWIFT_BUILD_STATIC_SDK_OVERLAY=FALSE",
"-DSWIFT_BUILD_STDLIB_EXTRA_TOOLCHAIN_CONTENT=FALSE",
]
cmake_args += [
"-DLLVM_ENABLE_LIBXML2=FALSE",
"-DLLVM_ENABLE_LIBEDIT=FALSE",
"-DLLVM_ENABLE_TERMINFO=FALSE",
"-DLLVM_ENABLE_ZLIB=FALSE",
]
# We are not using cmark but initialize the CMARK variables to something so
# that configure can succeed.
cmake_args += [
"-DCMARK_MAIN_INCLUDE_DIR=" + os.path.join(SWIFT_SOURCE_ROOT, "cmark"),
"-DCMARK_BUILD_INCLUDE_DIR=" + os.path.join(self.build_dir, "cmark"),
]
cmake_args += [
"-DLLVM_INCLUDE_TESTS=FALSE",
"-DCLANG_INCLUDE_TESTS=FALSE",
"-DSWIFT_INCLUDE_TESTS=FALSE",
]
cmake_args += [
"-DBOOTSTRAPPING_MODE=HOSTTOOLS",
"-DSWIFT_PATH_TO_STRING_PROCESSING_SOURCE=" +
os.path.join(SWIFT_SOURCE_ROOT,
"swift-experimental-string-processing"),
"-DSWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE=" +
os.path.join(SWIFT_SOURCE_ROOT, "swift-syntax"),
]
llvm_src_path = os.path.join(SWIFT_SOURCE_ROOT, "llvm") \
if isSwiftContainedInLLVMProject \
else os.path.join(SWIFT_SOURCE_ROOT, "llvm-project", "llvm")
cmake_args += [llvm_src_path]
self.call(cmake_args, env=environment)
def build_targets(self, build_dir, targets, env=None):
invocation = [self.toolchain.cmake, "--build", build_dir]
invocation += ["--", "-j%d" % self.jobs]
# libStaticMirror requires a DESTDIR for the build action
# Ensure that if one was not provided, we still have a temporary
# location for it.
if env is None or (env is not None and env["DESTDIR"] is not None):
if self.install_destdir:
env = {"DESTDIR": self.install_destdir}
else:
env = {"DESTDIR": os.path.join(build_dir, "install")}
if self.verbose:
invocation += ["-v"]
invocation += targets
self.call(invocation, env=env, without_sleeping=True)
def install(self):
print("--- Installing ---", file=sys.stderr)
env = None
if self.install_destdir:
env = {"DESTDIR": self.install_destdir}
self.build_targets(
self.build_dir, [staticMirrorLibTarget], env=env
)
def run(self):
shell.makedirs(self.build_dir, dry_run=self.dry_run)
with shell.pushd(self.build_dir, dry_run=self.dry_run):
self.configure(enable_debuginfo=True, profile_data=self.profile_data)
self.build_targets(self.build_dir, [staticMirrorLibTarget])
if self.install_destdir:
self.install()
def extract_symbols(install_destdir, install_prefix, install_symroot, jobs):
if not isDarwin:
return
extract_script = os.path.join(
SWIFT_SOURCE_ROOT, "swift", "utils", "parser-lib", "darwin-extract-symbols"
)
print("--- Extracting symbols ---", file=sys.stderr)
env = {
"INSTALL_DIR": install_destdir,
"INSTALL_PREFIX": install_prefix,
"INSTALL_SYMROOT": install_symroot,
"BUILD_JOBS": str(jobs),
}
shell.call([extract_script], env=env)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description="""
Builds Swift Static Mirror library.
Example invocations:
* Building for host (macOS, linux):
$ utils/build-tooling-libs --release --no-assertions --build-dir /tmp/tooling-libs-build
* Building for iOS
$ utils/build-tooling-libs --release --no-assertions --build-dir \
/tmp/tooling-libs-build-iossim --host iphonesimulator --architectures x86_64
$ utils/build-tooling-libs --release --no-assertions --build-dir \
/tmp/tooling-libs-build-ios --host iphoneos --architectures arm64
""",
)
optbuilder = parser.to_builder()
option = optbuilder.add_option
store = optbuilder.actions.store
store_true = optbuilder.actions.store_true
store_int = optbuilder.actions.store_int
store_path = optbuilder.actions.store_path
toolchain = host_toolchain(xcrun_toolchain="default")
default_host = "macosx" if isDarwin else "linux"
default_architectures = platform.machine()
default_jobs = multiprocessing.cpu_count()
default_build_dir = os.path.join(SWIFT_BUILD_ROOT, "tooling-libs")
default_install_prefix = (
defaults.DARWIN_INSTALL_PREFIX if isDarwin else defaults.UNIX_INSTALL_PREFIX
)
default_ninja = toolchain.ninja
default_swiftc = toolchain.swiftc
option("--release", store_true, help="build in release mode")
option(
"--lto",
store("lto_type"),
choices=["thin", "full"],
const="full",
metavar="LTO_TYPE",
help="use lto optimization."
"Options: thin, full. If no optional arg is provided, full is "
"chosen by default",
)
option("--no-assertions", store_true, help="disable assertions")
option(
["-v", "--verbose"],
store_true,
help="print the commands executed during the build",
)
option(
"--dry-run",
store_true,
help="print the commands to execute but not actually execute them",
)
option(
["-j", "--jobs"],
store_int("build_jobs"),
default=default_jobs,
help="the number of parallel build jobs to use (default = %s)" % default_jobs,
)
option(
"--build-dir",
store_path,
default=default_build_dir,
help="the path where the build products will be placed. (default = %s)"
% default_build_dir,
)
option(
"--host",
store,
choices=[
"macosx",
"linux",
"iphonesimulator",
"iphoneos",
"appletvsimulator",
"appletvos",
"watchsimulator",
"watchos",
],
default=default_host,
help="host platform to build for (default = %s)" % default_host,
)
option(
"--architectures",
store,
default=default_architectures,
help="space-separated list of architectures to build for. (default = %s)"
% default_architectures,
)
option("--no-install", store_true, help="disable install step")
option(
"--install-symroot", store_path, help="the path to install debug symbols into"
)
option(
"--install-destdir",
store_path,
help="the path to use as the filesystem root for the installation (default = "
"'$(build_dir)/install')",
)
option(
"--install-prefix",
store,
default=default_install_prefix,
help="the install prefix to use (default = %s)" % default_install_prefix,
)
option(
"--ninja-path",
store_path,
default=default_ninja,
help="the path to ninja (default = %s)" % default_ninja,
)
option(
"--swiftc-path",
store_path,
default=default_swiftc,
help="""
Override the swiftc compiler used to build SwiftCompilerSources.
This is needed to build the most recent state of main if it uses C++ interop
features that aren't in the host's toolchain yet. (default = %s)
""" % default_swiftc
)
parser = optbuilder.build()
args = parser.parse_args()
if not args.install_destdir and not args.no_install:
args.install_destdir = os.path.join(args.build_dir, "install")
if not isSwiftContainedInLLVMProject:
swift_src_path = os.path.join(SWIFT_SOURCE_ROOT, "swift")
swift_src_in_llvm_project_path = \
os.path.join(SWIFT_SOURCE_ROOT, "llvm-project", "swift")
# Need to symlink 'swift' into 'llvm-project' since we will be doing
# a unified configure with 'swift' as an external project.
if not os.path.exists(swift_src_in_llvm_project_path):
print("Symlinking '%s' to '%s'" %
(swift_src_path, swift_src_in_llvm_project_path), file=sys.stderr)
shell.symlink(swift_src_path, swift_src_in_llvm_project_path,
dry_run=args.dry_run, echo=args.verbose)
architectures = args.architectures.split(" ")
architectures = [arch for arch in architectures if arch != ""]
if platform.machine() in architectures:
# Make sure the machine architecture is at the front.
architectures.remove(platform.machine())
architectures = [platform.machine()] + architectures
if isDarwin:
objroot = args.build_dir
dstroot = args.install_destdir
symroot = args.install_symroot
prefix = args.install_prefix
native_build_dir = None
profile_data = None
dst_dirs = []
if args.host == "macosx" and architectures[0] == platform.machine():
# Build for the native machine.
arch = architectures.pop(0)
tmpargs = copy.copy(args)
tmpargs.build_dir = os.path.join(objroot, arch, "obj")
if not args.no_install:
tmpargs.install_destdir = os.path.join(objroot, arch, "dst")
tmpargs.install_prefix = "/"
native_build_dir = tmpargs.build_dir
dst_dirs.append(tmpargs.install_destdir)
builder = Builder(
toolchain, tmpargs, tmpargs.host, arch, profile_data=profile_data
)
builder.run()
else:
tmpargs = copy.copy(args)
# Build the tablegen binaries so we can use them for the cross-compile
# build.
native_build_dir = os.path.join(objroot, platform.machine() + "-tblgen")
tmpargs.lto_type = None
builder = Builder(toolchain, tmpargs, "macosx", platform.machine())
shell.makedirs(native_build_dir, dry_run=tmpargs.dry_run)
with shell.pushd(native_build_dir, dry_run=tmpargs.dry_run):
builder.configure(enable_debuginfo=False)
builder.build_targets(native_build_dir, ["llvm-tblgen", "clang-tblgen"])
for arch in architectures:
args.build_dir = os.path.join(objroot, arch, "obj")
if not args.no_install:
args.install_destdir = os.path.join(objroot, arch, "dst")
args.install_prefix = "/"
dst_dirs.append(args.install_destdir)
builder = Builder(
toolchain,
args,
args.host,
arch,
profile_data=profile_data,
native_build_dir=native_build_dir,
)
builder.run()
if not args.no_install:
lipo = os.path.join(SWIFT_SOURCE_ROOT, "swift", "utils", "recursive-lipo")
shell.call(
[lipo, "-v", "--destination", os.path.join(dstroot, "./" + prefix)]
+ dst_dirs
)
if args.install_symroot:
extract_symbols(dstroot, prefix, symroot, args.build_jobs)
return 0
assert (
args.architectures == platform.machine()
), "building for non-machine architecture is not supported for non-darwin host"
builder = Builder(toolchain, args, args.host, args.architectures)
builder.run()
return 0
if __name__ == "__main__":
try:
sys.exit(main())
except KeyboardInterrupt:
sys.exit(1)