[utils] Utility to download unpublished toolchains for main branch

utils/download-unpublished-toolchains.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2024 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+
+"""
+Utility to download unpublished toolchains
+"""
+
+import argparse
+import os
+import sys
+import tarfile
+import urllib.request
+
+
+_CI_URL_ = "https://ci.swift.org/job/oss-swift-package-"
+
+
+def get_build_url(platform, arch):
+    if arch == "x86_64" or arch == "Universal":
+        arch = ""
+    else:
+        arch = f"-{arch}"
+    return f"{_CI_URL_}{platform}{arch}/lastSuccessfulBuild/consoleText"
+
+
+def get_latest_toolchain_url(build_url):
+    toolchain_url = None
+    with urllib.request.urlopen(build_url) as response:
+        body = response.read().decode('utf-8')
+    lines = body.split('\n')
+    toolchain = [x for x in lines if 'Toolchain:' in x]
+    if len(toolchain) > 0:
+        toolchain_url = toolchain[0].removeprefix('Toolchain: ')
+    return toolchain_url
+
+
+def download_toolchain(toolchain_url, output_dir):
+    file_name = toolchain_url.rsplit('/', 1)[-1]
+    output_path = os.path.join(output_dir, file_name)
+    urllib.request.urlretrieve(toolchain_url, output_path)
+    return output_path
+
+
+def untar_toolchain(output_dir, tar_path):
+    with tarfile.open(tar_path, "r") as tf:
+        tf.extractall(path=output_dir)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--output-dir",
+        required=True,
+        help="Output dir to download the toolchain",
+    )
+
+    parser.add_argument(
+        "--platform",
+        required=True,
+        choices=["amazon-linux-2",
+                 "debian-12",
+                 "fedora-39",
+                 "macos",
+                 "ubi-9",
+                 "ubuntu-20_04",
+                 "ubuntu-22_04",
+                 "ubuntu-24_04"],
+        help="Platform",
+    )
+
+    parser.add_argument(
+        "--arch",
+        required=True,
+        choices=["x86_64", "aarch64", "Universal"],
+        help="architectures",
+    )
+
+    parser.add_argument(
+        "--untar",
+        action=argparse.BooleanOptionalAction,
+        help="Untar the toolchain in the output directory",
+    )
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    if not os.path.isdir(args.output_dir):
+        print(f"Error: Directory does not exist at {args.output_dir}.")
+        return -1
+
+    build_url = get_build_url(args.platform, args.arch)
+    toolchain_url = get_latest_toolchain_url(build_url)
+    if not toolchain_url:
+        print(f"Error: Unable to find toolchain for {args.platform}.")
+        return -1
+    print(f"[Downloading] {toolchain_url}")
+    output_path = download_toolchain(toolchain_url, args.output_dir)
+    print(f"[Toolchain] {output_path}")
+
+    if args.untar:
+        untar_toolchain(args.output_dir, output_path)
+        print(f"[Extracted] {args.output_dir}/usr")
+
+
+if __name__ == "__main__":
+    sys.exit(main())