mirror of
https://github.com/apple/swift.git
synced 2026-10-20 19:13:47 +02:00
Michael Gottesman
d6f320d7a2
A repository in an update-checkout config can override the URL that would otherwise be interpolated from the top-level ssh/https clone patterns by specifying a "url" in its "remote" object. That override applies to both protocols, so a downstream config that needs to point a repository at a different host than the rest of its repositories has to pick one protocol for it and force everyone onto that protocol, regardless of whether --clone-with-ssh was passed. In practice this means such repositories end up pinned to ssh URLs even for users cloning over https. Add two new optional keys to a repository's "remote" object, "ssh-url" and "https-url", so an override can be specified per protocol. The URL selection now goes: the override matching the requested protocol, then the protocol-agnostic "url", then the override for the other protocol, and finally the URL interpolated from the matching clone pattern and the remote "id". An override is preferred over interpolation even when it does not match the requested protocol, since it is the more specific piece of configuration; a repository that only specifies "https-url" is therefore still cloned over https under --clone-with-ssh. Existing configurations are unaffected: a "remote" with only a "url" continues to use it for both protocols. While here, factor the URL computation out of obtain_all_additional_swift_sources into remote_url_for_repo so it can be unit tested, and diagnose a "remote" that specifies none of the recognized keys instead of failing with a KeyError.
462 lines
15 KiB
Python
462 lines
15 KiB
Python
# ===--- test_clone.py ----------------------------------------------------===#
|
|
#
|
|
# This source file is part of the Swift.org open source project
|
|
#
|
|
# Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
|
|
# Licensed under Apache License v2.0 with Runtime Library Exception
|
|
#
|
|
# See https://swift.org/LICENSE.txt for license information
|
|
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
|
#
|
|
# ===----------------------------------------------------------------------===#
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import shutil
|
|
from unittest.mock import patch
|
|
import contextlib
|
|
from io import StringIO
|
|
|
|
from . import scheme_mock
|
|
from update_checkout.update_checkout import obtain_all_additional_swift_sources, main
|
|
|
|
|
|
class CloneTestCase(scheme_mock.SchemeMockTestCase):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super(CloneTestCase, self).__init__(*args, **kwargs)
|
|
|
|
def test_simple_clone(self):
|
|
self.call(
|
|
[
|
|
self.update_checkout_path,
|
|
"--config",
|
|
self.config_path,
|
|
"--source-root",
|
|
self.source_root,
|
|
"--clone",
|
|
]
|
|
)
|
|
|
|
for repo in self.repo_names:
|
|
repo_path = os.path.join(self.source_root, repo)
|
|
self.assertTrue(os.path.isdir(repo_path))
|
|
|
|
def test_clone_with_additional_scheme(self):
|
|
output = self.call(
|
|
[
|
|
self.update_checkout_path,
|
|
"--config",
|
|
self.config_path,
|
|
"--config",
|
|
self.additional_config_path,
|
|
"--source-root",
|
|
self.source_root,
|
|
"--clone",
|
|
"--scheme",
|
|
"extra",
|
|
"--verbose",
|
|
]
|
|
)
|
|
|
|
# Test that we're actually checking out the 'extra' scheme based on the output
|
|
self.assertIn("git checkout refs/heads/main", output)
|
|
|
|
def test_clone_missing_repos(self):
|
|
output = self.call(
|
|
[
|
|
self.update_checkout_path,
|
|
"--config",
|
|
self.config_path,
|
|
"--source-root",
|
|
self.source_root,
|
|
"--clone",
|
|
]
|
|
)
|
|
self.assertNotIn(
|
|
"You don't have all swift sources. Call this script with --clone to get them.",
|
|
output,
|
|
)
|
|
|
|
repo = self.repo_names[0]
|
|
repo_path = os.path.join(self.source_root, repo)
|
|
shutil.rmtree(repo_path)
|
|
output = self.call(
|
|
[
|
|
self.update_checkout_path,
|
|
"--config",
|
|
self.config_path,
|
|
"--source-root",
|
|
self.source_root,
|
|
]
|
|
)
|
|
self.assertIn(
|
|
"You don't have all swift sources. Call this script with --clone to get them.",
|
|
output,
|
|
)
|
|
|
|
def test_clone_with_git_config(self):
|
|
"""
|
|
Test that git is cloning with the following settings:
|
|
- core.symlinks = true
|
|
- core.autocrlf = false
|
|
"""
|
|
self.call(
|
|
[
|
|
self.update_checkout_path,
|
|
"--config",
|
|
self.config_path,
|
|
"--source-root",
|
|
self.source_root,
|
|
"--clone",
|
|
]
|
|
)
|
|
|
|
for repo in self.repo_names:
|
|
repo_path = os.path.join(self.source_root, repo)
|
|
output = subprocess.check_output(
|
|
["git", "-C", repo_path, "config", "--get", "core.symlinks"], text=True
|
|
)
|
|
self.assertIn("true", output)
|
|
output = subprocess.check_output(
|
|
["git", "-C", repo_path, "config", "--get", "core.autocrlf"], text=True
|
|
)
|
|
self.assertIn("false", output)
|
|
|
|
@patch("update_checkout.update_checkout.obtain_all_additional_swift_sources")
|
|
@patch("sys.exit", return_value=None)
|
|
def test_clone_with_incorrect_git_config(self, mock_exit, mock_obtain):
|
|
repo = self.repo_names[0]
|
|
|
|
def side_effect(*args, **kwargs):
|
|
result = obtain_all_additional_swift_sources(*args, **kwargs)
|
|
repo_path = os.path.join(self.source_root, repo)
|
|
subprocess.run(["git", "-C", repo_path, "config", "core.symlinks", "false"])
|
|
subprocess.run(["git", "-C", repo_path, "config", "core.autocrlf", "true"])
|
|
return result
|
|
|
|
mock_obtain.side_effect = side_effect
|
|
|
|
sys.argv = [
|
|
"update-checkout",
|
|
"--config",
|
|
self.config_path,
|
|
"--source-root",
|
|
self.source_root,
|
|
"--clone",
|
|
]
|
|
with contextlib.redirect_stdout(StringIO()) as stdout:
|
|
main()
|
|
output = stdout.getvalue()
|
|
self.assertIn(
|
|
f"[WARNING] '{repo}' was not cloned with 'core.symlinks=true'. This can cause build/tests failures.",
|
|
output,
|
|
)
|
|
self.assertIn(
|
|
f"[WARNING] '{repo}' was not cloned with 'core.autocrlf=false'. This can cause build/tests failures.",
|
|
output,
|
|
)
|
|
|
|
@patch("update_checkout.update_checkout.obtain_all_additional_swift_sources")
|
|
@patch("sys.exit", return_value=None)
|
|
def test_clone_with_missing_git_config_entry(self, mock_exit, mock_obtain):
|
|
repo = self.repo_names[0]
|
|
|
|
def side_effect(*args, **kwargs):
|
|
result = obtain_all_additional_swift_sources(*args, **kwargs)
|
|
repo_path = os.path.join(self.source_root, repo)
|
|
subprocess.run(
|
|
["git", "-C", repo_path, "config", "--unset", "core.symlinks"]
|
|
)
|
|
subprocess.run(
|
|
["git", "-C", repo_path, "config", "--unset", "core.autocrlf"]
|
|
)
|
|
return result
|
|
|
|
mock_obtain.side_effect = side_effect
|
|
|
|
sys.argv = [
|
|
"update-checkout",
|
|
"--config",
|
|
self.config_path,
|
|
"--source-root",
|
|
self.source_root,
|
|
"--clone",
|
|
]
|
|
with contextlib.redirect_stdout(StringIO()):
|
|
main()
|
|
|
|
@patch("update_checkout.update_checkout.obtain_all_additional_swift_sources")
|
|
@patch("sys.exit", return_value=None)
|
|
def test_clone_retry_on_exception(self, mock_exit, mock_obtain):
|
|
"""Test that an exception (e.g. concurrent.futures.TimeoutError) is
|
|
retried rather than propagating and crashing the process."""
|
|
import concurrent.futures
|
|
|
|
call_count = [0]
|
|
|
|
def side_effect(*args, **kwargs):
|
|
call_count[0] += 1
|
|
if call_count[0] <= 1:
|
|
raise concurrent.futures.TimeoutError()
|
|
return obtain_all_additional_swift_sources(*args, **kwargs)
|
|
|
|
mock_obtain.side_effect = side_effect
|
|
|
|
sys.argv = [
|
|
"update-checkout",
|
|
"--config",
|
|
self.config_path,
|
|
"--source-root",
|
|
self.source_root,
|
|
"--clone",
|
|
"--max-retries",
|
|
"1",
|
|
]
|
|
with contextlib.redirect_stdout(StringIO()):
|
|
main()
|
|
|
|
self.assertEqual(call_count[0], 2)
|
|
|
|
@patch("update_checkout.update_checkout.obtain_all_additional_swift_sources")
|
|
@patch("sys.exit", return_value=None)
|
|
def test_clone_with_retry(self, mock_exit, mock_obtain):
|
|
call_count = [0]
|
|
|
|
def side_effect(*args, **kwargs):
|
|
call_count[0] += 1
|
|
if call_count[0] <= 1:
|
|
return ([], [("fake-repo", "Simulated failure")])
|
|
else:
|
|
return obtain_all_additional_swift_sources(*args, **kwargs)
|
|
|
|
mock_obtain.side_effect = side_effect
|
|
|
|
# TODO: eventually call update_checkout using `main` rather than
|
|
# `subprocess.call` to be able to mock some methods.
|
|
sys.argv = [
|
|
"update-checkout",
|
|
"--config",
|
|
self.config_path,
|
|
"--source-root",
|
|
self.source_root,
|
|
"--clone",
|
|
"--max-retries",
|
|
"1",
|
|
]
|
|
with contextlib.redirect_stdout(StringIO()):
|
|
main()
|
|
|
|
self.assertEqual(call_count[0], 2)
|
|
|
|
|
|
class SchemeWithHashTestCase(scheme_mock.SchemeMockTestCase):
|
|
def __init__(self, *args, **kwargs):
|
|
super(SchemeWithHashTestCase, self).__init__(*args, **kwargs)
|
|
|
|
def setUp(self):
|
|
import json
|
|
|
|
super().setUp()
|
|
remote_repo_path = os.path.join(self.workspace, "remote", "repo1")
|
|
|
|
commits = (
|
|
self.call(["git", "rev-list", "main"], cwd=remote_repo_path, text=True)
|
|
.strip()
|
|
.split("\n")
|
|
)
|
|
|
|
self.commit_hash = commits[-1]
|
|
self.commit_scheme_name = "commit-hash-scheme"
|
|
commit_scheme = {
|
|
"aliases": [self.commit_scheme_name],
|
|
"repos": {
|
|
"repo1": self.commit_hash,
|
|
"repo2": "main",
|
|
},
|
|
}
|
|
self.add_branch_scheme(self.commit_scheme_name, commit_scheme)
|
|
|
|
with open(self.config_path, "w") as f:
|
|
json.dump(self.config, f)
|
|
|
|
def test_clone_with_skip_history(self):
|
|
self._test_clone_with_commit_hash(["--skip-history"])
|
|
|
|
def test_clone_with_reset_to_remote(self):
|
|
self._test_clone_with_commit_hash(["--reset-to-remote"])
|
|
|
|
def _test_clone_with_commit_hash(self, additional_flags):
|
|
"""
|
|
Test that cloning works with commit hashes.
|
|
"""
|
|
self.call(
|
|
[
|
|
self.update_checkout_path,
|
|
"--config",
|
|
self.config_path,
|
|
"--source-root",
|
|
self.source_root,
|
|
"--clone",
|
|
"--scheme",
|
|
self.commit_scheme_name,
|
|
"--verbose",
|
|
] + additional_flags
|
|
)
|
|
|
|
for repo in self.repo_names:
|
|
repo_path = os.path.join(self.source_root, repo)
|
|
self.assertTrue(os.path.isdir(repo_path))
|
|
|
|
current_commit = self.call(
|
|
["git", "rev-parse", "HEAD"], cwd=os.path.join(self.source_root, "repo1")
|
|
).strip()
|
|
self.assertEqual(current_commit, self.commit_hash)
|
|
|
|
|
|
class SchemeWithURLOverrideTestCase(scheme_mock.SchemeMockTestCase):
|
|
"""
|
|
Test the per-repository clone URL overrides. 'repo1' is given overrides
|
|
pointing at its own mock remote, plus unusable overrides that must not be
|
|
selected; 'repo2' is always skipped, since it can only be cloned through
|
|
the (unusable) ssh clone pattern of the mock config.
|
|
"""
|
|
|
|
UNUSABLE_URL = "file:///unusable/repo1"
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
# The https clone pattern of the mock config points at the mock
|
|
# remotes, so this is a URL that can actually be cloned from.
|
|
self.usable_url = self.config["https-clone-pattern"] % "repo1"
|
|
|
|
def _clone_repo1_with_overrides(self, overrides, additional_flags=[]):
|
|
import json
|
|
|
|
self.config["repos"]["repo1"]["remote"].update(overrides)
|
|
with open(self.config_path, "w") as f:
|
|
json.dump(self.config, f)
|
|
|
|
self.call(
|
|
self.base_args
|
|
+ [
|
|
"--clone",
|
|
"--skip-repository",
|
|
"repo2",
|
|
"--max-retries",
|
|
"0",
|
|
]
|
|
+ additional_flags
|
|
)
|
|
self.assertTrue(os.path.isdir(os.path.join(self.source_root, "repo1")))
|
|
|
|
def test_clone_with_url_override(self):
|
|
# A protocol-agnostic override is used for both protocols.
|
|
self._clone_repo1_with_overrides({"url": self.usable_url})
|
|
|
|
def test_clone_with_url_override_and_ssh(self):
|
|
self._clone_repo1_with_overrides({"url": self.usable_url}, ["--clone-with-ssh"])
|
|
|
|
def test_clone_with_https_url_override(self):
|
|
self._clone_repo1_with_overrides(
|
|
{"https-url": self.usable_url, "ssh-url": self.UNUSABLE_URL}
|
|
)
|
|
|
|
def test_clone_with_ssh_url_override(self):
|
|
self._clone_repo1_with_overrides(
|
|
{"ssh-url": self.usable_url, "https-url": self.UNUSABLE_URL},
|
|
["--clone-with-ssh"],
|
|
)
|
|
|
|
def test_clone_with_ssh_url_override_wins_over_url(self):
|
|
self._clone_repo1_with_overrides(
|
|
{"ssh-url": self.usable_url, "url": self.UNUSABLE_URL},
|
|
["--clone-with-ssh"],
|
|
)
|
|
|
|
def test_clone_with_https_url_override_wins_over_url(self):
|
|
self._clone_repo1_with_overrides(
|
|
{"https-url": self.usable_url, "url": self.UNUSABLE_URL}
|
|
)
|
|
|
|
|
|
class SchemeWithMissingRepoTestCase(scheme_mock.SchemeMockTestCase):
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
|
|
repos = self.repo_names
|
|
repos.pop()
|
|
|
|
self.scheme_name = "missing-repo"
|
|
scheme = {
|
|
"aliases": [self.scheme_name],
|
|
"repos": dict((repo, "main") for repo in repos),
|
|
}
|
|
self.add_branch_scheme(self.scheme_name, scheme)
|
|
|
|
# Test that we do not clone a repository that is not listed in the
|
|
# given branch-scheme.
|
|
def test_clone(self):
|
|
self.call(self.base_args + ["--scheme", self.scheme_name, "--clone"])
|
|
|
|
missing_repo_path = os.path.join(self.source_root, self.repo_names.pop())
|
|
self.assertFalse(os.path.isdir(missing_repo_path))
|
|
|
|
# Test that we do not update a repository that is not listed in the given
|
|
# branch-scheme---doing so will cause an irrelevant failure in an attempt
|
|
# to query the branch-scheme about the target branch for that repository.
|
|
def test_update(self):
|
|
# First, clone using the default branch-scheme, which has mappings for
|
|
# all repositories.
|
|
self.call(self.base_args + ["--clone"])
|
|
|
|
# Then, update using our custom scheme---a subset of the default one.
|
|
self.call(self.base_args + ["--scheme", self.scheme_name])
|
|
|
|
|
|
class SchemeWithAdditionalChanges(scheme_mock.SchemeMockTestCase):
|
|
def __init__(self, *args, **kwargs):
|
|
super(SchemeWithAdditionalChanges, self).__init__(*args, **kwargs)
|
|
|
|
def _add_commit_to_repo1(self):
|
|
local_repo_path = os.path.join(self.local_path, "repo1")
|
|
filename = "additional_content.txt"
|
|
filename_path = os.path.join(local_repo_path, filename)
|
|
with open(filename_path, "w") as f:
|
|
f.write("More stuff")
|
|
self.call(["git", "add", filename], cwd=local_repo_path)
|
|
self.call(["git", "commit", "-m", "Additional commit"], cwd=local_repo_path)
|
|
self.call(["git", "push", "origin", "main"], cwd=local_repo_path)
|
|
|
|
def test_call_update_checkout_twice_with_additional_commit(self):
|
|
"""
|
|
Test that calling update checkout a second time will pick up
|
|
additional commits done after the first time.
|
|
"""
|
|
update_checkout_command = [
|
|
self.update_checkout_path,
|
|
"--config",
|
|
self.config_path,
|
|
"--source-root",
|
|
self.source_root,
|
|
"--clone",
|
|
"--scheme",
|
|
"main",
|
|
"--verbose",
|
|
"--reset-to-remote",
|
|
]
|
|
|
|
self.call(update_checkout_command)
|
|
first_commit = self.call(
|
|
["git", "rev-parse", "HEAD"], cwd=os.path.join(self.source_root, "repo1")
|
|
).strip()
|
|
|
|
self._add_commit_to_repo1()
|
|
self.call(update_checkout_command)
|
|
second_commit = self.call(
|
|
["git", "rev-parse", "HEAD"], cwd=os.path.join(self.source_root, "repo1")
|
|
).strip()
|
|
|
|
self.assertNotEqual(first_commit, second_commit)
|