mirror of
https://github.com/apple/swift.git
synced 2026-06-27 12:25:55 +02:00
Erik Eckstein
45a2ae48ce
We don't measure Ounchecked anymore. On the other hand we want to benchmark the Osize build.
479 lines
18 KiB
Python
Executable File
479 lines
18 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
# ===--- Benchmark_Driver ------------------------------------------------===//
|
|
#
|
|
# This source file is part of the Swift.org open source project
|
|
#
|
|
# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
|
|
# Licensed under Apache License v2.0 with Runtime Library Exception
|
|
#
|
|
# See https://swift.org/LICENSE.txt for license information
|
|
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
|
#
|
|
# ===---------------------------------------------------------------------===//
|
|
|
|
import argparse
|
|
import datetime
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib
|
|
import urllib2
|
|
|
|
DRIVER_DIR = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
|
|
def parse_results(res, optset):
|
|
# Parse lines like this
|
|
# #,TEST,SAMPLES,MIN(μs),MAX(μs),MEAN(μs),SD(μs),MEDIAN(μs),PEAK_MEMORY(B)
|
|
score_re = re.compile(r"(\d+),[ \t]*(\w+)," +
|
|
",".join([r"[ \t]*([\d.]+)"] * 7))
|
|
# The Totals line would be parsed like this.
|
|
total_re = re.compile(r"()(Totals)," +
|
|
",".join([r"[ \t]*([\d.]+)"] * 7))
|
|
key_group = 2
|
|
val_group = 4
|
|
mem_group = 9
|
|
|
|
tests = []
|
|
for line in res.split():
|
|
m = score_re.match(line)
|
|
if not m:
|
|
m = total_re.match(line)
|
|
if not m:
|
|
continue
|
|
testresult = int(m.group(val_group))
|
|
testname = m.group(key_group)
|
|
test = {}
|
|
test['Data'] = [testresult]
|
|
test['Info'] = {}
|
|
test['Name'] = "nts.swift/" + optset + "." + testname + ".exec"
|
|
tests.append(test)
|
|
if testname != 'Totals':
|
|
mem_testresult = int(m.group(mem_group))
|
|
mem_test = {}
|
|
mem_test['Data'] = [mem_testresult]
|
|
mem_test['Info'] = {}
|
|
mem_test['Name'] = "nts.swift/mem_maxrss." + \
|
|
optset + "." + testname + ".mem"
|
|
tests.append(mem_test)
|
|
return tests
|
|
|
|
|
|
def submit_to_lnt(data, url):
|
|
print("\nSubmitting results to LNT server...")
|
|
json_report = {'input_data': json.dumps(data), 'commit': '1'}
|
|
data = urllib.urlencode(json_report)
|
|
response_str = urllib2.urlopen(urllib2.Request(url, data))
|
|
response = json.loads(response_str.read())
|
|
if 'success' in response:
|
|
print("Server response:\tSuccess")
|
|
else:
|
|
print("Server response:\tError")
|
|
print("Error:\t", response['error'])
|
|
sys.exit(1)
|
|
|
|
|
|
def instrument_test(driver_path, test, num_samples):
|
|
"""Run a test and instrument its peak memory use"""
|
|
test_outputs = []
|
|
for _ in range(num_samples):
|
|
test_output_raw = subprocess.check_output(
|
|
['time', '-lp', driver_path, test],
|
|
stderr=subprocess.STDOUT
|
|
)
|
|
peak_memory = re.match('\s*(\d+)\s*maximum resident set size',
|
|
test_output_raw.split('\n')[-15]).group(1)
|
|
test_outputs.append(test_output_raw.split()[1].split(',') +
|
|
[peak_memory])
|
|
|
|
# Average sample results
|
|
num_samples_index = 2
|
|
min_index = 3
|
|
max_index = 4
|
|
avg_start_index = 5
|
|
|
|
# TODO: Correctly take stdev
|
|
avg_test_output = test_outputs[0]
|
|
avg_test_output[avg_start_index:] = map(int,
|
|
avg_test_output[avg_start_index:])
|
|
for test_output in test_outputs[1:]:
|
|
for i in range(avg_start_index, len(test_output)):
|
|
avg_test_output[i] += int(test_output[i])
|
|
for i in range(avg_start_index, len(avg_test_output)):
|
|
avg_test_output[i] = int(round(avg_test_output[i] /
|
|
float(len(test_outputs))))
|
|
avg_test_output[num_samples_index] = num_samples
|
|
avg_test_output[min_index] = min(
|
|
test_outputs, key=lambda x: int(x[min_index]))[min_index]
|
|
avg_test_output[max_index] = max(
|
|
test_outputs, key=lambda x: int(x[max_index]))[max_index]
|
|
avg_test_output = map(str, avg_test_output)
|
|
|
|
return avg_test_output
|
|
|
|
|
|
BENCHMARK_OUTPUT_RE = re.compile('([^,]+),')
|
|
|
|
|
|
def get_tests(driver_path, args):
|
|
"""Return a list of available performance tests"""
|
|
driver = ([driver_path, '--list'])
|
|
if args.benchmarks or args.filters:
|
|
driver.append('--run-all')
|
|
tests = []
|
|
for l in subprocess.check_output(driver).split("\n")[1:]:
|
|
m = BENCHMARK_OUTPUT_RE.match(l)
|
|
if m is None:
|
|
continue
|
|
tests.append(m.group(1))
|
|
if args.filters:
|
|
regexes = [re.compile(pattern) for pattern in args.filters]
|
|
return sorted(list(set([name for pattern in regexes
|
|
for name in tests if pattern.match(name)])))
|
|
if not args.benchmarks:
|
|
return tests
|
|
tests.extend(map(str, range(1, len(tests) + 1))) # ordinal numbers
|
|
return sorted(list(set(tests).intersection(set(args.benchmarks))))
|
|
|
|
|
|
def get_current_git_branch(git_repo_path):
|
|
"""Return the selected branch for the repo `git_repo_path`"""
|
|
return subprocess.check_output(
|
|
['git', '-C', git_repo_path, 'rev-parse',
|
|
'--abbrev-ref', 'HEAD'], stderr=subprocess.STDOUT).strip()
|
|
|
|
|
|
def get_git_head_ID(git_repo_path):
|
|
"""Return the short identifier for the HEAD commit of the repo
|
|
`git_repo_path`"""
|
|
return subprocess.check_output(
|
|
['git', '-C', git_repo_path, 'rev-parse',
|
|
'--short', 'HEAD'], stderr=subprocess.STDOUT).strip()
|
|
|
|
|
|
def log_results(log_directory, driver, formatted_output, swift_repo=None):
|
|
"""Log `formatted_output` to a branch specific directory in
|
|
`log_directory`
|
|
"""
|
|
try:
|
|
branch = get_current_git_branch(swift_repo)
|
|
except (OSError, subprocess.CalledProcessError):
|
|
branch = None
|
|
try:
|
|
head_ID = '-' + get_git_head_ID(swift_repo)
|
|
except (OSError, subprocess.CalledProcessError):
|
|
head_ID = ''
|
|
timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime())
|
|
if branch:
|
|
output_directory = os.path.join(log_directory, branch)
|
|
else:
|
|
output_directory = log_directory
|
|
driver_name = os.path.basename(driver)
|
|
try:
|
|
os.makedirs(output_directory)
|
|
except OSError:
|
|
pass
|
|
log_file = os.path.join(output_directory,
|
|
driver_name + '-' + timestamp + head_ID + '.log')
|
|
print('Logging results to: %s' % log_file)
|
|
with open(log_file, 'w') as f:
|
|
f.write(formatted_output)
|
|
|
|
|
|
def run_benchmarks(driver, benchmarks=[], num_samples=10, verbose=False,
|
|
log_directory=None, swift_repo=None):
|
|
"""Run perf tests individually and return results in a format that's
|
|
compatible with `parse_results`. If `benchmarks` is not empty,
|
|
only run tests included in it.
|
|
"""
|
|
(total_tests, total_min, total_max, total_mean) = (0, 0, 0, 0)
|
|
output = []
|
|
headings = ['#', 'TEST', 'SAMPLES', 'MIN(μs)', 'MAX(μs)', 'MEAN(μs)',
|
|
'SD(μs)', 'MEDIAN(μs)', 'MAX_RSS(B)']
|
|
line_format = '{:>3} {:<25} {:>7} {:>7} {:>7} {:>8} {:>6} {:>10} {:>10}'
|
|
if verbose and log_directory:
|
|
print(line_format.format(*headings))
|
|
for test in benchmarks:
|
|
test_output = instrument_test(driver, test, num_samples)
|
|
if test_output[0] == 'Totals':
|
|
continue
|
|
if verbose:
|
|
if log_directory:
|
|
print(line_format.format(*test_output))
|
|
else:
|
|
print(','.join(test_output))
|
|
output.append(test_output)
|
|
(samples, _min, _max, mean) = map(int, test_output[2:6])
|
|
total_tests += 1
|
|
total_min += _min
|
|
total_max += _max
|
|
total_mean += mean
|
|
if not output:
|
|
return
|
|
formatted_output = '\n'.join([','.join(l) for l in output])
|
|
totals = map(str, ['Totals', total_tests, total_min, total_max,
|
|
total_mean, '0', '0', '0'])
|
|
totals_output = '\n\n' + ','.join(totals)
|
|
if verbose:
|
|
if log_directory:
|
|
print(line_format.format(*([''] + totals)))
|
|
else:
|
|
print(totals_output[1:])
|
|
formatted_output += totals_output
|
|
if log_directory:
|
|
log_results(log_directory, driver, formatted_output, swift_repo)
|
|
return formatted_output
|
|
|
|
|
|
def submit(args):
|
|
print("SVN revision:\t", args.revision)
|
|
print("Machine name:\t", args.machine)
|
|
print("Iterations:\t", args.iterations)
|
|
print("Optimizations:\t", ','.join(args.optimization))
|
|
print("LNT host:\t", args.lnt_host)
|
|
starttime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
|
|
print("Start time:\t", starttime)
|
|
data = {}
|
|
data['Tests'] = []
|
|
data['Machine'] = {'Info': {'name': args.machine}, 'Name': args.machine}
|
|
print("\nRunning benchmarks...")
|
|
for optset in args.optimization:
|
|
print("Opt level:\t", optset)
|
|
file = os.path.join(args.tests, "Benchmark_" + optset)
|
|
try:
|
|
res = run_benchmarks(
|
|
file, benchmarks=get_tests(file, args),
|
|
num_samples=args.iterations)
|
|
data['Tests'].extend(parse_results(res, optset))
|
|
except subprocess.CalledProcessError as e:
|
|
print("Execution failed.. Test results are empty.")
|
|
print("Process output:\n", e.output)
|
|
|
|
endtime = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
|
|
data['Run'] = {'End Time': endtime,
|
|
'Info': {'inferred_run_order': str(args.revision),
|
|
'run_order': str(args.revision),
|
|
'tag': 'nts',
|
|
'test_suite_revision': 'None'},
|
|
'Start Time': starttime}
|
|
print("End time:\t", endtime)
|
|
|
|
submit_to_lnt(data, args.lnt_host)
|
|
return 0
|
|
|
|
|
|
def run(args):
|
|
optset = args.optimization
|
|
file = os.path.join(args.tests, "Benchmark_" + optset)
|
|
run_benchmarks(
|
|
file, benchmarks=get_tests(file, args),
|
|
num_samples=args.iterations, verbose=True,
|
|
log_directory=args.output_dir,
|
|
swift_repo=args.swift_repo)
|
|
return 0
|
|
|
|
|
|
def format_name(log_path):
|
|
"""Return the filename and directory for a log file"""
|
|
return '/'.join(log_path.split('/')[-2:])
|
|
|
|
|
|
def compare_logs(compare_script, new_log, old_log, log_dir, opt):
|
|
"""Return diff of log files at paths `new_log` and `old_log`"""
|
|
print('Comparing %s %s ...' % (format_name(old_log), format_name(new_log)))
|
|
subprocess.call([compare_script, '--old-file', old_log,
|
|
'--new-file', new_log, '--format', 'markdown',
|
|
'--output', os.path.join(log_dir, 'latest_compare_{0}.md'
|
|
.format(opt))])
|
|
|
|
|
|
def compare(args):
|
|
log_dir = args.log_dir
|
|
swift_repo = args.swift_repo
|
|
compare_script = args.compare_script
|
|
baseline_branch = args.baseline_branch
|
|
current_branch = get_current_git_branch(swift_repo)
|
|
current_branch_dir = os.path.join(log_dir, current_branch)
|
|
baseline_branch_dir = os.path.join(log_dir, baseline_branch)
|
|
|
|
if current_branch != baseline_branch and \
|
|
not os.path.isdir(baseline_branch_dir):
|
|
print(('Unable to find benchmark logs for {baseline_branch} branch. ' +
|
|
'Set a baseline benchmark log by passing --benchmark to ' +
|
|
'build-script while on {baseline_branch} branch.')
|
|
.format(baseline_branch=baseline_branch))
|
|
return 1
|
|
|
|
recent_logs = {}
|
|
for branch_dir in [current_branch_dir, baseline_branch_dir]:
|
|
for opt in ['O', 'Onone']:
|
|
recent_logs[os.path.basename(branch_dir) + '_' + opt] = sorted(
|
|
glob.glob(os.path.join(
|
|
branch_dir, 'Benchmark_' + opt + '-*.log')),
|
|
key=os.path.getctime, reverse=True)
|
|
|
|
if current_branch == baseline_branch:
|
|
if len(recent_logs[baseline_branch + '_O']) > 1 and \
|
|
len(recent_logs[baseline_branch + '_Onone']) > 1:
|
|
compare_logs(compare_script,
|
|
recent_logs[baseline_branch + '_O'][0],
|
|
recent_logs[baseline_branch + '_O'][1],
|
|
log_dir, 'O')
|
|
compare_logs(compare_script,
|
|
recent_logs[baseline_branch + '_Onone'][0],
|
|
recent_logs[baseline_branch + '_Onone'][1],
|
|
log_dir, 'Onone')
|
|
else:
|
|
print(('{baseline_branch}/{baseline_branch} comparison ' +
|
|
'skipped: no previous {baseline_branch} logs')
|
|
.format(baseline_branch=baseline_branch))
|
|
else:
|
|
# TODO: Check for outdated baseline branch log
|
|
if len(recent_logs[current_branch + '_O']) == 0 or \
|
|
len(recent_logs[current_branch + '_Onone']) == 0:
|
|
print('branch sanity failure: missing branch logs')
|
|
return 1
|
|
|
|
if len(recent_logs[current_branch + '_O']) == 1 or \
|
|
len(recent_logs[current_branch + '_Onone']) == 1:
|
|
print('branch/branch comparison skipped: no previous branch logs')
|
|
else:
|
|
compare_logs(compare_script,
|
|
recent_logs[current_branch + '_O'][0],
|
|
recent_logs[current_branch + '_O'][1],
|
|
log_dir, 'O')
|
|
compare_logs(compare_script,
|
|
recent_logs[current_branch + '_Onone'][0],
|
|
recent_logs[current_branch + '_Onone'][1],
|
|
log_dir, 'Onone')
|
|
|
|
if len(recent_logs[baseline_branch + '_O']) == 0 or \
|
|
len(recent_logs[baseline_branch + '_Onone']) == 0:
|
|
print(('branch/{baseline_branch} failure: no {baseline_branch} ' +
|
|
'logs')
|
|
.format(baseline_branch=baseline_branch))
|
|
return 1
|
|
else:
|
|
compare_logs(compare_script,
|
|
recent_logs[current_branch + '_O'][0],
|
|
recent_logs[baseline_branch + '_O'][0],
|
|
log_dir, 'O')
|
|
compare_logs(compare_script,
|
|
recent_logs[current_branch + '_Onone'][0],
|
|
recent_logs[baseline_branch + '_Onone'][0],
|
|
log_dir, 'Onone')
|
|
|
|
# TODO: Fail on large regressions
|
|
|
|
return 0
|
|
|
|
|
|
def positive_int(value):
|
|
ivalue = int(value)
|
|
if not (ivalue > 0):
|
|
raise ValueError
|
|
return ivalue
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
epilog='Example: ./Benchmark_Driver run -i 5 -f Prefix -f .*Suffix.*'
|
|
)
|
|
subparsers = parser.add_subparsers(
|
|
title='Swift benchmark driver commands',
|
|
help='See COMMAND -h for additional arguments', metavar='<command>')
|
|
|
|
parent_parser = argparse.ArgumentParser(add_help=False)
|
|
benchmarks_group = parent_parser.add_mutually_exclusive_group()
|
|
benchmarks_group.add_argument(
|
|
'benchmarks',
|
|
default=[],
|
|
help='benchmark to run (default: all)', nargs='*', metavar="BENCHMARK")
|
|
benchmarks_group.add_argument(
|
|
'-f', '--filter', dest='filters', action='append',
|
|
help='run all tests whose name match regular expression PATTERN, ' +
|
|
'multiple filters are supported', metavar="PATTERN")
|
|
parent_parser.add_argument(
|
|
'-t', '--tests',
|
|
help='directory containing Benchmark_O{,none,size} ' +
|
|
'(default: DRIVER_DIR)',
|
|
default=DRIVER_DIR)
|
|
|
|
submit_parser = subparsers.add_parser(
|
|
'submit',
|
|
help='Run benchmarks and submit results to LNT',
|
|
parents=[parent_parser])
|
|
submit_parser.add_argument(
|
|
'-o', '--optimization', nargs='+',
|
|
help='optimization levels to use (default: O Onone Osize)',
|
|
default=['O', 'Onone', 'Osize'])
|
|
submit_parser.add_argument(
|
|
'-i', '--iterations',
|
|
help='number of times to run each test (default: 10)',
|
|
type=positive_int, default=10)
|
|
submit_parser.add_argument(
|
|
'-m', '--machine', required=True,
|
|
help='LNT machine name')
|
|
submit_parser.add_argument(
|
|
'-r', '--revision', required=True,
|
|
help='SVN revision of compiler to identify the LNT run', type=int)
|
|
submit_parser.add_argument(
|
|
'-l', '--lnt_host', required=True,
|
|
help='LNT host to submit results to')
|
|
submit_parser.set_defaults(func=submit)
|
|
|
|
run_parser = subparsers.add_parser(
|
|
'run',
|
|
help='Run benchmarks and output results to stdout',
|
|
parents=[parent_parser])
|
|
run_parser.add_argument(
|
|
'-o', '--optimization',
|
|
metavar='OPT',
|
|
choices=['O', 'Onone', 'Osize'],
|
|
help='optimization level to use: {O,Onone,Osize}, (default: O)',
|
|
default='O')
|
|
run_parser.add_argument(
|
|
'-i', '--iterations',
|
|
help='number of times to run each test (default: 1)',
|
|
type=positive_int, default=1)
|
|
run_parser.add_argument(
|
|
'--output-dir',
|
|
help='log results to directory (default: no logging)')
|
|
run_parser.add_argument(
|
|
'--swift-repo',
|
|
help='absolute path to Swift source repo for branch comparison')
|
|
run_parser.set_defaults(func=run)
|
|
|
|
compare_parser = subparsers.add_parser(
|
|
'compare',
|
|
help='Compare benchmark results')
|
|
compare_parser.add_argument(
|
|
'--log-dir', required=True,
|
|
help='directory containing benchmark logs')
|
|
compare_parser.add_argument(
|
|
'--swift-repo', required=True,
|
|
help='absolute path to Swift source repo')
|
|
compare_parser.add_argument(
|
|
'--compare-script', required=True,
|
|
help='absolute path to compare script')
|
|
compare_parser.add_argument(
|
|
'--baseline-branch', default='master',
|
|
help='attempt to compare results to baseline results for specified '
|
|
'branch (default: master)')
|
|
compare_parser.set_defaults(func=compare)
|
|
|
|
args = parser.parse_args()
|
|
if args.func != compare and isinstance(args.optimization, list):
|
|
args.optimization = sorted(list(set(args.optimization)))
|
|
return args.func(args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
exit(main())
|