mirror of
https://github.com/apple/swift.git
synced 2025-12-14 20:36:38 +01:00
9c998d7ce8
Previously, we had some white listing lists for allocations that were assigned into globals during a benchmarks running. Now we instead run two different iterations with the second running an additional time. Then we subtract the counts. This enables me to get rid of the whitelist.
136 lines
4.8 KiB
Python
136 lines
4.8 KiB
Python
#!/usr/bin/env python
|
|
|
|
# ===--- Benchmark_RuntimeLeaksRunner.in ----------------------------------===//
|
|
#
|
|
# This source file is part of the Swift.org open source project
|
|
#
|
|
# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
|
|
# Licensed under Apache License v2.0 with Runtime Library Exception
|
|
#
|
|
# See http://swift.org/LICENSE.txt for license information
|
|
# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
|
#
|
|
# ===----------------------------------------------------------------------===//
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
sys.path.append("@PATH_TO_DRIVER_LIBRARY@")
|
|
|
|
import perf_test_driver
|
|
|
|
# This is a hacked up XFAIL list. It should really be a json file, but it will
|
|
# work for now. Add in the exact name of the pass to XFAIL.
|
|
XFAIL_LIST = [
|
|
]
|
|
|
|
# Global objective-c classes created by various frameworks. We do not care about these.
|
|
IGNORABLE_GLOBAL_OBJC_CLASSES = set([
|
|
'__NSPlaceholderDate',
|
|
'NSCache',
|
|
'__NSPlaceholderTimeZone',
|
|
'NSPlaceholderNumber',
|
|
'NSPlaceholderString',
|
|
'__NSPlaceholderArray',
|
|
'__NSPlaceholderDictionary',
|
|
'_NSPlaceholderData',
|
|
'_NSJSONReader'
|
|
])
|
|
|
|
|
|
class LeaksRunnerResult(perf_test_driver.Result):
|
|
|
|
def __init__(self, name, count=None):
|
|
# True = 1, False = 0.
|
|
#
|
|
# TODO: Why are we using booleans to communicate status here?
|
|
status = count is None or count > 0
|
|
perf_test_driver.Result.__init__(self, name, status, "", XFAIL_LIST)
|
|
self.count = count
|
|
|
|
def get_count(self):
|
|
if self.count is not None:
|
|
return str(self.count)
|
|
return "N/A"
|
|
|
|
def print_data(self, max_test_len):
|
|
fmt = '{:<%d}{:<10}{:}' % (max_test_len + 5)
|
|
print(fmt.format(self.get_name(), self.get_result(),
|
|
self.get_count()))
|
|
|
|
class LeaksRunnerBenchmarkDriver(perf_test_driver.BenchmarkDriver):
|
|
|
|
def __init__(self, binary, xfail_list, num_samples, num_iters):
|
|
perf_test_driver.BenchmarkDriver.__init__(
|
|
self, binary, xfail_list,
|
|
enable_parallel=True)
|
|
self.num_samples = num_samples
|
|
self.num_iters = num_iters
|
|
|
|
def print_data_header(self, max_test_len):
|
|
fmt = '{:<%d}{:<10}{:}' % (max_test_len + 5)
|
|
print(fmt.format('Name', 'Result', 'RC Delta'))
|
|
|
|
# Propagate any data from this class that is needed for individual
|
|
# tests. The reason this is needed is to avoid issues with attempting to
|
|
# access a value in a different process.
|
|
def prepare_input(self, name):
|
|
return {'num_samples': self.num_samples, 'num_iters': self.num_iters}
|
|
|
|
def run_test(self, data, num_iters):
|
|
try:
|
|
p = subprocess.Popen([data['path'], "--run-all", "--num-samples={}".format(data['num_samples']),
|
|
"--num-iters={}".format(num_iters), data['test_name']],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
error_out = p.communicate()[1].split("\n")
|
|
except OSError:
|
|
print("Child Process Failed! (%s,%s)" % (data['path'], data['test_name']))
|
|
return None
|
|
|
|
try:
|
|
# We grab the second line since swift globals get lazily created in the
|
|
# first iteration.
|
|
d = json.loads(error_out[1])
|
|
d['objc_objects'] = [x for x in d['objc_objects'] if x not in IGNORABLE_GLOBAL_OBJC_CLASSES]
|
|
d['objc_count'] = len(d['objc_objects'])
|
|
|
|
total_count = d['objc_count'] + d['swift_count']
|
|
return total_count
|
|
except (KeyError, ValueError):
|
|
print("Failed parse output! (%s,%s)" % (data['path'], data['test_name']))
|
|
return None
|
|
|
|
|
|
def process_input(self, data):
|
|
test_name = '({},{})'.format(data['opt'], data['test_name'])
|
|
print("Running {}...".format(test_name))
|
|
sys.stdout.flush()
|
|
total_count1 = self.run_test(data, data['num_iters'])
|
|
if total_count1 is None:
|
|
return LeaksRunnerResult(test_name)
|
|
total_count2 = self.run_test(data, data['num_iters']+1)
|
|
if total_count2 is None:
|
|
return LeaksRunnerResult(test_name)
|
|
return LeaksRunnerResult(test_name, total_count2 - total_count1)
|
|
|
|
SWIFT_BIN_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
def parse_args():
|
|
import argparse
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('-filter', type=str, default=None,
|
|
help='Filter out any test that does not match the given regex')
|
|
parser.add_argument('-num-samples', type=int, default=2)
|
|
parser.add_argument('-num-iters', type=int, default=2)
|
|
return parser.parse_args()
|
|
|
|
if __name__ == "__main__":
|
|
args = parse_args()
|
|
l = LeaksRunnerBenchmarkDriver(SWIFT_BIN_DIR, XFAIL_LIST, args.num_samples, args.num_iters)
|
|
if l.run(args.filter):
|
|
sys.exit(0)
|
|
else:
|
|
sys.exit(-1)
|