#!/usr/bin/python
# -*- coding: utf-8 -*-
# ===--- compare_perf_tests.py -------------------------------------------===//
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See http://swift.org/LICENSE.txt for license information
# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ===---------------------------------------------------------------------===//
import argparse
import csv
import sys
TESTNAME = 1
SAMPLES = 2
MIN = 3
MAX = 4
MEAN = 5
SD = 6
MEDIAN = 7
HTML = """
{0}
"""
HTML_TABLE = """
"""
HTML_ROW = """
| {0} |
{1} |
{2} |
{3} |
{5} |
"""
MARKDOWN_ROW = "{0} | {1} | {2} | {3} | {4} \n"
HEADER_SPLIT = "---"
MARKDOWN_DETAIL = """
{0} ({1})
{2}
"""
PAIN_FORMAT = """
Regression: {0}
Improvement: {1}
No Changes: {2}
"""
def main():
global RATIO_MIN
global RATIO_MAX
old_results = {}
new_results = {}
old_max_results = {}
new_max_results = {}
ratio_list = {}
delta_list = {}
unknown_list = {}
complete_perf_list = []
increased_perf_list = []
decreased_perf_list = []
normal_perf_list = []
parser = argparse.ArgumentParser(description="Compare Performance tests.")
parser.add_argument('--old-file',
help='Baseline performance test suite (csv file)',
required=True)
parser.add_argument('--new-file',
help='New performance test suite (csv file)',
required=True)
parser.add_argument('--format',
help='Supported format git, html and markdown',
default="markdown")
parser.add_argument('--output', help='Output file name')
parser.add_argument('--changes-only',
help='Output only affected tests', action='store_true')
parser.add_argument('--new-branch',
help='Name of the new branch', default="NEW_MIN")
parser.add_argument('--old-branch',
help='Name of the old branch', default="OLD_MIN")
parser.add_argument('--delta-threshold',
help='delta threshold', default="0.05")
args = parser.parse_args()
old_file = args.old_file
new_file = args.new_file
new_branch = args.new_branch
old_branch = args.old_branch
old_data = csv.reader(open(old_file))
new_data = csv.reader(open(new_file))
RATIO_MIN = 1 - float(args.delta_threshold)
RATIO_MAX = 1 + float(args.delta_threshold)
for row in old_data:
if (len(row) > 7 and row[MIN].isdigit()):
if row[TESTNAME] in old_results:
if old_results[row[TESTNAME]] > row[MIN]:
old_results[row[TESTNAME]] = int(row[MIN])
if old_max_results[row[TESTNAME]] < row[MAX]:
old_max_results[row[TESTNAME]] = int(row[MAX])
else:
old_results[row[TESTNAME]] = int(row[MIN])
old_max_results[row[TESTNAME]] = int(row[MAX])
for row in new_data:
if (len(row) > 7 and row[MIN].isdigit()):
if row[TESTNAME] in new_results:
if new_results[row[TESTNAME]] > row[MIN]:
new_results[row[TESTNAME]] = int(row[MIN])
if new_max_results[row[TESTNAME]] < row[MAX]:
new_max_results[row[TESTNAME]] = int(row[MAX])
new_results[row[TESTNAME]] = int(row[MIN])
new_max_results[row[TESTNAME]] = int(row[MAX])
ratio_total = 0
for key in new_results.keys():
ratio = (old_results[key]+0.001)/(new_results[key]+0.001)
ratio_list[key] = round(ratio, 2)
ratio_total *= ratio
delta = (((float(new_results[key]+0.001) /
(old_results[key]+0.001)) - 1) * 100)
delta_list[key] = round(delta, 2)
if ((old_results[key] < new_results[key] and
new_results[key] < old_max_results[key]) or
(new_results[key] < old_results[key] and
old_results[key] < new_max_results[key])):
unknown_list[key] = "(?)"
else:
unknown_list[key] = ""
(complete_perf_list,
increased_perf_list,
decreased_perf_list,
normal_perf_list) = sort_ratio_list(ratio_list, args.changes_only)
"""
Create markdown formated table
"""
test_name_width = max_width(ratio_list, title='TEST', key_len=True)
new_time_width = max_width(new_results, title=new_branch)
old_time_width = max_width(old_results, title=old_branch)
delta_width = max_width(delta_list, title='DELTA (%)')
markdown_table_header = "\n" + MARKDOWN_ROW.format(
"TEST".ljust(test_name_width),
old_branch.ljust(old_time_width),
new_branch.ljust(new_time_width),
"DELTA (%)".ljust(delta_width),
"SPEEDUP".ljust(2))
markdown_table_header += MARKDOWN_ROW.format(
HEADER_SPLIT.ljust(test_name_width),
HEADER_SPLIT.ljust(old_time_width),
HEADER_SPLIT.ljust(new_time_width),
HEADER_SPLIT.ljust(delta_width),
HEADER_SPLIT.ljust(2))
markdown_regression = ""
for i, key in enumerate(decreased_perf_list):
ratio = "{0:.2f}x".format(ratio_list[key])
if i == 0:
markdown_regression = markdown_table_header
markdown_regression += MARKDOWN_ROW.format(
key.ljust(test_name_width),
str(old_results[key]).ljust(old_time_width),
str(new_results[key]).ljust(new_time_width),
("{0:+.1f}%".format(delta_list[key])).ljust(delta_width),
"**{0}{1}**".format(str(ratio).ljust(2), unknown_list[key]))
markdown_improvement = ""
for i, key in enumerate(increased_perf_list):
ratio = "{0:.2f}x".format(ratio_list[key])
if i == 0:
markdown_improvement = markdown_table_header
markdown_improvement += MARKDOWN_ROW.format(
key.ljust(test_name_width),
str(old_results[key]).ljust(old_time_width),
str(new_results[key]).ljust(new_time_width),
("{0:+.1f}%".format(delta_list[key])).ljust(delta_width),
"**{0}{1}**".format(str(ratio).ljust(2), unknown_list[key]))
markdown_normal = ""
for i, key in enumerate(normal_perf_list):
ratio = "{0:.2f}x".format(ratio_list[key])
if i == 0:
markdown_normal = markdown_table_header
markdown_normal += MARKDOWN_ROW.format(
key.ljust(test_name_width),
str(old_results[key]).ljust(old_time_width),
str(new_results[key]).ljust(new_time_width),
("{0:+.1f}%".format(delta_list[key])).ljust(delta_width),
"{0}{1}".format(str(ratio).ljust(2), unknown_list[key]))
markdown_data = MARKDOWN_DETAIL.format("Regression",
len(decreased_perf_list),
markdown_regression, "open")
markdown_data += MARKDOWN_DETAIL.format("Improvement",
len(increased_perf_list),
markdown_improvement, "")
markdown_data += MARKDOWN_DETAIL.format("No Changes",
len(normal_perf_list),
markdown_normal, "")
if args.format:
if args.format.lower() != "markdown":
pain_data = PAIN_FORMAT.format(markdown_regression,
markdown_improvement,
markdown_normal)
print(pain_data.replace("|", " ").replace("-", " "))
else:
print(markdown_data)
if args.format:
if args.format.lower() == "html":
"""
Create HTML formated table
"""
html_data = convert_to_html(ratio_list, old_results, new_results,
delta_list, unknown_list, old_branch,
new_branch, args.changes_only)
if args.output:
write_to_file(args.output, html_data)
else:
print("Error: missing --output flag.")
exit(1)
elif args.format.lower() == "markdown" and args.output:
write_to_file(args.output, markdown_data)
elif args.format.lower() != "markdown":
print("{0} is unknown format.".format(args.format))
exit(1)
def convert_to_html(ratio_list, old_results, new_results, delta_list,
unknown_list, old_branch, new_branch, changes_only):
(complete_perf_list,
increased_perf_list,
decreased_perf_list,
normal_perf_list) = sort_ratio_list(ratio_list, changes_only)
html_rows = ""
for key in complete_perf_list:
if ratio_list[key] < RATIO_MIN:
color = "red"
elif ratio_list[key] > RATIO_MAX:
color = "green"
else:
color = "black"
if len(decreased_perf_list) > 0 and key == decreased_perf_list[0]:
html_rows += HTML_ROW.format(
"Regression:",
"", "", "", "black", "", "")
if len(increased_perf_list) > 0 and key == increased_perf_list[0]:
html_rows += HTML_ROW.format(
"Improvement:",
"", "", "", "black", "", "")
if len(normal_perf_list) > 0 and key == normal_perf_list[0]:
html_rows += HTML_ROW.format(
"No Changes:",
"", "", "", "black", "", "")
html_rows += HTML_ROW.format(key, old_results[key],
new_results[key],
"{0:+.1f}%".format(delta_list[key]),
color,
"{0:.2f}x {1}".format(ratio_list[key],
unknown_list[key]))
html_table = HTML_TABLE.format("TEST", old_branch, new_branch,
"DELTA (%)", "SPEEDUP", html_rows)
html_data = HTML.format(html_table)
return html_data
def write_to_file(file_name, data):
"""
Write data to given file
"""
file = open(file_name, "w")
file.write(data)
file.close
def sort_ratio_list(ratio_list, changes_only=False):
"""
Return 3 sorted list imporvment, regression and normal.
"""
decreased_perf_list = []
increased_perf_list = []
sorted_normal_perf_list = []
normal_perf_list = {}
for key, v in sorted(ratio_list.items(), key=lambda x: x[1]):
if ratio_list[key] < RATIO_MIN:
decreased_perf_list.append(key)
elif ratio_list[key] > RATIO_MAX:
increased_perf_list.append(key)
else:
normal_perf_list[key] = v
for key, v in sorted(normal_perf_list.items(), key=lambda x: x[1],
reverse=True):
sorted_normal_perf_list.append(key)
if changes_only:
complete_perf_list = decreased_perf_list + increased_perf_list
else:
complete_perf_list = (decreased_perf_list + increased_perf_list +
sorted_normal_perf_list)
return (complete_perf_list, increased_perf_list,
decreased_perf_list, sorted_normal_perf_list)
def nthroot(y, n):
x, xp = 1, -1
while abs(x - xp) > 1:
xp, x = x, x - x/n + y/(n * x**(n-1))
while x**n > y:
x -= 1
return x
def max_width(items, title, key_len=False):
"""
Returns the max length of string in the list
"""
width = len(str(title))
for key in items.keys():
if key_len:
if width < len(str(key)):
width = len(str(key))
else:
if width < len(str(items[key])):
width = len(str(items[key]))
return width
if __name__ == "__main__":
sys.exit(main())