#!/usr/bin/python # # ==-- process-stats-dir - summarize one or more Swift -stats-output-dirs --==# # # This source file is part of the Swift.org open source project # # Copyright (c) 2014-2017 Apple Inc. and the Swift project authors # Licensed under Apache License v2.0 with Runtime Library Exception # # See https://swift.org/LICENSE.txt for license information # See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors # # ==------------------------------------------------------------------------==# # # This file processes the contents of one or more directories generated by # `swiftc -stats-output-dir` and emits summary data, traces etc. for analysis. import argparse import csv import datetime import json import os import platform import random import re import sys import time import urllib import urllib2 class JobStats: def __init__(self, jobkind, jobid, module, start_usec, dur_usec, jobargs, stats): self.jobkind = jobkind self.jobid = jobid self.module = module self.start_usec = start_usec self.dur_usec = dur_usec self.jobargs = jobargs self.stats = stats def is_driver_job(self): return self.jobkind == 'driver' def is_frontend_job(self): return self.jobkind == 'frontend' def driver_jobs_ran(self): assert(self.is_driver_job()) return self.stats.get("Driver.NumDriverJobsRun", 0) def driver_jobs_skipped(self): assert(self.is_driver_job()) return self.stats.get("Driver.NumDriverJobsSkipped", 0) def driver_jobs_total(self): assert(self.is_driver_job()) return self.driver_jobs_ran() + self.driver_jobs_skipped() def merged_with(self, other): merged_stats = {} for k, v in self.stats.items() + other.stats.items(): merged_stats[k] = v + merged_stats.get(k, 0.0) merged_kind = self.jobkind if other.jobkind != merged_kind: merged_kind = "" merged_module = self.module if other.module != merged_module: merged_module = "" merged_start = min(self.start_usec, other.start_usec) merged_end = max(self.start_usec + self.dur_usec, other.start_usec + other.dur_usec) merged_dur = merged_end - merged_start return JobStats(merged_kind, random.randint(0, 1000000000), merged_module, merged_start, merged_dur, self.jobargs + other.jobargs, merged_stats) def incrementality_percentage(self): assert(self.is_driver_job()) ran = self.driver_jobs_ran() total = self.driver_jobs_total() return round((float(ran) / float(total)) * 100.0, 2) # Return a JSON-formattable object of the form preferred by google chrome's # 'catapult' trace-viewer. def to_catapult_trace_obj(self): return {"name": self.module, "cat": self.jobkind, "ph": "X", # "X" == "complete event" "pid": self.jobid, "tid": 1, "ts": self.start_usec, "dur": self.dur_usec, "args": self.jobargs} def start_timestr(self): t = datetime.datetime.fromtimestamp(self.start_usec / 1000000.0) return t.strftime("%Y-%m-%d %H:%M:%S") def end_timestr(self): t = datetime.datetime.fromtimestamp((self.start_usec + self.dur_usec) / 1000000.0) return t.strftime("%Y-%m-%d %H:%M:%S") def pick_lnt_metric_suffix(self, metric_name): if "BytesOutput" in metric_name: return "code_size" if "RSS" in metric_name or "BytesAllocated" in metric_name: return "mem" return "compile" # Return a JSON-formattable object of the form preferred by LNT's # 'submit' format. def to_lnt_test_obj(self, args): run_info = { "run_order": str(args.lnt_order), "tag": str(args.lnt_tag), } run_info.update(dict(args.lnt_run_info)) stats = self.stats return { "Machine": { "Name": args.lnt_machine, "Info": dict(args.lnt_machine_info) }, "Run": { "Start Time": self.start_timestr(), "End Time": self.end_timestr(), "Info": run_info }, "Tests": [ { "Data": [v], "Info": {}, "Name": "%s.%s.%s.%s" % (args.lnt_tag, self.module, k, self.pick_lnt_metric_suffix(k)) } for (k, v) in stats.items() ] } # Return an array of JobStats objects def load_stats_dir(path): jobstats = [] auxpat = (r"(?P[^-]+)-(?P[^-]+)-(?P[^-]+)" + r"-(?P[^-]+)-(?P[^-]+)") fpat = (r"^stats-(?P\d+)-swift-(?P\w+)-" + auxpat + r"-(?P\d+)(-.*)?.json$") for root, dirs, files in os.walk(path): for f in files: m = re.match(fpat, f) if m: # NB: "pid" in fpat is a random number, not unix pid. mg = m.groupdict() jobkind = mg['kind'] jobid = int(mg['pid']) start_usec = int(mg['start']) module = mg["module"] jobargs = [mg["input"], mg["triple"], mg["out"], mg["opt"]] j = json.load(open(os.path.join(root, f))) dur_usec = 1 patstr = (r"time\.swift-" + jobkind + r"\." + auxpat + r"\.wall$") pat = re.compile(patstr) stats = dict() for (k, v) in j.items(): if k.startswith("time."): v = int(1000000.0 * float(v)) stats[k] = v tm = re.match(pat, k) if tm: dur_usec = v e = JobStats(jobkind=jobkind, jobid=jobid, module=module, start_usec=start_usec, dur_usec=dur_usec, jobargs=jobargs, stats=stats) jobstats.append(e) return jobstats # Passed args with 2-element remainder ["old", "new"], return a list of tuples # of the form [(name, (oldstats, newstats))] where each name is a common subdir # of each of "old" and "new", and the stats are those found in the respective # dirs. def load_paired_stats_dirs(args): assert(len(args.remainder) == 2) paired_stats = [] (old, new) = args.remainder for p in sorted(os.listdir(old)): full_old = os.path.join(old, p) full_new = os.path.join(new, p) if not (os.path.exists(full_old) and os.path.isdir(full_old) and os.path.exists(full_new) and os.path.isdir(full_new)): continue old_stats = load_stats_dir(full_old) new_stats = load_stats_dir(full_new) if len(old_stats) == 0 or len(new_stats) == 0: continue paired_stats.append((p, (old_stats, new_stats))) return paired_stats def write_catapult_trace(args): allstats = [] for path in args.remainder: allstats += load_stats_dir(path) json.dump([s.to_catapult_trace_obj() for s in allstats], args.output) def write_lnt_values(args): for d in args.remainder: stats = load_stats_dir(d) merged = merge_all_jobstats(stats) j = merged.to_lnt_test_obj(args) if args.lnt_submit is None: json.dump(j, args.output, indent=4) else: url = args.lnt_submit print "\nsubmitting to LNT server: " + url json_report = {'input_data': json.dumps(j), 'commit': '1'} data = urllib.urlencode(json_report) response_str = urllib2.urlopen(urllib2.Request(url, data)) response = json.loads(response_str.read()) print "### response:" print response if 'success' in response: print "server response:\tSuccess" else: print "server response:\tError" print "error:\t", response['error'] sys.exit(1) def merge_all_jobstats(jobstats): m = None for j in jobstats: if m is None: m = j else: m = m.merged_with(j) return m def show_paired_incrementality(args): fieldnames = ["old_pct", "old_skip", "new_pct", "new_skip", "delta_pct", "delta_skip", "name"] out = csv.DictWriter(args.output, fieldnames, dialect='excel-tab') out.writeheader() for (name, (oldstats, newstats)) in load_paired_stats_dirs(args): olddriver = merge_all_jobstats([x for x in oldstats if x.is_driver_job()]) newdriver = merge_all_jobstats([x for x in newstats if x.is_driver_job()]) if olddriver is None or newdriver is None: continue oldpct = olddriver.incrementality_percentage() newpct = newdriver.incrementality_percentage() deltapct = newpct - oldpct oldskip = olddriver.driver_jobs_skipped() newskip = newdriver.driver_jobs_skipped() deltaskip = newskip - oldskip out.writerow(dict(name=name, old_pct=oldpct, old_skip=oldskip, new_pct=newpct, new_skip=newskip, delta_pct=deltapct, delta_skip=deltaskip)) def show_incrementality(args): fieldnames = ["incrementality", "name"] out = csv.DictWriter(args.output, fieldnames, dialect='excel-tab') out.writeheader() for path in args.remainder: stats = load_stats_dir(path) for s in stats: if s.is_driver_job(): pct = s.incrementality_percentage() out.writerow(dict(name=os.path.basename(path), incrementality=pct)) def diff_and_pct(old, new): if old == 0: if new == 0: return (0, 0.0) else: return (new, 100.0) delta = (new - old) delta_pct = round((float(delta) / float(old)) * 100.0, 2) return (delta, delta_pct) def update_epoch_value(d, name, epoch, value): changed = 0 if name in d: (existing_epoch, existing_value) = d[name] if existing_epoch > epoch: print("note: keeping newer value %d from epoch %d for %s" % (existing_value, existing_epoch, name)) epoch = existing_epoch value = existing_value elif existing_value == value: epoch = existing_epoch else: (_, delta_pct) = diff_and_pct(existing_value, value) print ("note: changing value %d -> %d (%.2f%%) for %s" % (existing_value, value, delta_pct, name)) changed = 1 d[name] = (epoch, value) return (epoch, value, changed) def read_stats_dict_from_csv(f): infieldnames = ["epoch", "name", "value"] c = csv.DictReader(f, infieldnames, dialect='excel-tab', quoting=csv.QUOTE_NONNUMERIC) d = {} for row in c: epoch = int(row["epoch"]) name = row["name"] value = int(row["value"]) update_epoch_value(d, name, epoch, value) return d # The idea here is that a "baseline" is a (tab-separated) CSV file full of # the counters you want to track, each prefixed by an epoch timestamp of # the last time the value was reset. # # When you set a fresh baseline, all stats in the provided stats dir are # written to the baseline. When you set against an _existing_ baseline, # only the counters mentioned in the existing baseline are updated, and # only if their values differ. # # Finally, since it's a line-oriented CSV file, you can put: # # mybaseline.csv merge=union # # in your .gitattributes file, and forget about merge conflicts. The reader # function above will take the later epoch anytime it detects duplicates, # so union-merging is harmless. Duplicates will be eliminated whenever the # next baseline-set is done. def set_csv_baseline(args): existing = None if os.path.exists(args.set_csv_baseline): with open(args.set_csv_baseline, "r") as f: existing = read_stats_dict_from_csv(f) print ("updating %d baseline entries in %s" % (len(existing), args.set_csv_baseline)) else: print "making new baseline " + args.set_csv_baseline fieldnames = ["epoch", "name", "value"] with open(args.set_csv_baseline, "wb") as f: out = csv.DictWriter(f, fieldnames, dialect='excel-tab', quoting=csv.QUOTE_NONNUMERIC) m = merge_all_jobstats([s for d in args.remainder for s in load_stats_dir(d)]) changed = 0 newepoch = int(time.time()) for name in sorted(m.stats.keys()): epoch = newepoch value = m.stats[name] if existing is not None: if name not in existing: continue (epoch, value, chg) = update_epoch_value(existing, name, epoch, value) changed += chg out.writerow(dict(epoch=int(epoch), name=name, value=int(value))) if existing is not None: print "changed %d entries in baseline" % changed return 0 def compare_to_csv_baseline(args): old_stats = read_stats_dict_from_csv(args.compare_to_csv_baseline) m = merge_all_jobstats([s for d in args.remainder for s in load_stats_dir(d)]) new_stats = m.stats regressions = 0 outfieldnames = ["old", "new", "delta_pct", "name"] out = csv.DictWriter(args.output, outfieldnames, dialect='excel-tab') out.writeheader() for stat_name in sorted(old_stats.keys()): (_, old) = old_stats[stat_name] new = new_stats.get(stat_name, 0) (delta, delta_pct) = diff_and_pct(old, new) if (stat_name.startswith("time.") and abs(delta) < args.delta_usec_thresh): continue if abs(delta_pct) < args.delta_pct_thresh: continue out.writerow(dict(name=stat_name, old=int(old), new=int(new), delta_pct=delta_pct)) if delta > 0: regressions += 1 return regressions def main(): parser = argparse.ArgumentParser() parser.add_argument("--verbose", action="store_true", help="Report activity verbosely") parser.add_argument("--output", default="-", type=argparse.FileType('wb', 0), help="Write output to file") parser.add_argument("--paired", action="store_true", help="Process two dirs-of-stats-dirs, pairwise") parser.add_argument("--delta-pct-thresh", type=float, default=0.01, help="Percentage change required to report") parser.add_argument("--delta-usec-thresh", type=int, default=100000, help="Absolute delta on times required to report") parser.add_argument("--lnt-machine", type=str, default=platform.node(), help="Machine name for LNT submission") parser.add_argument("--lnt-run-info", action='append', default=[], type=lambda kv: kv.split("="), help="Extra key=value pairs for LNT run-info") parser.add_argument("--lnt-machine-info", action='append', default=[], type=lambda kv: kv.split("="), help="Extra key=value pairs for LNT machine-info") parser.add_argument("--lnt-order", type=str, default=str(int(time.time())), help="Order for LNT submission") parser.add_argument("--lnt-tag", type=str, default="swift-compile", help="Tag for LNT submission") parser.add_argument("--lnt-submit", type=str, default=None, help="URL to submit LNT data to (rather than print)") modes = parser.add_mutually_exclusive_group(required=True) modes.add_argument("--catapult", action="store_true", help="emit a 'catapult'-compatible trace of events") modes.add_argument("--incrementality", action="store_true", help="summarize the 'incrementality' of a build") modes.add_argument("--set-csv-baseline", type=str, default=None, help="Merge stats from a stats-dir into a CSV baseline") modes.add_argument("--compare-to-csv-baseline", type=argparse.FileType('rb', 0), default=None, metavar="BASELINE.csv", help="Compare stats dir to named CSV baseline") modes.add_argument("--lnt", action="store_true", help="Emit an LNT-compatible test summary") parser.add_argument('remainder', nargs=argparse.REMAINDER, help="stats-dirs to process") args = parser.parse_args() if len(args.remainder) == 0: parser.print_help() return 1 if args.catapult: write_catapult_trace(args) elif args.set_csv_baseline is not None: return set_csv_baseline(args) elif args.compare_to_csv_baseline is not None: return compare_to_csv_baseline(args) elif args.incrementality: if args.paired: show_paired_incrementality(args) else: show_incrementality(args) elif args.lnt: write_lnt_values(args) return None sys.exit(main())