#!/usr/bin/env python3 import argparse import base64 import datetime import json import math import re import sqlite3 import statistics import urllib.request def format_time(time): # If the fractional part of the second is not exactly 3 or 6 digits long, # fromisoformat fails, so we pad it with zeroes. rfc3339_re = r"(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}\.)(\d{1,6})([+-].*)?" match = re.fullmatch(rfc3339_re, time) time = f"{match.group(1)}{match.group(2):<06}{match.group(3) or ''}" time = datetime.datetime.fromisoformat(time) # Velcom stores its time in UTC but omits the time zone offset in the # database, so we add it back here. time = time.replace(tzinfo=datetime.timezone.utc) return time.isoformat() def get_run_data(con, run_id): ( runner_name, runner_info, start_time, stop_time, commit_hash, error_type, error, ) = con.execute( """ SELECT runner_name, runner_info, start_time, stop_time, commit_hash, error_type, error FROM run WHERE id = ? """, [run_id], ).fetchone() output = [] measurements = {} for ( measurement_id, benchmark, metric, unit, error, ) in con.execute( """ SELECT id, benchmark, metric, unit, error FROM measurement WHERE run_id = ? """, [run_id], ): if error: for line in error.splitlines(): output.append((2, line)) continue values = con.execute( "SELECT value FROM measurement_value WHERE measurement_id = ?", [measurement_id], ).fetchall() values = [value for (value,) in values] measurements[f"{metric}/{benchmark}"] = { "value": statistics.mean(values), "unit": unit, } if error_type: output.append((2, f"The entire run failed with error of type {error_type}.")) output.append((2, "")) for line in error.splitlines(): output.append((2, line)) data = { "id": run_id, "hash": commit_hash, "bench_method": "imported from velcom", "start": format_time(start_time), "end": format_time(stop_time), "exit_code": -1 if error_type else 0, "output": output, "measurements": measurements, } return runner_name, runner_info, data def send_run_data(url, token, worker_name, worker_info, data): body = { "info": worker_info, "secret": "nothing to see here", "status": {"type": "idle"}, "submit_run": data, } request = urllib.request.Request(f"{url.rstrip('/')}/api/worker/status") request.method = "POST" # Easier than using HTTPBasicAuthHandler credentials = base64.b64encode(f"{worker_name}:{token}".encode("utf-8")) request.add_header("Authorization", f"Basic {credentials.decode('utf-8')}") request.add_header("Content-Type", "application/json; charset=utf-8") request.data = json.dumps(body).encode("utf-8") urllib.request.urlopen(request) def main(): parser = argparse.ArgumentParser() parser.add_argument("velcom_db") parser.add_argument("repo_id") parser.add_argument("url") parser.add_argument("token") args = parser.parse_args() print(args) con = sqlite3.connect(args.velcom_db, isolation_level=None) con.execute("BEGIN") run_ids = con.execute( """ SELECT id FROM run WHERE repo_id = ? AND commit_hash IS NOT NULL ORDER BY start_time ASC """, [args.repo_id], ) run_ids = [run_id for (run_id,) in run_ids] for i, run_id in enumerate(run_ids): print(f"Sending run {run_id} ({i+1}/{len(run_ids)})") worker_name, worker_info, data = get_run_data(con, run_id) try: send_run_data(args.url, args.token, worker_name, worker_info, data) except urllib.request.HTTPError as e: for line in e.read().decode("utf-8").splitlines(): print(f"# {line}") print() if __name__ == "__main__": main()