#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Revalidate retained public benchmark evidence; execute no guest programs."""
import argparse
import hashlib
import itertools
import json
import math
from pathlib import Path
import statistics
ROOT = Path(__file__).resolve().parents[1]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--record', type=Path, default=ROOT/'research/results/cmoc-benchmark-recorded.json')
parser.add_argument('--reference', type=Path, default=ROOT/'experiments/cmoc-benchmarks/reference.py')
parser.add_argument('--summary-output', type=Path, help='Optional new output; retained evidence is never overwritten')
args = parser.parse_args()
if args.summary_output and args.summary_output.exists(): parser.error('Choose a new summary path')
record = json.loads(args.record.read_text())
if record['status'] != 'passed': raise ValueError('Recorded study did not pass')
source = args.reference.read_bytes()
if hashlib.sha256(source).hexdigest() != record['source_hashes']['reference.py']:
raise ValueError('Reference model differs from the original recorded input')
sources = {'reference.py': source}
model = {'__name__': 'pinned_cmoc_reference'}
exec(compile(sources['reference.py'], 'pinned CoCoBenchmarks/reference.py', 'exec'), model)
if model['CASES'] != record['cases']: raise ValueError('Workload order mismatch')
axes = [record[k] for k in ('host_arches', 'cases', 'profiles', 'cpu_modes', 'seeds')]
wanted = set(itertools.product(*axes, range(record['repeats'])))
seen = set(); deterministic = {}; binaries = {}; groups = {}
for row in record['runs']:
key = tuple(row[k] for k in ('host_arch', 'case', 'profile', 'cpu', 'seed', 'repeat'))
if key in seen or key not in wanted: raise ValueError('Duplicate or foreign matrix cell')
seen.add(key)
expected = model['expected'](row['seed'], record['cases'].index(row['case']))
if row['output'] != expected: raise ValueError('Serialized output mismatch: ' + str(key))
if row['md'] != (1 if row['cpu'] == '6309-native' else 0): raise ValueError('Guest mode mismatch')
repeated = (row['cycles'], row['output'], row['sizes'], row['binary_sha256'])
repeat_key = key[:-1]
if repeat_key in deterministic and deterministic[repeat_key] != repeated:
raise ValueError('Nondeterministic retained repeat')
deterministic[repeat_key] = repeated
binary_key = key[1:5]
if binary_key in binaries and binaries[binary_key] != row['binary_sha256']:
raise ValueError('Host compiler slices differ')
binaries[binary_key] = row['binary_sha256']
groups.setdefault(key[:4], []).append(row)
if seen != wanted or len(seen) != 864: raise ValueError('Incomplete expected 864-cell matrix')
if [(c['cpu'], c['cycles'], c['expected_cycles']) for c in record['calibration']] != [
('6809', 6, 6), ('6309-emulation', 6, 6), ('6309-native', 3, 3)]:
raise ValueError('Cycle calibration mismatch')
means = {key: {'cycles': statistics.mean(r['cycles'] for r in rows),
'initialized_bytes': statistics.mean(r['sizes']['initialized_bytes'] for r in rows)}
for key, rows in groups.items()}
summaries = []
for host, cpu in itertools.product(record['host_arches'], record['cpu_modes']):
def value(case, profile, metric): return means[host, case, profile, cpu][metric]
def gain(a, b):
return 100 * (1 - math.exp(statistics.mean(math.log(value(c, b, 'cycles') /
value(c, a, 'cycles')) for c in record['cases'])))
release_bytes = sum(value(c, 'release', 'initialized_bytes') for c in record['cases'])
omit_bytes = sum(value(c, 'release_omit', 'initialized_bytes') for c in record['cases'])
summaries.append({'compiler_host_architecture': host, 'guest_mode': cpu,
'release_vs_debug_cycle_reduction_percent': gain('debug', 'release'),
'omit_vs_release_cycle_reduction_percent': gain('release', 'release_omit'),
'omit_vs_release_summed_initialized_byte_reduction_percent': 100 * (1 - omit_bytes / release_bytes),
'release_summed_initialized_bytes': release_bytes, 'omit_summed_initialized_bytes': omit_bytes})
table = []
for case in record['cases']:
row = {'workload': case}
for profile in record['profiles']:
row[profile] = means[record['host_arches'][0], case, profile, '6809']
row['omit_cycle_reduction_percent'] = 100 * (1 - row['release_omit']['cycles'] / row['release']['cycles'])
table.append(row)
result = {'validated_retained_executions':len(seen),'new_guest_executions':0,
'serialized_outputs_revalidated':len(seen),'complete_matrix':True,
'repeat_determinism':True,'host_binary_agreement':True,
'summaries':summaries,'mc6809_workloads':table}
text = json.dumps(result,indent=2)+'\n'
if args.summary_output:
args.summary_output.parent.mkdir(parents=True,exist_ok=True)
args.summary_output.write_text(text)
print(text,end='')
if __name__=='__main__':main()
Evidence record
tools/import_cmoc_benchmark.py
From Windows Application to Portable Emulation Platform
5,268 bytes
SHA-2569a1fe4b1215d18d84062ce75edc7ace34ffc560c4d0cbf42fb1ea2c7678dccd5