#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-3.0-or-later """Revalidate retained public benchmark evidence; execute no guest programs.""" import argparse import hashlib import itertools import json import math from pathlib import Path import statistics ROOT = Path(__file__).resolve().parents[1] def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--record', type=Path, default=ROOT/'research/results/cmoc-benchmark-recorded.json') parser.add_argument('--reference', type=Path, default=ROOT/'experiments/cmoc-benchmarks/reference.py') parser.add_argument('--summary-output', type=Path, help='Optional new output; retained evidence is never overwritten') args = parser.parse_args() if args.summary_output and args.summary_output.exists(): parser.error('Choose a new summary path') record = json.loads(args.record.read_text()) if record['status'] != 'passed': raise ValueError('Recorded study did not pass') source = args.reference.read_bytes() if hashlib.sha256(source).hexdigest() != record['source_hashes']['reference.py']: raise ValueError('Reference model differs from the original recorded input') sources = {'reference.py': source} model = {'__name__': 'pinned_cmoc_reference'} exec(compile(sources['reference.py'], 'pinned CoCoBenchmarks/reference.py', 'exec'), model) if model['CASES'] != record['cases']: raise ValueError('Workload order mismatch') axes = [record[k] for k in ('host_arches', 'cases', 'profiles', 'cpu_modes', 'seeds')] wanted = set(itertools.product(*axes, range(record['repeats']))) seen = set(); deterministic = {}; binaries = {}; groups = {} for row in record['runs']: key = tuple(row[k] for k in ('host_arch', 'case', 'profile', 'cpu', 'seed', 'repeat')) if key in seen or key not in wanted: raise ValueError('Duplicate or foreign matrix cell') seen.add(key) expected = model['expected'](row['seed'], record['cases'].index(row['case'])) if row['output'] != expected: raise ValueError('Serialized output mismatch: ' + str(key)) if row['md'] != (1 if row['cpu'] == '6309-native' else 0): raise ValueError('Guest mode mismatch') repeated = (row['cycles'], row['output'], row['sizes'], row['binary_sha256']) repeat_key = key[:-1] if repeat_key in deterministic and deterministic[repeat_key] != repeated: raise ValueError('Nondeterministic retained repeat') deterministic[repeat_key] = repeated binary_key = key[1:5] if binary_key in binaries and binaries[binary_key] != row['binary_sha256']: raise ValueError('Host compiler slices differ') binaries[binary_key] = row['binary_sha256'] groups.setdefault(key[:4], []).append(row) if seen != wanted or len(seen) != 864: raise ValueError('Incomplete expected 864-cell matrix') if [(c['cpu'], c['cycles'], c['expected_cycles']) for c in record['calibration']] != [ ('6809', 6, 6), ('6309-emulation', 6, 6), ('6309-native', 3, 3)]: raise ValueError('Cycle calibration mismatch') means = {key: {'cycles': statistics.mean(r['cycles'] for r in rows), 'initialized_bytes': statistics.mean(r['sizes']['initialized_bytes'] for r in rows)} for key, rows in groups.items()} summaries = [] for host, cpu in itertools.product(record['host_arches'], record['cpu_modes']): def value(case, profile, metric): return means[host, case, profile, cpu][metric] def gain(a, b): return 100 * (1 - math.exp(statistics.mean(math.log(value(c, b, 'cycles') / value(c, a, 'cycles')) for c in record['cases']))) release_bytes = sum(value(c, 'release', 'initialized_bytes') for c in record['cases']) omit_bytes = sum(value(c, 'release_omit', 'initialized_bytes') for c in record['cases']) summaries.append({'compiler_host_architecture': host, 'guest_mode': cpu, 'release_vs_debug_cycle_reduction_percent': gain('debug', 'release'), 'omit_vs_release_cycle_reduction_percent': gain('release', 'release_omit'), 'omit_vs_release_summed_initialized_byte_reduction_percent': 100 * (1 - omit_bytes / release_bytes), 'release_summed_initialized_bytes': release_bytes, 'omit_summed_initialized_bytes': omit_bytes}) table = [] for case in record['cases']: row = {'workload': case} for profile in record['profiles']: row[profile] = means[record['host_arches'][0], case, profile, '6809'] row['omit_cycle_reduction_percent'] = 100 * (1 - row['release_omit']['cycles'] / row['release']['cycles']) table.append(row) result = {'validated_retained_executions':len(seen),'new_guest_executions':0, 'serialized_outputs_revalidated':len(seen),'complete_matrix':True, 'repeat_determinism':True,'host_binary_agreement':True, 'summaries':summaries,'mc6809_workloads':table} text = json.dumps(result,indent=2)+'\n' if args.summary_output: args.summary_output.parent.mkdir(parents=True,exist_ok=True) args.summary_output.write_text(text) print(text,end='') if __name__=='__main__':main()