← Supporting evidence

Evidence record

tools/import_cmoc_benchmark.py

From Windows Application to Portable Emulation Platform

5,268 bytes

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-3.0-or-later
"""Revalidate retained public benchmark evidence; execute no guest programs."""
import argparse
import hashlib
import itertools
import json
import math
from pathlib import Path
import statistics

ROOT = Path(__file__).resolve().parents[1]

def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--record', type=Path, default=ROOT/'research/results/cmoc-benchmark-recorded.json')
    parser.add_argument('--reference', type=Path, default=ROOT/'experiments/cmoc-benchmarks/reference.py')
    parser.add_argument('--summary-output', type=Path, help='Optional new output; retained evidence is never overwritten')
    args = parser.parse_args()
    if args.summary_output and args.summary_output.exists(): parser.error('Choose a new summary path')
    record = json.loads(args.record.read_text())
    if record['status'] != 'passed': raise ValueError('Recorded study did not pass')
    source = args.reference.read_bytes()
    if hashlib.sha256(source).hexdigest() != record['source_hashes']['reference.py']:
        raise ValueError('Reference model differs from the original recorded input')
    sources = {'reference.py': source}
    model = {'__name__': 'pinned_cmoc_reference'}
    exec(compile(sources['reference.py'], 'pinned CoCoBenchmarks/reference.py', 'exec'), model)
    if model['CASES'] != record['cases']: raise ValueError('Workload order mismatch')
    axes = [record[k] for k in ('host_arches', 'cases', 'profiles', 'cpu_modes', 'seeds')]
    wanted = set(itertools.product(*axes, range(record['repeats'])))
    seen = set(); deterministic = {}; binaries = {}; groups = {}
    for row in record['runs']:
        key = tuple(row[k] for k in ('host_arch', 'case', 'profile', 'cpu', 'seed', 'repeat'))
        if key in seen or key not in wanted: raise ValueError('Duplicate or foreign matrix cell')
        seen.add(key)
        expected = model['expected'](row['seed'], record['cases'].index(row['case']))
        if row['output'] != expected: raise ValueError('Serialized output mismatch: ' + str(key))
        if row['md'] != (1 if row['cpu'] == '6309-native' else 0): raise ValueError('Guest mode mismatch')
        repeated = (row['cycles'], row['output'], row['sizes'], row['binary_sha256'])
        repeat_key = key[:-1]
        if repeat_key in deterministic and deterministic[repeat_key] != repeated:
            raise ValueError('Nondeterministic retained repeat')
        deterministic[repeat_key] = repeated
        binary_key = key[1:5]
        if binary_key in binaries and binaries[binary_key] != row['binary_sha256']:
            raise ValueError('Host compiler slices differ')
        binaries[binary_key] = row['binary_sha256']
        groups.setdefault(key[:4], []).append(row)
    if seen != wanted or len(seen) != 864: raise ValueError('Incomplete expected 864-cell matrix')
    if [(c['cpu'], c['cycles'], c['expected_cycles']) for c in record['calibration']] != [
            ('6809', 6, 6), ('6309-emulation', 6, 6), ('6309-native', 3, 3)]:
        raise ValueError('Cycle calibration mismatch')
    means = {key: {'cycles': statistics.mean(r['cycles'] for r in rows),
                   'initialized_bytes': statistics.mean(r['sizes']['initialized_bytes'] for r in rows)}
             for key, rows in groups.items()}
    summaries = []
    for host, cpu in itertools.product(record['host_arches'], record['cpu_modes']):
        def value(case, profile, metric): return means[host, case, profile, cpu][metric]
        def gain(a, b):
            return 100 * (1 - math.exp(statistics.mean(math.log(value(c, b, 'cycles') /
                       value(c, a, 'cycles')) for c in record['cases'])))
        release_bytes = sum(value(c, 'release', 'initialized_bytes') for c in record['cases'])
        omit_bytes = sum(value(c, 'release_omit', 'initialized_bytes') for c in record['cases'])
        summaries.append({'compiler_host_architecture': host, 'guest_mode': cpu,
            'release_vs_debug_cycle_reduction_percent': gain('debug', 'release'),
            'omit_vs_release_cycle_reduction_percent': gain('release', 'release_omit'),
            'omit_vs_release_summed_initialized_byte_reduction_percent': 100 * (1 - omit_bytes / release_bytes),
            'release_summed_initialized_bytes': release_bytes, 'omit_summed_initialized_bytes': omit_bytes})
    table = []
    for case in record['cases']:
        row = {'workload': case}
        for profile in record['profiles']:
            row[profile] = means[record['host_arches'][0], case, profile, '6809']
        row['omit_cycle_reduction_percent'] = 100 * (1 - row['release_omit']['cycles'] / row['release']['cycles'])
        table.append(row)
    result = {'validated_retained_executions':len(seen),'new_guest_executions':0,
        'serialized_outputs_revalidated':len(seen),'complete_matrix':True,
        'repeat_determinism':True,'host_binary_agreement':True,
        'summaries':summaries,'mc6809_workloads':table}
    text = json.dumps(result,indent=2)+'\n'
    if args.summary_output:
        args.summary_output.parent.mkdir(parents=True,exist_ok=True)
        args.summary_output.write_text(text)
    print(text,end='')

if __name__=='__main__':main()

SHA-2569a1fe4b1215d18d84062ce75edc7ace34ffc560c4d0cbf42fb1ea2c7678dccd5

A closer look

Open original