← Supporting evidence

Evidence record

experiments/cmoc-benchmarks/test_benchmark.py

From Windows Application to Portable Emulation Platform

2,613 bytes

# SPDX-License-Identifier: GPL-3.0-or-later
"""Guard evidence accounting: full images, malformed data and repeat weighting."""
import tempfile
import unittest
from pathlib import Path
from benchmark import image_sizes, summarize, markdown, PROFILES


class EvidenceTests(unittest.TestCase):
    def image(self, data):
        with tempfile.TemporaryDirectory() as folder:
            binary = Path(folder) / 'test.bin'
            binary.write_bytes(data)
            return image_sizes(binary, '\n'.join([
                'Section: start (main.o) load at 2800, length 0002',
                'Section: code (runtime.o) load at 2802, length 0003',
                'Section: bss (main.o) load at 2805, length 0010',
                'Section: program_end (main.o) load at 2815, length 0000',
            ]))

    def test_sizes_include_linked_helpers_and_bss_separately(self):
        result = self.image(bytes.fromhex('00 0005 2800 0102030405 ff 0000 2800'))
        self.assertEqual(result, dict(bin_file_bytes=15, initialized_bytes=5,
                                    bss_bytes=16, linked_code_bytes=5, static_ram_span_bytes=21))

    def test_malformed_image_cannot_be_measured(self):
        for data in [b'', bytes.fromhex('00 0005 2800 0102'),
                     bytes.fromhex('00 0001 2800 01'),
                     bytes.fromhex('ff 0000 2800 00')]:
            with self.subTest(data=data), self.assertRaises(ValueError):
                self.image(data)

    def test_repeats_do_not_reweight_seeds(self):
        rows = []
        for seed, cycles, repeats in [(1, 100, 2), (2, 300, 4)]:
            rows += [dict(host_arch='arm64',cpu='6809',case='one',profile='release',
                          seed=seed,cycles=cycles,sizes=dict(initialized_bytes=10,static_ram_span_bytes=20))] * repeats
        self.assertEqual(summarize({'runs': rows})[0]['mean_cycles'], 200)

    def test_report_handles_native_only_selection(self):
        rows = [dict(host_arch='arm64',cpu='6309-native',case='one',profile=p,
                     seed=1,cycles=n,sizes=dict(initialized_bytes=10,static_ram_span_bytes=20))
                for p,n in zip(PROFILES, [300,200,100])]
        record = dict(runs=rows,host_arches=['arm64'],cpu_modes=['6309-native'],cases=['one'],
                      toolchain={'cmoc':'test'},tool_hashes={'libexec/cmoc':'test'},
                      vcc_source_revision='test',vcc_source_dirty=False)
        text = markdown(record)
        self.assertIn('6309-native measurements', text)
        self.assertIn('50.00% fewer cycles', text)


if __name__ == '__main__':
    unittest.main()

SHA-256f2fee14c9fc6c5a99517a5c74969dca4efecd8a5d9d04a01ded19dd028f5d459

A closer look

Open original