55 lines
3.1 KiB
Python
55 lines
3.1 KiB
Python
|
"""Independently compare browser responses/decode measurements to the current manifests."""
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
import shutil
|
||
|
|
from urllib.parse import urlparse
|
||
|
|
|
||
|
|
ROOT = Path(__file__).resolve().parents[4]
|
||
|
|
EVIDENCE = Path(__file__).resolve().parent
|
||
|
|
OUTPUT = EVIDENCE / 'browser-final'
|
||
|
|
report = json.loads((EVIDENCE / 'browser-results.json').read_text())
|
||
|
|
assert report['stats']['expected'] == 30
|
||
|
|
assert all(report['stats'][key] == 0 for key in ('unexpected', 'flaky', 'skipped'))
|
||
|
|
assert report.get('errors', []) == []
|
||
|
|
expected = {}
|
||
|
|
for directory in ('audio', 'ambience'):
|
||
|
|
manifest = json.loads((ROOT / 'public' / directory / 'manifest.json').read_text())
|
||
|
|
for entry in manifest['files']:
|
||
|
|
expected[f'/{directory}/{entry["file"]}'] = entry
|
||
|
|
summary = []
|
||
|
|
(EVIDENCE / 'screenshots').mkdir()
|
||
|
|
for project in ('desktop', 'mobile'):
|
||
|
|
paths = list(OUTPUT.glob(f'*-{project}/browser-audio-integrity.json'))
|
||
|
|
assert len(paths) == 1
|
||
|
|
data = json.loads(paths[0].read_text())
|
||
|
|
assert len(data['files']) == 109
|
||
|
|
assert data['external'] == data['errors'] == data['failed'] == []
|
||
|
|
assert {urlparse(row['url']).path for row in data['files']} == set(expected)
|
||
|
|
max_rms_difference = max_peak_difference = 0
|
||
|
|
for row in data['files']:
|
||
|
|
entry = expected[urlparse(row['url']).path]
|
||
|
|
assert row['sha256'] == entry['sha256'] and row['bytes'] == entry['bytes']
|
||
|
|
assert row['duration'] == entry['duration'] and row['sampleRate'] == 24000 and row['channels'] == 1
|
||
|
|
rms_difference = abs(row['rms'] - entry['rms'])
|
||
|
|
peak_difference = abs(row['peak'] - entry['peak'])
|
||
|
|
assert max(rms_difference, peak_difference) <= 1 / 32768
|
||
|
|
max_rms_difference = max(max_rms_difference, rms_difference)
|
||
|
|
max_peak_difference = max(max_peak_difference, peak_difference)
|
||
|
|
name = f'{project}-audio-integrity.json'
|
||
|
|
shutil.copy2(paths[0], EVIDENCE / name)
|
||
|
|
summary.append({'project':project, 'chrome':data['chrome'], 'filesMatched':109,
|
||
|
|
'externalRequests':0, 'consoleOrPageErrors':0, 'failedRequests':0, 'evidence':name,
|
||
|
|
'levelTolerance':1 / 32768, 'maximumRmsDifference':max_rms_difference, 'maximumPeakDifference':max_peak_difference,
|
||
|
|
'levelToleranceReason':'One PCM16 step allows browser float decode scaling; SHA-256 and bytes must match exactly.'})
|
||
|
|
for source, target in [('home.png','start.png'), ('game.png','game.png'), ('results.png','normal-results.png'), ('early-exit.png','early-exit.png')]:
|
||
|
|
matches = list(OUTPUT.glob(f'*-{project}/{source}'))
|
||
|
|
assert len(matches) == 1, (project, source, matches)
|
||
|
|
shutil.copy2(matches[0], EVIDENCE / 'screenshots' / f'{project}-{target}')
|
||
|
|
matches = list(OUTPUT.glob(f'*-{project}/attachments/runtime-audit-*.json'))
|
||
|
|
assert len(matches) == 1
|
||
|
|
audit = json.loads(matches[0].read_text())
|
||
|
|
assert audit['errors'] == audit['failed'] == audit['external'] == []
|
||
|
|
shutil.copy2(matches[0], EVIDENCE / f'{project}-runtime-audit.json')
|
||
|
|
with (EVIDENCE / 'browser-integrity-summary.json').open('x') as out: out.write(json.dumps(summary, indent=2) + '\n')
|
||
|
|
print(json.dumps({'browserStats':report['stats'], 'integrity':summary}, indent=2))
|