69 lines
3.8 KiB
Python
69 lines
3.8 KiB
Python
"""Independently compare browser responses/decode measurements to the current manifests."""
|
|
import base64
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
from urllib.parse import urlparse
|
|
|
|
ROOT = Path(__file__).resolve().parents[4]
|
|
EVIDENCE = Path(__file__).resolve().parent
|
|
OUTPUT = EVIDENCE / 'browser-final'
|
|
report = json.loads((EVIDENCE / 'browser-results.json').read_text())
|
|
assert report['stats']['expected'] == 30
|
|
assert all(report['stats'][key] == 0 for key in ('unexpected', 'flaky', 'skipped'))
|
|
assert report.get('errors', []) == []
|
|
expected = {}
|
|
for directory in ('audio', 'ambience'):
|
|
manifest = json.loads((ROOT / 'public' / directory / 'manifest.json').read_text())
|
|
for entry in manifest['files']:
|
|
expected[f'/{directory}/{entry["file"]}'] = entry
|
|
summary = []
|
|
(EVIDENCE / 'screenshots').mkdir(exist_ok=True)
|
|
runtime_attachments = {}
|
|
def visit(suites):
|
|
for suite in suites:
|
|
for spec in suite.get('specs', []):
|
|
for test in spec['tests']:
|
|
for result in test['results']:
|
|
for attachment in result.get('attachments', []):
|
|
if attachment['name'] == 'runtime-audit':
|
|
project = test['projectName']
|
|
assert project not in runtime_attachments
|
|
runtime_attachments[project] = base64.b64decode(attachment['body'], validate=True)
|
|
visit(suite.get('suites', []))
|
|
visit(report['suites'])
|
|
assert set(runtime_attachments) == {'desktop', 'mobile'}
|
|
for project in ('desktop', 'mobile'):
|
|
paths = list(OUTPUT.glob(f'*-{project}/browser-audio-integrity.json'))
|
|
assert len(paths) == 1
|
|
data = json.loads(paths[0].read_text())
|
|
assert len(data['files']) == 109
|
|
assert data['external'] == data['errors'] == data['failed'] == []
|
|
assert {urlparse(row['url']).path for row in data['files']} == set(expected)
|
|
max_rms_difference = max_peak_difference = 0
|
|
for row in data['files']:
|
|
entry = expected[urlparse(row['url']).path]
|
|
assert row['sha256'] == entry['sha256'] and row['bytes'] == entry['bytes']
|
|
assert row['duration'] == entry['duration'] and row['sampleRate'] == 24000 and row['channels'] == 1
|
|
rms_difference = abs(row['rms'] - entry['rms'])
|
|
peak_difference = abs(row['peak'] - entry['peak'])
|
|
assert max(rms_difference, peak_difference) <= 1 / 32768
|
|
max_rms_difference = max(max_rms_difference, rms_difference)
|
|
max_peak_difference = max(max_peak_difference, peak_difference)
|
|
name = f'{project}-audio-integrity.json'
|
|
shutil.copy2(paths[0], EVIDENCE / name)
|
|
summary.append({'project':project, 'chrome':data['chrome'], 'filesMatched':109,
|
|
'externalRequests':0, 'consoleOrPageErrors':0, 'failedRequests':0, 'evidence':name,
|
|
'levelTolerance':1 / 32768, 'maximumRmsDifference':max_rms_difference, 'maximumPeakDifference':max_peak_difference,
|
|
'levelToleranceReason':'One PCM16 step allows browser float decode scaling; SHA-256 and bytes must match exactly.'})
|
|
for source, target in [('home.png','start.png'), ('game.png','game.png'), ('results.png','normal-results.png'), ('early-exit.png','early-exit.png')]:
|
|
matches = list(OUTPUT.glob(f'*-{project}/{source}'))
|
|
assert len(matches) == 1, (project, source, matches)
|
|
shutil.copy2(matches[0], EVIDENCE / 'screenshots' / f'{project}-{target}')
|
|
raw = runtime_attachments[project]
|
|
audit = json.loads(raw)
|
|
assert audit['errors'] == audit['failed'] == audit['external'] == []
|
|
with (EVIDENCE / f'{project}-runtime-audit.json').open('xb') as out: out.write(raw)
|
|
with (EVIDENCE / 'browser-integrity-summary.json').open('x') as out: out.write(json.dumps(summary, indent=2) + '\n')
|
|
print(json.dumps({'browserStats':report['stats'], 'integrity':summary}, indent=2))
|