AriWeb-Suite/test/hal.test.mjs

/** test/hal.test.mjs — HAL 제어 평면 회귀 테스트(등록·인증·heartbeat·작업 배정·모듈 배포 부분 실패·부트스트랩 관문·외부 Agent 제한·긴급 복구·재해복구·지표 보존·운영자 접근). */
import { test, eq, ok, rejects, tmpDir, done, sleep } from '../lib/test.mjs';
import { invoke } from '../lib/http.mjs';
import { signHeaders } from '../lib/serviceauth.mjs';
import { createHalDeps } from '../hal/deps.mjs';
import { buildApp } from '../hal/app.mjs';
import * as core from '../hal/core.mjs';
import * as ops from '../hal/ops.mjs';
import * as emergency from '../hal/emergency.mjs';
import * as dr from '../hal/dr.mjs';
import { totpCode } from '../hal/totp.mjs';
import { ipAllowed } from '../hal/cidr.mjs';
import { writeFileSync, mkdirSync, existsSync } from 'node:fs';
import { join } from 'node:path';
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';

function makeHal(over = {}) {
  const dataDir = join(tmpDir(), 'hal'); mkdirSync(dataDir, { recursive: true });
  const config = { dataDir, file: join(dataDir, 'config.json'), port: 0, host: '127.0.0.1', publicUrl: 'https://hal.example.test', portalUrl: 'http://127.0.0.1:1', opsAllowCidrs: ['127.0.0.1/32'], metricsRetentionDays: 90, offlineAfterMs: 90000, transferTtlMs: 3600000, emergencyTtlMinutes: 30, agentPollMs: 20, agentWaitTimeoutMs: 4000, fanoutTimeoutMs: 8000, values: {}, set(p) { Object.assign(this, p); }, ...over };
  const d = createHalDeps({ config, dbFile: ':memory:', fetchImpl: async (url) => ({ ok: true, status: 200, text: async () => 'marker-ok', json: async () => ({ ok: true }) }) }); d.jobsRunning = true;
  const app = buildApp(d); return { d, app, dataDir };
}
const sig = (keyId, secret, method, path, body) => Object.fromEntries(Object.entries(signHeaders(keyId, secret, method, path, body === undefined ? undefined : JSON.stringify(body))).map(([k, v]) => [k.toLowerCase(), v]));
const call = (app, keyId, secret, method, path, body, extra = {}) => invoke(app, method, path, { body, headers: { ...sig(keyId, secret, method, path, body), ...extra } });
/** 서버 + Agent 등록 한 벌 */
async function registerAgent({ d, app }, { role = 'web', name = '웹1', kind = 'managed', siteId = null, groupId = null } = {}) {
  const srv = kind === 'managed' ? core.createServer(d, { name, role }) : null;
  const t = core.issueRegistrationToken(d, { kind, serverId: srv?.id, siteId, groupId, label: name });
  const r = await invoke(app, 'POST', '/agent/v1/register', { body: { token: t.token, facts: { hostname: name }, version: '0.1.1' } }); ok(r.json.ok, JSON.stringify(r.json));
  const a = { agentId: r.json.agentId, secret: r.json.secret, serverId: r.json.serverId, hb: (body = {}) => call(app, `agent:${r.json.agentId}`, r.json.secret, 'POST', '/agent/v1/heartbeat', { version: '0.1.1', metrics: { cpu: 5, mem: { total: 100, used: 50 }, disk: { total: 1000, used: 100 }, load1: 0.1 }, modules: {}, clockSkewMs: 0, ...body }), next: () => call(app, `agent:${r.json.agentId}`, r.json.secret, 'GET', '/agent/v1/jobs/next?wait=0'), result: (id, body) => call(app, `agent:${r.json.agentId}`, r.json.secret, 'POST', `/agent/v1/jobs/${id}/result`, body), progress: (id, body) => call(app, `agent:${r.json.agentId}`, r.json.secret, 'POST', `/agent/v1/jobs/${id}/progress`, body) };
  await a.hb(); if (srv) core.approveServer(d, srv.id, 'op'); return a;
}
/** 부트스트랩 완료 상태로 만들기 */
function bootstrapDone(d, version = '0.1.1') { core.claimLive(d, { instanceId: d.instanceId, version }); core.setBootstrap(d, { release_version: version, main_commit: 'abc', artifact_sha256: 'x' }); }
/** Agent 역할을 흉내: 다음 작업을 받아 결과를 보낸다 */
async function actAgent(a, fn) { const n = await a.next(); const job = n.json.job; if (!job) return null; const out = fn ? await fn(job) : { status: 'done', result: { ok: true } }; await a.result(job.id, out); return job; }
const portalSecret = 'portal-secret-example';
/** 작업이 끝날 때까지 tick 을 기다리지 않고 계속 던진다(부모 작업이 자식을 기다리는 동안 자식도 실행되게) */
async function pump(h, jobId, maxMs = 8000) { const ps = []; const t0 = Date.now(); while (Date.now() - t0 < maxMs) { const st = h.d.jobs.get(jobId).status; if (['done', 'failed', 'cancelled', 'needs_user', 'awaiting_external'].includes(st) && !h.d.jobs.list({ status: ['queued', 'running'] }).length) break; ps.push(h.d.jobs.tick().catch(() => {})); await sleep(10); } await Promise.all(ps); return h.d.jobs.get(jobId); }
/** Agent 가 다음 작업이 올 때까지 기다렸다가 처리 */
async function serveOne(a, fn, maxMs = 4000) { const t0 = Date.now(); while (Date.now() - t0 < maxMs) { const n = await a.next(); if (n.json.job) { const out = fn(n.json.job); await a.result(n.json.job.id, out); return n.json.job; } await sleep(10); } return null; }

await test('등록 토큰: 1회만, 만료 거부, 잘못된 토큰 거부', async () => {
  const h = makeHal(); const srv = core.createServer(h.d, { name: 'a', role: 'web' }); const t = core.issueRegistrationToken(h.d, { serverId: srv.id });
  let r = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: t.token, facts: {}, version: '0.1.1' } }); ok(r.json.ok); ok(r.json.secret && r.json.secret.length > 20);
  r = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: t.token } }); eq(r.json.error, 'token_used');
  r = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: 'nope' } }); eq(r.json.error, 'bad_token');
  const t2 = core.issueRegistrationToken(h.d, { serverId: srv.id, ttlMs: -1 }); r = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: t2.token } }); eq(r.json.error, 'token_expired');
  ok(h.d.vault.has(`service:agent:${core.getServer(h.d, srv.id).agent_id}`), '비밀은 보관소에');
});
await test('Agent HMAC 인증: 정상·서명 오류·시계 차이·해지', async () => {
  const h = makeHal(); const a = await registerAgent(h);
  let r = await a.hb(); eq(r.status, 200, JSON.stringify(r.json));
  r = await call(h.app, `agent:${a.agentId}`, 'wrong-secret', 'POST', '/agent/v1/heartbeat', {}); eq(r.status, 401); ok(r.json.message.includes('bad_signature'));
  const hdr = sig(`agent:${a.agentId}`, a.secret, 'POST', '/agent/v1/heartbeat', {}); hdr['x-service-ts'] = String(Date.now() - 10 * 60000); r = await invoke(h.app, 'POST', '/agent/v1/heartbeat', { body: {}, headers: hdr }); eq(r.status, 401); ok(r.json.message.includes('clock_skew'));
  core.revokeAgent(h.d, a.agentId); r = await a.hb(); eq(r.status, 401);
});
await test('heartbeat → 지표·용량·모듈 버전 저장, 무응답 → offline, 지표 90일 보존', async () => {
  const h = makeHal({ offlineAfterMs: 50 }); const a = await registerAgent(h);
  await a.hb({ modules: { webmaker: { version: '1.7.3', active: true, health: 'ok' } }, clockSkewMs: 120000 });
  const s = core.publicServer(h.d, core.getServer(h.d, a.serverId)); eq(s.modules.webmaker.actual, '1.7.3'); eq(s.capacity.diskTotalBytes, 1000); ok(core.metrics(h.d, a.serverId).length >= 1);
  ok(h.d.store.get('SELECT id FROM notifications WHERE incident_key=?', `server:${a.serverId}:clock`), '시계 차이 경고');
  await sleep(80); eq(core.markOffline(h.d).length, 1); eq(core.listServers(h.d)[0].status, 'offline'); await a.hb(); eq(core.listServers(h.d)[0].status, 'ready', 'heartbeat 복귀');
  h.d.store.run('INSERT INTO metrics_samples(server_id, at, cpu) VALUES(?,?,?)', a.serverId, new Date(Date.now() - 100 * 86400000).toISOString(), 1); eq(core.purgeMetrics(h.d), 1); ok(core.metrics(h.d, a.serverId, { since: new Date(0).toISOString() }).every((m) => Date.now() - Date.parse(m.at) < 91 * 86400000));
});
await test('Webmaker 인벤토리 배치: 2MiB heartbeat와 분리·순서 검증·완료 시 원자 반영·후속 heartbeat 보존', async () => {
  const h = makeHal(); const a = await registerAgent(h); const path = '/agent/v1/inventory/webmaker'; const send = (body) => call(h.app, `agent:${a.agentId}`, a.secret, 'POST', path, body);
  const sites = ['a', 'b', 'c'].map((x) => ({ id: x.repeat(32), title: `site-${x}`, domains: [`${x}.test`], files: 1, bytes: 2 })); sites[0].privateContact = 'do-not-store@example.test'; sites[0].nested = { secret: 'do-not-store' }; const meta = { total: 3, errors: [], truncated: false, scannedAt: '2026-10-27T00:00:00.000Z' };
  let r = await send({ scanId: 'scan-1', offset: 1, itemTotal: 3, meta, sites: [sites[1]] }); eq(r.status, 400); eq(r.json.error, 'inventory_batch_order');
  r = await send({ scanId: 'scan-1', offset: 0, itemTotal: 3, meta, sites: sites.slice(0, 2) }); eq(r.status, 200); eq(r.json.complete, false); eq(core.getServer(h.d, a.serverId).facts.webmakerInventory, undefined, '중간 배치는 공개하지 않음');
  r = await send({ scanId: 'scan-1', offset: 2, itemTotal: 3, meta, sites: sites.slice(2) }); eq(r.status, 200); eq(r.json.complete, true); const stored = core.getServer(h.d, a.serverId).facts.webmakerInventory; eq(stored.sites.map((x) => x.id), sites.map((x) => x.id)); eq(stored.truncated, false); ok(!JSON.stringify(stored).includes('privateContact') && !JSON.stringify(stored).includes('do-not-store'), '허용되지 않은 개인정보·중첩 필드 저장 금지'); eq(h.d.store.get('SELECT COUNT(*) AS n FROM agent_inventory_staging').n, 0);
  r = await send({ scanId: 'scan-partial', offset: 0, itemTotal: 1, meta: { ...meta, total: 2, truncated: true }, sites: sites.slice(0, 1) }); eq(r.status, 400); eq(r.json.error, 'inventory_incomplete');
  r = await send({ scanId: 'scan-bad', offset: 0, itemTotal: 1, meta, sites: [{ id: '../bad', title: 'bad' }] }); eq(r.status, 400); eq(r.json.error, 'bad_inventory_batch');
  r = await send({ scanId: 'scan-too-many', offset: 0, itemTotal: 10001, meta, sites: [] }); eq(r.status, 400); eq(r.json.error, 'bad_inventory_batch');
  await a.hb({ facts: { hostname: 'web-after' } }); const facts = core.getServer(h.d, a.serverId).facts; eq(facts.hostname, 'web-after'); eq(facts.webmakerInventory.returned, 3, '작은 heartbeat가 완성된 인벤토리를 지우지 않음');
  r = await send({ scanId: 'scan-cascade', offset: 0, itemTotal: 2, meta: { ...meta, total: 2 }, sites: sites.slice(0, 1) }); eq(r.status, 200); eq(h.d.store.get('SELECT COUNT(*) AS n FROM agent_inventory_staging').n, 1); ok(h.d.store.get("SELECT name FROM sqlite_master WHERE type='trigger' AND name='agent_inventory_staging_agent_cleanup'"), '이전 스키마에도 적용 가능한 Agent 삭제 cleanup trigger'); h.d.store.run('DELETE FROM agents WHERE id=?', a.agentId); eq(h.d.store.get('SELECT COUNT(*) AS n FROM agent_inventory_staging').n, 0, 'Agent 삭제 시 미완료 staging cascade');
});
await test('setBootstrap: HAL 기동 전(행 없음) CLI bootstrap-release 의 기록이 남는다(실제 환경 결함 수정)', async () => {
  const h = makeHal(); eq(h.d.store.get('SELECT COUNT(*) AS n FROM bootstrap_state').n, 0); core.setBootstrap(h.d, { release_version: '0.1.1', main_commit: 'abc', artifact_sha256: 'def' });
  eq(core.bootstrapState(h.d).release_version, '0.1.1'); eq(core.bootstrapState(h.d).main_commit, 'abc');
});
await test('pickServer: web·backup 역할 서버가 없으면 control 서버를 쓴다(서버 한 대 운영, 실제 환경 결함 수정) · 전용 서버가 있으면 그쪽 우선', async () => {
  const h = makeHal(); h.d.vault.set('service:portal', portalSecret); bootstrapDone(h.d);
  const ctl = core.createServer(h.d, { name: 'ctl', role: 'control' }); const t1 = core.issueRegistrationToken(h.d, { kind: 'managed', serverId: ctl.id }); const a1 = core.registerAgent(h.d, { token: t1.token, facts: { hostname: 'ctl', nginx: true }, version: '0.1.1' }); core.approveServer(h.d, ctl.id, 'op'); core.heartbeat(h.d, core.getAgent(h.d, a1.agentId), { metrics: { disk: { total: 100e9, used: 10e9 } }, modules: {} });
  eq(core.pickServer(h.d, { role: 'web' })?.serverId, ctl.id, 'web 요청 → control'); eq(core.pickServer(h.d, { role: 'backup' })?.serverId, ctl.id, 'backup 요청 → control'); eq(core.pickServer(h.d, { role: 'mail' }), null, 'mail 은 대체하지 않는다');
  const web = core.createServer(h.d, { name: 'web', role: 'web' }); const t2 = core.issueRegistrationToken(h.d, { kind: 'managed', serverId: web.id }); const a2 = core.registerAgent(h.d, { token: t2.token, facts: { hostname: 'web', nginx: true }, version: '0.1.1' }); core.approveServer(h.d, web.id, 'op'); core.heartbeat(h.d, core.getAgent(h.d, a2.agentId), { metrics: { disk: { total: 100e9, used: 10e9 } }, modules: {} });
  eq(core.pickServer(h.d, { role: 'web' })?.serverId, web.id, '전용 web 서버 우선');
});
await test('부트스트랩 관문: 첫 Agent 승인 전 포털 API 503, 승인·HAL 실행 버전 뒤 완료·수동 설치 잠김', async () => {
  const h = makeHal(); h.d.vault.set('service:portal', portalSecret);
  let r = await call(h.app, 'portal', portalSecret, 'GET', '/portal/v1/servers'); eq(r.status, 503); eq(r.json.error, 'bootstrap_incomplete');
  core.claimLive(h.d, { instanceId: h.d.instanceId, version: '0.1.1' }); const a = await registerAgent(h, { role: 'control' });
  ok(core.bootstrapComplete(h.d)); eq(core.bootstrapState(h.d).manual_install_locked, 1); eq(core.bootstrapState(h.d).first_agent_id, a.agentId);
  r = await call(h.app, 'portal', portalSecret, 'GET', '/portal/v1/servers'); eq(r.status, 200); eq(r.json.servers.length, 1); ok(!('facts' in r.json.servers[0]));
  r = await call(h.app, 'portal', 'bad', 'GET', '/portal/v1/servers'); eq(r.status, 401);
});
await test('하나의 HAL: 다른 인스턴스의 임대가 살아 있으면 기동 거부, takeover 허용', () => {
  const h = makeHal(); core.claimLive(h.d, { instanceId: 'hal_A', version: '0.1.1' });
  const e = (() => { try { core.claimLive(h.d, { instanceId: 'hal_B', version: '0.1.1' }); } catch (x) { return x; } })(); eq(e.code, 'another_hal_live');
  ok(core.claimLive(h.d, { instanceId: 'hal_B', version: '0.1.1', takeover: true })); ok(core.claimLive(h.d, { instanceId: 'hal_C', version: '0.1.1', now: Date.now() + 120000 }), '임대 만료 뒤 허용');
});
await test('포털 작업: 멱등 생성, Agent 배정·long-poll·결과 → HAL 작업 완료(외부 확인 포함), 이벤트 outbox', async () => {
  const h = makeHal(); h.d.vault.set('service:portal', portalSecret); bootstrapDone(h.d); const a = await registerAgent(h);
  let r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/jobs', { kind: 'site.deploy', target: a.serverId, payload: { siteId: 's1', externalCheckUrl: 'https://s1.example.test/', expectMarker: 'marker' }, idempotencyKey: 'dep-1', groupId: 'g', subjectType: 'site', subjectId: 's1' }); eq(r.status, 200, JSON.stringify(r.json)); ok(r.json.created); const id = r.json.halJobId;
  r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/jobs', { kind: 'site.deploy', target: a.serverId, payload: {}, idempotencyKey: 'dep-1' }); eq(r.json.created, false); eq(r.json.halJobId, id);
  r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/jobs', { kind: 'module.deploy', target: a.serverId }); eq(r.status, 400, '포털은 module.* 요청 불가');
  const p = h.d.jobs.tick(); await sleep(30);
  let n = await a.next(); eq(n.json.job.kind, 'site.deploy'); eq(n.json.job.payload.siteId, 's1'); ok(!('_dedupe' in n.json.job.payload));
  await a.progress(n.json.job.id, { progress: 40, steps: [{ id: 'x', state: 'running' }] }); await sleep(40); ok(h.d.jobs.get(id).progress >= 40);
  await a.result(n.json.job.id, { status: 'done', result: { version: 'v1', siteUrl: 'https://s1.example.test/' } }); await p;
  const j = h.d.jobs.get(id); eq(j.status, 'done'); eq(j.result.version, 'v1'); ok(j.result.externalCheck.ok); eq(j.steps.map((s) => s.state), ['done', 'done', 'done']);
  r = await call(h.app, 'portal', portalSecret, 'GET', `/portal/v1/jobs/${id}`); eq(r.json.status, 'done'); eq(r.json.result.version, 'v1');
  ok(h.d.store.get('SELECT COUNT(*) AS n FROM portal_events_outbox').n >= 2, '상태 변화마다 outbox');
  // 실패 결과 → failed, 재시도 API → queued
  r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/jobs', { kind: 'site.check', target: a.serverId, payload: { siteId: 's1', urls: [] } }); const id2 = r.json.halJobId; const p2 = h.d.jobs.tick(); await sleep(30); await actAgent(a, () => ({ status: 'failed', error: 'disk full', errorCode: 'enospc' })); await p2; eq(h.d.jobs.get(id2).status, 'failed'); eq(h.d.jobs.get(id2).error_code, 'enospc');
  r = await call(h.app, 'portal', portalSecret, 'POST', `/portal/v1/jobs/${id2}/retry`); eq(r.json.status, 'queued');
  // needs_user 결과 → HAL 작업 needs_user
  const p3 = h.d.jobs.tick(); await sleep(30); await actAgent(a, () => ({ status: 'needs_user', userMessage: 'sudo 필요' })); await p3; eq(h.d.jobs.get(id2).status, 'needs_user'); ok(h.d.jobs.get(id2).user_message.includes('sudo'));
});
await test('모듈 배포: 릴리스 승인·HAL 먼저·대상별 자식 작업·일부 실패는 partial, 실제 버전 불일치는 실패, 되돌리기', async () => {
  const h = makeHal(); bootstrapDone(h.d); const a1 = await registerAgent(h, { name: 'w1', role: 'control' }); const a2 = await registerAgent(h, { name: 'w2', role: 'web' });
  const art = join(h.dataDir, 'artifacts', '0.1.1'); mkdirSync(art, { recursive: true }); writeFileSync(join(art, 'portal-0.1.1.tar.gz'), 'x'); writeFileSync(join(art, 'hal-0.1.1.tar.gz'), 'y');
  const sum = (f) => createHash('sha256').update(f).digest('hex');
  core.registerRelease(h.d, { version: '0.1.1', mainCommit: 'abc', builtAt: new Date().toISOString(), artifacts: { portal: { file: 'portal-0.1.1.tar.gz', sha256: sum('x') }, hal: { file: 'hal-0.1.1.tar.gz', sha256: sum('y') } }, checks: { testsPassed: true, dirty: false, provenanceClean: true, provenanceProblems: [] } });   // 승인(아래)은 세 검사가 모두 깨끗해야 한다(0.2.0)
  const enq = (module, version = '0.1.1', ids = [a1.serverId, a2.serverId]) => h.d.jobs.enqueue({ kind: 'module.deploy', subject: { module, version, serverIds: ids }, createdBy: 'op', actorType: 'operator' }).job;
  let j = enq('portal'); await h.d.jobs.drain(); eq(h.d.jobs.get(j.id).error_code, 'release_not_approved');
  core.approveRelease(h.d, '0.1.1', 'op'); core.setBootstrap(h.d, { hal_running_version: '0.1.0' });
  j = enq('portal'); await h.d.jobs.drain(); eq(h.d.jobs.get(j.id).error_code, 'hal_first', 'HAL 이 먼저');
  // [0.3.17] HAL 자체는 배포 대상이 아니다(먼저 설치하는 도구) — HAL 은 별도 절차로 갱신되고 실행 버전은 기동 때 기록된다
  j = enq('hal', '0.1.1', [a1.serverId]); await h.d.jobs.drain(); eq(h.d.jobs.get(j.id).error_code, 'hal_not_deployable'); core.setBootstrap(h.d, { hal_running_version: '0.1.1' }); let pp;
  // 포털을 두 서버에: w1 성공, w2 실패 → partial
  j = enq('portal'); pp = pump(h, j.id); await serveOne(a1, () => ({ status: 'done', result: { actualVersion: '0.1.1', health: { ok: true }, active: true } })); await serveOne(a2, () => ({ status: 'failed', error: 'health failed', errorCode: 'health_failed' })); await pp;
  const parent = h.d.jobs.get(j.id); eq(parent.status, 'failed', JSON.stringify(parent.error)); eq(parent.error_code, 'partial_failure'); const sum2 = h.d.jobs.childrenSummary(j.id); eq(sum2.overall, 'partial'); eq(sum2.total, 2);
  eq(h.d.store.get('SELECT health, actual_version FROM module_targets WHERE module=? AND server_id=?', 'portal', a1.serverId).actual_version, '0.1.1');
  // 버전 불일치(Agent 가 다른 버전 보고) → 실패
  j = enq('portal', '0.1.1', [a2.serverId]); pp = pump(h, j.id); await serveOne(a2, () => ({ status: 'done', result: { actualVersion: '0.1.0', health: { ok: true } } })); await pp; eq(h.d.jobs.childrenSummary(j.id).children[0].status, 'failed');
  ok(h.d.store.get('SELECT 1 AS x FROM module_targets WHERE module=? AND server_id=? AND target_version<>actual_version', 'portal', a2.serverId));
  // 되돌리기: 이전 버전으로
  j = h.d.jobs.enqueue({ kind: 'module.rollback', subject: { module: 'portal', toVersion: '0.1.0', serverIds: [a1.serverId] }, createdBy: 'op', actorType: 'operator' }).job; pp = pump(h, j.id); const rb = await serveOne(a1, () => ({ status: 'done', result: { actualVersion: '0.1.0', health: { ok: true }, active: true } })); await pp; eq(rb.kind, 'module.rollback'); eq(h.d.jobs.get(j.id).status, 'done', JSON.stringify(h.d.jobs.get(j.id).error));
});
await test('외부 Agent: 허용 작업만, 다른 사이트 거부, 상태 조회', async () => {
  const h = makeHal(); bootstrapDone(h.d); await registerAgent(h, { role: 'control' }); const ext = await registerAgent(h, { kind: 'external', siteId: 's9', groupId: 'g1', name: '고객서버' });
  const e = (() => { try { core.dispatchAgentJob(h.d, { halJobId: 'x', agentId: ext.agentId, kind: 'backup.create', payload: {} }); } catch (x) { return x; } })(); eq(e.code, 'kind_not_allowed_external');
  const e2 = (() => { try { core.dispatchAgentJob(h.d, { halJobId: 'x', agentId: ext.agentId, kind: 'site.deploy', payload: { siteId: 'other' } }); } catch (x) { return x; } })(); eq(e2.code, 'site_mismatch');
  ok(core.dispatchAgentJob(h.d, { halJobId: 'x', agentId: ext.agentId, kind: 'site.deploy', payload: { siteId: 's9' } }));
  const st = core.externalAgentStatus(h.d, ext.agentId); eq(st.status, 'connected'); eq(st.siteId, 's9');
  h.d.vault.set('service:portal', portalSecret); const r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/external-agents/token', { siteId: 's10', groupId: 'g1' }); ok(r.json.token); ok(r.json.installCommand.includes('--external --site s10'));
});
await test('운영자: 최초 설정 토큰, 로그인·잠금, 네트워크 관문·접근권, 재인증 필요, CSRF, MFA', async () => {
  const h = makeHal(); writeFileSync(join(h.dataDir, 'setup.token'), 'setup-example-token');
  let r = await invoke(h.app, 'POST', '/ops/api/setup', { body: { setupToken: 'wrong', login: 'ozops', password: 'password-example-1', displayName: '오즈' } }); eq(r.status, 403);
  r = await invoke(h.app, 'POST', '/ops/api/setup', { body: { setupToken: 'setup-example-token', login: 'ozops', password: 'password-example-1', displayName: '오즈' } }); eq(r.status, 200, JSON.stringify(r.json)); ok(!existsSync(join(h.dataDir, 'setup.token')));
  r = await invoke(h.app, 'POST', '/ops/api/setup', { body: { setupToken: 'setup-example-token', login: 'ozops2', password: 'password-example-1' } }); eq(r.status, 403, '두 번째 설정 거부');
  // 허용 네트워크 밖: 로그인도 403
  r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' }, ip: '203.0.113.5' }); eq(r.status, 403); eq(r.json.error, 'access_denied');
  for (let i = 0; i < 5; i++) { r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'bad' } }); } eq(r.status, 400); r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' } }); eq(r.json.error, 'locked');
  h.d.store.run('UPDATE ops_accounts SET failed_logins=0, lock_until=NULL');
  r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' } }); eq(r.status, 200); const cookie = r.cookies.aw_ops; const csrf = r.json.csrf; ok(cookie && csrf);
  const asOp = (method, path, body, extra = {}) => invoke(h.app, method, path, { body, cookies: { aw_ops: cookie }, headers: { 'x-csrf': csrf, ...extra } });
  r = await asOp('GET', '/ops/api/me'); eq(r.json.operator.login, 'ozops'); ok(r.json.recentlyAuthed);
  r = await invoke(h.app, 'POST', '/ops/api/servers', { body: { name: 'x', role: 'web' }, cookies: { aw_ops: cookie } }); eq(r.status, 403); eq(r.json.error, 'csrf');
  r = await asOp('POST', '/ops/api/servers', { name: 'x', role: 'web' }); eq(r.status, 200); const sid = r.json.server.id;
  h.d.store.run('UPDATE sessions SET reauth_at=?', new Date(Date.now() - 11 * 60000).toISOString());
  r = await asOp('PATCH', `/ops/api/servers/${sid}`, { status: 'isolated' }); eq(r.json.error, 'reauth_required'); r = await asOp('POST', '/ops/api/reauth', { password: 'password-example-1' }); eq(r.status, 200); r = await asOp('PATCH', `/ops/api/servers/${sid}`, { status: 'isolated' }); eq(r.status, 200);
  // 접근권: 밖에서 URL 로 기기 묶기 → 쿠키로 로그인 화면 허용
  r = await asOp('POST', '/ops/api/access-grants', { hours: 2 }); const url = new URL(r.json.grant.url); r = await invoke(h.app, 'GET', '/ops/access?code=' + url.searchParams.get('code'), { ip: '203.0.113.5', headers: { 'user-agent': 'UA-1' } }); eq(r.status, 302); const gc = r.cookies.aw_ops_grant; ok(gc);
  r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' }, ip: '203.0.113.5', cookies: { aw_ops_grant: gc }, headers: { 'user-agent': 'UA-1' } }); eq(r.status, 200); eq(r.json.via, 'grant');
  r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' }, ip: '203.0.113.5', cookies: { aw_ops_grant: gc }, headers: { 'user-agent': 'UA-2' } }); eq(r.status, 403, '다른 기기(UA) 거부');
  // MFA
  r = await asOp('POST', '/ops/api/mfa/begin'); const secret = r.json.secret; r = await asOp('POST', '/ops/api/mfa/confirm', { code: '000000' }); eq(r.json.error, 'bad_totp'); r = await asOp('POST', '/ops/api/mfa/confirm', { code: totpCode(secret) }); eq(r.json.recoveryCodes.length, 8); const rc = r.json.recoveryCodes[0];
  r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' } }); ok(r.json.mfaRequired); r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1', totp: rc } }); eq(r.status, 200); r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1', totp: rc } }); eq(r.status, 400, '복구 코드는 1회');
  ok(ipAllowed('192.168.20.5', ['192.168.0.0/16']) && !ipAllowed('10.1.1.1', ['192.168.0.0/16']) && ipAllowed('::ffff:127.0.0.1', ['127.0.0.1/32']));
});
await test('긴급 복구: 정상 상태 선언 거부, 3단계 서로 다른 확인, 승인 부족·만료 자격·허용 범위 밖·일반 배포 거부, 정상 복구·조정·종료', async () => {
  const h = makeHal(); let e = (() => { try { emergency.declare(h.d, { incidentKey: 'INC-1', reason: 'HAL DB 손상', scope: ['hal', 'hal_data'], by: 'op' }); } catch (x) { return x; } })(); eq(e.code, 'hal_healthy');
  h.d.store.run('INSERT INTO incidents(id, key, severity, title, started_at, updated_at) VALUES(?,?,?,?,?,?)', 'inc1', 'INC-1', 'critical', 'HAL 장애', new Date().toISOString(), new Date().toISOString());
  const s = emergency.declare(h.d, { incidentKey: 'INC-1', reason: 'HAL DB 손상으로 작업 엔진 사용 불가', scope: ['hal', 'hal_data'], by: 'op' }); eq(s.status, 'declared');
  e = (() => { try { emergency.issueCredential(h.d, s.id, 'op'); } catch (x) { return x; } })(); eq(e.code, 'approvals_insufficient');
  e = (() => { try { emergency.approve(h.d, s.id, { step: 2, by: 'op', identifiers: ['hal'] }); } catch (x) { return x; } })(); eq(e.code, 'wrong_step');
  emergency.approve(h.d, s.id, { step: 1, by: 'op', impactText: 'HAL 제어 평면 중단, 고객 서비스는 유지' });
  e = (() => { try { emergency.approve(h.d, s.id, { step: 2, by: 'op', identifiers: ['hal'] }); } catch (x) { return x; } })(); eq(e.code, 'identifiers_mismatch');
  emergency.approve(h.d, s.id, { step: 2, by: 'op', identifiers: ['hal_data', 'hal'] });
  e = (() => { try { emergency.approve(h.d, s.id, { step: 3, by: 'op', finalConfirm: '아무거나', revertible: true }); } catch (x) { return x; } })(); eq(e.code, 'final_confirm_mismatch');
  const s3 = emergency.approve(h.d, s.id, { step: 3, by: 'op', finalConfirm: '복구 실행 INC-1', revertible: true }); eq(s3.status, 'approved'); eq(s3.approvals.length, 3); ok(s3.approvals.every((a) => a.at));
  const cred = emergency.issueCredential(h.d, s.id, 'op'); ok(cred.credential);
  let actions = []; h.d.emergencyRunner = async (action, target) => { actions.push(action); return { ok: true, action }; };
  e = await rejects(() => emergency.perform(h.d, s.id, { credential: cred.credential, action: 'module.deploy', by: 'op' }), 'action_forbidden');
  e = await rejects(() => emergency.perform(h.d, s.id, { credential: cred.credential, action: 'cert.reissue_internal', by: 'op' }), 'out_of_scope');
  e = await rejects(() => emergency.perform(h.d, s.id, { credential: 'bad', action: 'hal.restart', by: 'op' }), 'bad_credential');
  ok((await emergency.perform(h.d, s.id, { credential: cred.credential, action: 'hal.restart', by: 'op' })).ok); eq(actions, ['hal.restart']);
  const rejectedLogged = emergency.getSession(h.d, s.id).actions.filter((a) => a.result === 'rejected').length; eq(rejectedLogged, 2, '거부도 기록');
  h.d.store.run('UPDATE emergency_sessions SET credential_expires_at=? WHERE id=?', new Date(Date.now() - 1000).toISOString(), s.id);
  await rejects(() => emergency.perform(h.d, s.id, { credential: cred.credential, action: 'hal.restart', by: 'op' }), 'credential_expired');
  h.d.store.run('UPDATE emergency_sessions SET credential_expires_at=? WHERE id=?', new Date(Date.now() + 60000).toISOString(), s.id);
  const v = emergency.verifyAndRevoke(h.d, s.id, { by: 'op', verify: { halRestarted: true } }); ok(v.credential_revoked_at); await rejects(() => emergency.perform(h.d, s.id, { credential: cred.credential, action: 'hal.restart', by: 'op' }), 'credential_revoked');
  const rc = emergency.reconcile(h.d, s.id, 'op'); ok(Array.isArray(rc.drift)); eq(emergency.close(h.d, s.id, 'op').status, 'closed');
  ok(h.d.store.all('SELECT * FROM audit_log WHERE action LIKE \'emergency.%\'').every((a) => a.immutable === 1));
});
await test('재해복구: 설정 검증·복구 사본 생성·검증·보존, 복원 승인·기존 제어 중지 확인, 완료 확인 항목, 훈련 → 준비 상태', async () => {
  const h = makeHal(); h.d.store.exec('CREATE TABLE t(x)'); eq(dr.readiness(h.d).ready, false); eq(dr.readiness(h.d).missingConfig.length, 4);
  const ext = join(tmpDir(), 'ext'); dr.setConfig(h.d, { rto_minutes: 60, rpo_minutes: 30, snapshot_interval_minutes: 60, retention_count: 2, external_store: ext }, 'op');
  // 메모리 DB 는 VACUUM INTO 가 가능하다
  const s1 = dr.snapshot(h.d, { kind: 'manual', by: 'op' }); eq(s1.verify_state, 'ok'); ok(existsSync(s1.location)); ok(s1.external_copied_at, '외부 보관'); ok(s1.sha256);
  const list = execFileSync('tar', ['-tzf', s1.location]).toString(); ok(list.includes('hal.sqlite') && list.includes('meta.json') && !list.includes('vault.key'));
  dr.snapshot(h.d); dr.snapshot(h.d); eq(dr.listSnapshots(h.d).filter((s) => !s.deleted_at).length, 2, '보존 개수');
  const latest = dr.listSnapshots(h.d).find((x) => !x.deleted_at); const rq = dr.requestRestore(h.d, { snapshotId: latest.id, by: 'op' }); let e = (() => { try { dr.startRestore(h.d, rq.id, { by: 'op', oldControlStopped: true, confirmText: '기존 제어 위치를 중지했습니다' }); } catch (x) { return x; } })(); eq(e.code, 'restore_not_approved');
  dr.approveRestore(h.d, rq.id, 'op2'); e = (() => { try { dr.startRestore(h.d, rq.id, { by: 'op', oldControlStopped: false, confirmText: '' }); } catch (x) { return x; } })(); eq(e.code, 'old_control_not_confirmed');
  const st = dr.startRestore(h.d, rq.id, { by: 'op', oldControlStopped: true, confirmText: '기존 제어 위치를 중지했습니다' }); ok(st.command.includes('hal-restore.sh'));
  e = (() => { try { dr.finishRestore(h.d, rq.id, { by: 'op', checks: { operatorLogin: true } }); } catch (x) { return x; } })(); eq(e.code, 'checks_incomplete');
  eq(dr.finishRestore(h.d, rq.id, { by: 'op', checks: { operatorLogin: true, halControl: true, https: true, customerService: true } }).status, 'completed');
  const dl = dr.startDrill(h.d, { by: 'op', snapshotId: latest.id }); eq(dr.finishDrill(h.d, dl.id, { by: 'op', steps: { hal: true, operatorLogin: true, vault: true, certBase: true, httpsCustomer: false }, measuredRtoMinutes: 20, measuredRpoMinutes: 10 }).result, 'failed');
  const dl2 = dr.startDrill(h.d, { by: 'op' }); eq(dr.finishDrill(h.d, dl2.id, { by: 'op', steps: { hal: true, operatorLogin: true, vault: true, certBase: true, httpsCustomer: true }, measuredRtoMinutes: 20, measuredRpoMinutes: 10 }).result, 'success'); ok(dr.readiness(h.d).ready);
});
await test('공개 상태·운영 현황·전송 업로드·artifact 내려받기', async () => {
  const h = makeHal(); bootstrapDone(h.d); const a = await registerAgent(h);
  let r = await invoke(h.app, 'GET', '/api/status'); ok(r.json.components.length >= 3); ok(!JSON.stringify(r.json).includes(a.serverId), '내부 id 없음');
  r = await invoke(h.app, 'GET', '/status'); ok(r.text.includes('Ariweb 서비스 상태'));
  const body = Buffer.from('hello-transfer'); const sum = createHash('sha256').update(body).digest('hex');
  const low = (o) => Object.fromEntries(Object.entries(o).map(([k, v]) => [k.toLowerCase(), v])); const hdr = low(signHeaders(`agent:${a.agentId}`, a.secret, 'POST', '/agent/v1/transfers/tr_t1', '')); r = await invoke(h.app, 'POST', '/agent/v1/transfers/tr_t1', { body: body.toString(), headers: { ...hdr, 'content-type': 'application/octet-stream', 'x-transfer-sha256': sum } }); eq(r.status, 200, JSON.stringify(r.json)); eq(r.json.sha256, sum);
  ok(core.getTransfer(h.d, 'tr_t1'));
  r = await invoke(h.app, 'POST', '/agent/v1/transfers/tr_t2', { body: 'x', headers: { ...low(signHeaders(`agent:${a.agentId}`, a.secret, 'POST', '/agent/v1/transfers/tr_t2', '')), 'content-type': 'application/octet-stream', 'x-transfer-sha256': 'bad' } }); eq(r.status, 400);
  const art = join(h.dataDir, 'artifacts', '0.1.1'); mkdirSync(art, { recursive: true }); writeFileSync(join(art, 'agent-0.1.1.tar.gz'), 'tgz'); core.registerRelease(h.d, { version: '0.1.1', mainCommit: 'abc', builtAt: 'now', artifacts: { agent: { file: 'agent-0.1.1.tar.gz', sha256: createHash('sha256').update('tgz').digest('hex') } } });
  r = await call(h.app, `agent:${a.agentId}`, a.secret, 'GET', '/agent/v1/releases/0.1.1/artifact?module=agent'); eq(r.status, 200); eq(r.text, 'tgz');
  const t = core.issueRegistrationToken(h.d, { kind: 'managed' }); r = await invoke(h.app, 'GET', '/agent/v1/bootstrap-artifact?module=agent&version=0.1.1', { headers: { 'x-registration-token': t.token } }); eq(r.status, 200); eq(r.text, 'tgz');
});
await test('서버 이전 작업: 단계 순서와 verify_new 실패 시 전환하지 않음', async () => {
  const h = makeHal(); bootstrapDone(h.d); const a1 = await registerAgent(h, { name: 'from' }); const a2 = await registerAgent(h, { name: 'to' });
  const j = h.d.jobs.enqueue({ kind: 'site.relocate', subject: { payload: { siteId: 's1', fromServerId: a1.serverId, toServerId: a2.serverId, kind: 'static' } }, subjectType: 'site', subjectId: 's1' }).job;
  const seen = []; const pp = pump(h, j.id, 10000); const serve = async (a, fn) => { const job = await serveOne(a, fn); if (job) seen.push(job.kind); return job; };
  await serve(a2, () => ({ status: 'done', result: { prepared: true } })); await serve(a1, () => ({ status: 'done', result: { transferId: 'tr_rl1', checksum: 'c1' } })); await serve(a2, () => ({ status: 'done', result: { ok: true } })); await serve(a1, () => ({ status: 'done', result: { transferId: 'tr_rl2', checksum: 'c2' } })); await serve(a2, () => ({ status: 'done', result: { ok: true, checksum: 'c2' } })); await serve(a2, () => ({ status: 'done', result: { ok: false, reason: '체크섬 다름' } }));
  await pp;
  eq(seen, ['site.prepare', 'site.export', 'site.import', 'site.export', 'site.import', 'site.stage_check']); eq(h.d.jobs.get(j.id).status, 'failed'); eq(h.d.jobs.get(j.id).error_code, 'verify_new_failed'); ok(!h.d.store.get('SELECT 1 AS x FROM agent_jobs WHERE hal_job_id=? AND kind=\'site.activate\'', j.id), '전환하지 않음');
});
done();