OZ-Group/AriWeb-Suite
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HAL 제어 평면(릴리스·서버·Agent·작업·모듈 배포·운영자 계정·긴급 복구·재해복구·공개 상태), Node Agent, 배포 스크립트·유닛·nginx 템플릿, 릴리스 도구, 포털 HAL 클라이언트 (§28·§29·§32~§34)

Browse Source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
zzabisagent-lab
2026-10-26 11:58:15 +00:00
co-authored by Claude Fable 5.1
parent 403d0fb845
commit 6b5a014041
49 changed files with 2584 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/agent.test.mjs
+116
View File
@@ -0,0 +1,116 @@
/** test/agent.test.mjs — Node Agent 실행기 회귀 테스트. 임시 루트 + 명령 stub(ARIWEB_AGENT_STUB_CMDS=1)으로 실제 시스템을 건드리지 않는다. tar 는 실제로 실행한다. */
process.env.ARIWEB_AGENT_STUB_CMDS = '1';
import { test, eq, ok, tmpDir, done } from '../lib/test.mjs';
import { createSys } from '../agent/sys.mjs';
import { createRunner } from '../agent/runner.mjs';
import { KINDS } from '../agent/jobs/index.mjs';
import { ingestLog } from '../agent/jobs/site.mjs';
import { sample } from '../agent/metrics.mjs';
import { mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync, readlinkSync, copyFileSync, statSync, appendFileSync } from 'node:fs';
import { join } from 'node:path';
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
/** 가짜 HAL 클라이언트: 전송은 메모리(파일 경로) 저장, artifact 는 미리 둔 파일 */
function fakeClient({ artifacts = {} } = {}) {
const transfers = new Map(); const calls = []; const up = join(tmpDir(), 'uploads'); mkdirSync(up, { recursive: true });
return { calls, transfers,
async request(method, path, body) { calls.push({ method, path, body }); return { ok: true }; },
async download(path, dest, expected) { const src = artifacts[path]; if (!src) { const e = new Error('404'); e.code = 'http_404'; throw e; } copyFileSync(src, dest); const sum = createHash('sha256').update(readFileSync(dest)).digest('hex'); if (expected && expected !== sum) throw Object.assign(new Error('checksum'), { code: 'checksum_mismatch' }); return { sha256: sum, size: statSync(dest).size }; },
async upload(id, file) { const copy = join(up, id + '.bin'); copyFileSync(file, copy); const sum = createHash('sha256').update(readFileSync(copy)).digest('hex'); this.transfers.set(id, { path: copy, sha256: sum }); return { id, sha256: sum, size: statSync(copy).size }; },
async downloadTransfer(id, dest, expected) { const t = this.transfers.get(id); if (!t) throw Object.assign(new Error('transfer missing'), { code: 'transfer_not_found' }); copyFileSync(t.path, dest); if (expected && String(expected).replace(/^sha256:/, '') !== t.sha256) throw Object.assign(new Error('checksum'), { code: 'checksum_mismatch' }); return { sha256: t.sha256, size: statSync(dest).size }; },
clockSkewMs: 0 };
}
function env(over = {}) {
const base = tmpDir(); const roots = { sites: join(base, 'sites'), srv: join(base, 'srv'), nginx: join(base, 'nginx'), backups: join(base, 'backups'), logs: join(base, 'logs'), acme: join(base, 'acme'), letsencrypt: join(base, 'le'), wildcardCert: join(base, 'le', 'wild'), platformDomain: 'ariweb.net', maintPage: '' };
for (const r of Object.values(roots)) if (r) mkdirSync(r, { recursive: true }); mkdirSync(join(roots.nginx, 'sites-available'), { recursive: true }); mkdirSync(join(roots.nginx, 'sites-enabled'), { recursive: true });
const sys = createSys({ stub: true }); const client = fakeClient(over); const dataDir = join(base, 'agent-data'); mkdirSync(dataDir, { recursive: true });
const runner = createRunner({ kinds: KINDS, roots, sys, client, dataDir }); let n = 0;
return { base, roots, sys, client, dataDir, runner, run: (kind, payload, id = `aj_${++n}`) => runner.execute({ id, kind, payload }) };
}
/** 사이트 번들 디렉터리 만들기 */
function bundle(base, name, files = { 'index.html': '<h1>hello marker-1</h1>' }) { const d = join(base, 'bundles', name); mkdirSync(d, { recursive: true }); for (const [f, c] of Object.entries(files)) { mkdirSync(join(d, f, '..'), { recursive: true }); writeFileSync(join(d, f), c); } return d; }
await test('site.deploy: 레이아웃(releases/current)·버전 기록·vhost·내부 확인·previous, rollback, 실패 시 이전 버전 유지와 leftovers, remove', async () => {
const e = env(); const b1 = bundle(e.base, 'b1'); mkdirSync(e.roots.wildcardCert, { recursive: true }); writeFileSync(join(e.roots.wildcardCert, 'fullchain.pem'), 'x');
let r = await e.run('site.deploy', { siteId: 'site1', versionRef: 'v1', bundlePath: b1, domains: ['site1.ariweb.net'], primaryDomain: 'site1.ariweb.net', expectMarker: 'marker-1' }); eq(r.status, 'done', JSON.stringify(r));
const sd = join(e.roots.sites, 'site1'); const cur = readlinkSync(join(sd, 'current')); ok(cur.includes('/releases/')); ok(existsSync(join(cur, 'index.html'))); ok(JSON.parse(readFileSync(join(cur, '.ariweb-version.json'), 'utf8')).versionRef === 'v1');
ok(r.result.https, '와일드카드 인증서 적용'); eq(r.result.certs['site1.ariweb.net'].source, 'wildcard'); ok(existsSync(join(e.roots.nginx, 'sites-enabled', 'aw-site1.conf')));
const conf = readFileSync(join(e.roots.nginx, 'sites-available', 'aw-site1.conf'), 'utf8'); ok(conf.includes('server_name site1.ariweb.net;') && conf.includes('listen 443 ssl') && conf.includes('awstats') && conf.includes('.maintenance'));
ok(e.sys.calls.some((c) => c.args.includes('nginx') && c.args.includes('-t')), 'nginx -t 실행'); eq(r.result.previousVersion, null);
// 두 번째 배포 → previous, rollback
const b2 = bundle(e.base, 'b2', { 'index.html': '<h1>v2 marker-2</h1>' }); r = await e.run('site.deploy', { siteId: 'site1', versionRef: 'v2', bundlePath: b2, domains: ['site1.ariweb.net'], expectMarker: 'marker-2' }); eq(r.status, 'done'); ok(r.result.previousVersion); ok(readFileSync(join(sd, 'current', 'index.html'), 'utf8').includes('v2'));
r = await e.run('site.rollback', { siteId: 'site1', domains: ['site1.ariweb.net'] }); eq(r.status, 'done', JSON.stringify(r)); ok(readFileSync(join(sd, 'current', 'index.html'), 'utf8').includes('marker-1'));
// 실패: 마커 없음 → 이전 버전 유지, leftovers 보고
const b3 = bundle(e.base, 'b3', { 'index.html': 'broken' }); r = await e.run('site.deploy', { siteId: 'site1', versionRef: 'v3', bundlePath: b3, domains: ['site1.ariweb.net'], expectMarker: 'marker-9' }); eq(r.status, 'failed'); eq(r.errorCode, 'internal_check_failed'); ok(r.result.previousKept); ok(r.result.leftovers.length >= 1); ok(readFileSync(join(sd, 'current', 'index.html'), 'utf8').includes('marker-1'), '이전 버전 유지');
r = await e.run('site.deploy', { siteId: '../etc', bundlePath: b1 }); eq(r.status, 'failed'); eq(r.errorCode, 'bad_site_id');
r = await e.run('site.remove', { siteId: 'site1', keepData: true }); eq(r.status, 'done'); ok(!existsSync(join(sd, 'current'))); ok(existsSync(join(sd, 'data'))); ok(!existsSync(join(e.roots.nginx, 'sites-enabled', 'aw-site1.conf')));
});
await test('실행기: 단계 저장과 재시작 뒤 재개(끝난 단계 건너뜀), needs_user 보고', async () => {
const e = env(); let aRuns = 0; let fail = true;
e.runner.kinds['test.resume'] = { async run(ctx) { await ctx.step('a', () => { aRuns++; return 1; }); if (fail) throw new Error('crash'); await ctx.step('b', () => 2); return { ok: true }; } };
let r = await e.runner.execute({ id: 'aj_r1', kind: 'test.resume', payload: {} }); eq(r.status, 'failed'); eq(aRuns, 1);
// 실패한 작업 상태 파일은 지워지지만, 재시작 시나리오(실행 중 죽음)는 상태 파일이 남는다 → 흉내: 상태 파일을 다시 써 둔다
writeFileSync(join(e.dataDir, 'jobs', 'aj_r1.json'), JSON.stringify({ steps: [{ id: 'a', title: 'a', state: 'done', result: 1 }], log: [] })); fail = false;
r = await e.runner.execute({ id: 'aj_r1', kind: 'test.resume', payload: {} }); eq(r.status, 'done'); eq(aRuns, 1, '끝난 단계는 다시 실행하지 않음'); eq(r.steps.map((s) => s.id), ['a', 'b']);
e.runner.kinds['test.ask'] = { async run(ctx) { ctx.needsUser('sudo 가 없습니다', 'sudoers 설치'); } }; r = await e.runner.execute({ id: 'aj_r2', kind: 'test.ask', payload: {} }); eq(r.status, 'needs_user'); ok(r.userMessage.includes('sudoers'));
ok(e.client.calls.some((c) => c.path.endsWith('/result')), '결과 보고');
});
await test('site.usage: awstats 로그 증분 파싱·일별 집계·오류율·p95·디렉터리 크기', async () => {
const e = env(); const b = bundle(e.base, 'u', { 'index.html': 'x'.repeat(1000) }); await e.run('site.deploy', { siteId: 'u1', bundlePath: b }); const log = join(e.roots.logs, 'aw-u1.access.log'); const today = new Date().toISOString().slice(0, 10);
writeFileSync(log, `${today}T10:00:00+00:00|u1.preview.ariweb.net|300|5000|200|0.050|/\n${today}T10:00:01+00:00|u1.preview.ariweb.net|200|1000|500|2.000|/x\n`);
let r = await e.run('site.usage', { siteIds: ['u1'] }); const u = r.result.sites.u1; eq(u.req24h, 2); eq(u.outBytesDay, 6000); eq(u.inBytesDay, 500); eq(u.errorRate, 50); eq(u.p95ms, 1000); ok(u.diskBytes >= 1000); eq(u.series7d.length, 7);
appendFileSync(log, `${today}T11:00:00+00:00|u1.preview.ariweb.net|100|100|200|0.010|/\n`); r = await e.run('site.usage', { siteIds: ['u1'] }); eq(r.result.sites.u1.req24h, 3, '증분만 더함'); eq(r.result.sites.u1.outBytesDay, 6100);
const st = { days: {}, offsets: {} }; ingestLog({ roots: e.roots }, 'u1', log, st); eq(Object.values(st.days)[0].req, 3);
const m = sample(); ok(m.mem && m.mem.total > 0); ok(typeof m.load1 === 'number');
});
await test('backup: 생성(파일·데이터·DB·설정)·검증·원격 저장(전송)·복원(보호 시점·격리 확인)·삭제', async () => {
const e = env(); const b = bundle(e.base, 'k', { 'index.html': '<p>marker-k</p>', 'ariweb.json': '{"runtime":"static"}' }); await e.run('site.deploy', { siteId: 'k1', bundlePath: b, expectMarker: 'marker-k' });
const sd = join(e.roots.sites, 'k1'); mkdirSync(join(sd, 'data', 'uploads'), { recursive: true }); writeFileSync(join(sd, 'data', 'uploads', 'a.txt'), 'upload-a'); mkdirSync(join(sd, 'data', 'db'), { recursive: true });
const { DatabaseSync } = await import('node:sqlite'); const db = new DatabaseSync(join(sd, 'data', 'db', 'site.sqlite')); db.exec('CREATE TABLE t(x); INSERT INTO t VALUES(42)'); db.close();
let r = await e.run('backup.create', { siteId: 'k1', scope: {}, uploadToHal: true, transferId: 'tr_bk_1' }); eq(r.status, 'done', JSON.stringify(r)); ok(r.result.checksum); eq(r.result.scope.db, ['site.sqlite']); ok(r.result.transferId === 'tr_bk_1'); const list = execFileSync('tar', ['-tzf', e.client.transfers.get('tr_bk_1').path]).toString(); ok(list.includes('./meta.json') && list.includes('./files.tar') && list.includes('./data.tar') && list.includes('./db/site.sqlite') && list.includes('./config/vhost.conf'), list);
// 저장 서버 역할: 전송에서 받아 보관·검증
const e2 = env(); e2.client.transfers.set('tr_bk_1', e.client.transfers.get('tr_bk_1')); r = await e2.run('backup.store', { siteId: 'k1', backupId: 'bk-x', transferId: 'tr_bk_1', sha256: e.client.transfers.get('tr_bk_1').sha256 }); eq(r.status, 'done', JSON.stringify(r)); const loc = r.result.path; ok(existsSync(loc));
r = await e2.run('backup.verify', { location: loc, checksum: r.result.checksum }); ok(r.result.ok); r = await e2.run('backup.verify', { location: loc, checksum: 'deadbeef' }); eq(r.result.ok, false); eq(r.result.reason, 'checksum_mismatch');
r = await e2.run('backup.fetch', { location: loc, transferId: 'tr_rs_1' }); eq(r.status, 'done'); ok(e2.client.transfers.get('tr_rs_1'));
// 사이트를 망가뜨린 뒤 복원
writeFileSync(join(sd, 'current', 'index.html'), 'broken'); writeFileSync(join(sd, 'data', 'uploads', 'a.txt'), 'changed'); e.client.transfers.set('tr_rs_1', e2.client.transfers.get('tr_rs_1'));
r = await e.run('backup.restore', { siteId: 'k1', transferId: 'tr_rs_1', checksum: e2.client.transfers.get('tr_rs_1').sha256, protectPoint: true, isolatedVerify: true }); eq(r.status, 'done', JSON.stringify(r)); ok(r.result.protectPointLocation && existsSync(r.result.protectPointLocation)); ok(r.result.isolatedVerify);
ok(readFileSync(join(sd, 'current', 'index.html'), 'utf8').includes('marker-k')); eq(readFileSync(join(sd, 'data', 'uploads', 'a.txt'), 'utf8'), 'upload-a'); ok(readdirSync(sd).some((x) => x.startsWith('data.pre-restore-')), '이전 데이터 보존'); const db2 = new DatabaseSync(join(sd, 'data', 'db', 'site.sqlite')); eq(db2.prepare('SELECT x FROM t').get().x, 42); db2.close();
r = await e.run('backup.restore', { siteId: 'other', location: loc, protectPoint: false }); eq(r.status, 'failed'); eq(r.errorCode, 'backup_site_mismatch');
r = await e2.run('backup.delete', { location: loc }); ok(r.result.deleted); ok(!existsSync(loc)); r = await e2.run('backup.delete', { location: '/etc/passwd' }); eq(r.result.deleted, false, '저장소 밖은 지우지 않음');
});
await test('relocate 보조: export → import → stage_check → activate(새 서버) → deactivate(이전 서버 사본 보존), freeze', async () => {
const src = env(); const dst = env(); dst.client.transfers = src.client.transfers;
const b = bundle(src.base, 'r', { 'index.html': 'reloc marker-r' }); await src.run('site.deploy', { siteId: 'r1', bundlePath: b, domains: ['r1.example.com'] }); mkdirSync(join(src.roots.sites, 'r1', 'data'), { recursive: true }); writeFileSync(join(src.roots.sites, 'r1', 'data', 'd.txt'), 'data-r');
let r = await dst.run('site.prepare', { siteId: 'r1' }); eq(r.status, 'done');
r = await src.run('site.freeze', { siteId: 'r1', freeze: true }); ok(existsSync(join(src.roots.sites, 'r1', '.frozen')));
r = await src.run('site.export', { siteId: 'r1', transferId: 'tr_rl1', includeData: true }); eq(r.status, 'done', JSON.stringify(r)); const ck = r.result.checksum;
r = await dst.run('site.import', { siteId: 'r1', transferId: 'tr_rl1', checksum: ck, stage: 'final' }); eq(r.status, 'done', JSON.stringify(r)); ok(r.result.ok);
r = await dst.run('site.stage_check', { siteId: 'r1', expectedChecksum: 'wrong' }); eq(r.result.ok, false); r = await dst.run('site.stage_check', { siteId: 'r1', expectedChecksum: ck }); ok(r.result.ok);
r = await dst.run('site.activate', { siteId: 'r1', domains: ['r1.example.com'], primaryDomain: 'r1.example.com' }); eq(r.status, 'done', JSON.stringify(r)); ok(readFileSync(join(dst.roots.sites, 'r1', 'current', 'index.html'), 'utf8').includes('marker-r')); eq(readFileSync(join(dst.roots.sites, 'r1', 'data', 'd.txt'), 'utf8'), 'data-r'); ok(existsSync(join(dst.roots.nginx, 'sites-enabled', 'aw-r1.conf')));
r = await src.run('site.deactivate', { siteId: 'r1', retainDays: 3 }); eq(r.status, 'done'); ok(!existsSync(join(src.roots.nginx, 'sites-enabled', 'aw-r1.conf'))); ok(existsSync(join(src.roots.sites, 'r1', 'current')), '사본 보존'); ok(!existsSync(join(src.roots.sites, 'r1', '.frozen'))); ok(JSON.parse(readFileSync(join(src.roots.sites, 'r1', '.retained.json'), 'utf8')).until);
});
await test('domain.cert: 와일드카드 재사용 / certbot(stub) 뒤 HTTPS vhost / domain.probe stub, domain.vhost 유지보수', async () => {
const e = env(); mkdirSync(e.roots.wildcardCert, { recursive: true }); writeFileSync(join(e.roots.wildcardCert, 'fullchain.pem'), 'x'); const b = bundle(e.base, 'c'); await e.run('site.deploy', { siteId: 'c1', bundlePath: b });
let r = await e.run('domain.cert', { siteId: 'c1', domains: ['c1.ariweb.net'] }); eq(r.status, 'done'); eq(r.result.source, 'wildcard'); eq(r.result.issued, ['c1.ariweb.net']);
r = await e.run('domain.cert', { siteId: 'c1', domains: ['www.example.com', 'example.com'], email: 'ops@example.test' }); eq(r.status, 'done', JSON.stringify(r)); eq(r.result.source, 'own'); ok(e.sys.calls.some((c) => c.args.includes('certbot') && c.args.includes('--webroot'))); ok(readFileSync(join(e.roots.nginx, 'sites-available', 'aw-c1.conf'), 'utf8').includes('listen 443 ssl'));
r = await e.run('domain.probe', { name: 'x.example.test', _stubAnswer: { a: ['1.2.3.4'], aOk: true, txtOk: true } }); ok(r.result.aOk);
r = await e.run('domain.vhost', { siteId: 'c1', domains: ['example.com'], primaryDomain: 'example.com', maintenance: true, redirectOld: true, oldDomains: ['old.example.com'] }); eq(r.status, 'done'); ok(existsSync(join(e.roots.sites, 'c1', '.maintenance'))); ok(readFileSync(join(e.roots.nginx, 'sites-available', 'aw-c1.conf'), 'utf8').includes('old.example.com'));
});
await test('module.install: artifact 검증·releases/current·유닛(기존 Environment 보존)·사후 절차·실제 버전, rollback, health, 해시 불일치 실패', async () => {
const e = env(); const app = join(e.base, 'bundles', 'portal'); mkdirSync(join(app, 'portal'), { recursive: true }); writeFileSync(join(app, 'package.json'), JSON.stringify({ name: 'x', version: '0.1.1' })); writeFileSync(join(app, 'portal', 'index.mjs'), ''); const tgz = join(e.base, 'portal-0.1.1.tar.gz'); execFileSync('tar', ['-czf', tgz, '-C', app, '.']); const sum = createHash('sha256').update(readFileSync(tgz)).digest('hex');
e.client = fakeClient({ artifacts: { '/art/portal': tgz } }); const runner = createRunner({ kinds: KINDS, roots: e.roots, sys: e.sys, client: e.client, dataDir: e.dataDir }); const appDir = join(e.roots.srv, 'portal');
const desc = { name: 'portal', unit: 'ariweb-portal', unitTemplate: 'ariweb-portal.service.tmpl', user: 'master', health: { url: 'http://127.0.0.1:1/api/health', expect: 'ok' }, version: { file: 'package.json' }, layout: 'releases', postStart: [] };
let r = await runner.execute({ id: 'm1', kind: 'module.install', payload: { module: 'portal', version: '0.1.1', artifactUrl: '/art/portal', sha256: 'bad', descriptor: desc, appDir, unitTemplate: 'WorkingDirectory={{APP_DIR}}/current\n{{ENV_LINES}}\nExecStart={{NODE}} x' } }); eq(r.status, 'failed'); eq(r.errorCode, 'checksum_mismatch');
r = await runner.execute({ id: 'm2', kind: 'module.install', payload: { module: 'portal', version: '0.1.1', artifactUrl: '/art/portal', sha256: sum, descriptor: desc, appDir, unitTemplate: 'WorkingDirectory={{APP_DIR}}/current\n{{ENV_LINES}}\nExecStart={{NODE}} x' } }); eq(r.status, 'done', JSON.stringify(r)); eq(r.result.actualVersion, '0.1.1'); eq(readlinkSync(join(appDir, 'current')), join(appDir, 'releases', '0.1.1')); ok(e.sys.calls.some((c) => c.args.join(' ').includes('systemctl restart ariweb-portal')));
const unitFile = e.sys.calls.find((c) => c.cmd === 'sudo' && c.args.includes('tee')); ok(unitFile || existsSync('/etc/systemd/system/ariweb-portal.service') === false, '유닛 파일 쓰기 시도');
// 두 번째 버전 설치 뒤 rollback
writeFileSync(join(app, 'package.json'), JSON.stringify({ name: 'x', version: '0.1.2' })); const tgz2 = join(e.base, 'portal-0.1.2.tar.gz'); execFileSync('tar', ['-czf', tgz2, '-C', app, '.']); e.client.transfers; const sum2 = createHash('sha256').update(readFileSync(tgz2)).digest('hex'); const runner2 = createRunner({ kinds: KINDS, roots: e.roots, sys: e.sys, client: fakeClient({ artifacts: { '/art/portal2': tgz2 } }), dataDir: e.dataDir });
r = await runner2.execute({ id: 'm3', kind: 'module.install', payload: { module: 'portal', version: '0.1.2', artifactUrl: '/art/portal2', sha256: sum2, descriptor: desc, appDir } }); eq(r.result.actualVersion, '0.1.2'); eq(r.result.previousVersion, '0.1.1');
r = await runner2.execute({ id: 'm4', kind: 'module.rollback', payload: { module: 'portal', version: '0.1.2', toVersion: '0.1.1', descriptor: desc, appDir } }); eq(r.status, 'done', JSON.stringify(r)); eq(r.result.actualVersion, '0.1.1');
r = await runner2.execute({ id: 'm5', kind: 'module.health', payload: { module: 'portal', descriptor: desc, appDir, unit: 'ariweb-portal' } }); eq(r.result.actualVersion, '0.1.1'); ok(r.result.health.ok);
// agent 자기 갱신은 재시작을 결과 보고 뒤로 미룬다
r = await runner2.execute({ id: 'm6', kind: 'module.install', payload: { module: 'agent', version: '0.1.2', artifactUrl: '/art/portal2', sha256: sum2, descriptor: { ...desc, name: 'agent', unit: 'ariweb-agent' }, appDir: join(e.roots.srv, 'agent') } }); eq(r.result.restartAfterReport, 'ariweb-agent');
});
done();
test/hal.test.mjs
+213
View File
@@ -0,0 +1,213 @@
/** test/hal.test.mjs — HAL 제어 평면 회귀 테스트(등록·인증·heartbeat·작업 배정·모듈 배포 부분 실패·부트스트랩 관문·외부 Agent 제한·긴급 복구·재해복구·지표 보존·운영자 접근). */
import { test, eq, ok, rejects, tmpDir, done, sleep } from '../lib/test.mjs';
import { invoke } from '../lib/http.mjs';
import { signHeaders } from '../lib/serviceauth.mjs';
import { createHalDeps } from '../hal/deps.mjs';
import { buildApp } from '../hal/app.mjs';
import * as core from '../hal/core.mjs';
import * as ops from '../hal/ops.mjs';
import * as emergency from '../hal/emergency.mjs';
import * as dr from '../hal/dr.mjs';
import { totpCode } from '../hal/totp.mjs';
import { ipAllowed } from '../hal/cidr.mjs';
import { writeFileSync, mkdirSync, existsSync } from 'node:fs';
import { join } from 'node:path';
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
function makeHal(over = {}) {
const dataDir = join(tmpDir(), 'hal'); mkdirSync(dataDir, { recursive: true });
const config = { dataDir, file: join(dataDir, 'config.json'), port: 0, host: '127.0.0.1', publicUrl: 'https://hal.example.test', portalUrl: 'http://127.0.0.1:1', opsAllowCidrs: ['127.0.0.1/32'], metricsRetentionDays: 90, offlineAfterMs: 90000, transferTtlMs: 3600000, emergencyTtlMinutes: 30, agentPollMs: 20, agentWaitTimeoutMs: 4000, fanoutTimeoutMs: 8000, values: {}, set(p) { Object.assign(this, p); }, ...over };
const d = createHalDeps({ config, dbFile: ':memory:', fetchImpl: async (url) => ({ ok: true, status: 200, text: async () => 'marker-ok', json: async () => ({ ok: true }) }) }); d.jobsRunning = true;
const app = buildApp(d); return { d, app, dataDir };
}
const sig = (keyId, secret, method, path, body) => Object.fromEntries(Object.entries(signHeaders(keyId, secret, method, path, body === undefined ? undefined : JSON.stringify(body))).map(([k, v]) => [k.toLowerCase(), v]));
const call = (app, keyId, secret, method, path, body, extra = {}) => invoke(app, method, path, { body, headers: { ...sig(keyId, secret, method, path, body), ...extra } });
/** 서버 + Agent 등록 한 벌 */
async function registerAgent({ d, app }, { role = 'web', name = '웹1', kind = 'managed', siteId = null, groupId = null } = {}) {
const srv = kind === 'managed' ? core.createServer(d, { name, role }) : null;
const t = core.issueRegistrationToken(d, { kind, serverId: srv?.id, siteId, groupId, label: name });
const r = await invoke(app, 'POST', '/agent/v1/register', { body: { token: t.token, facts: { hostname: name }, version: '0.1.1' } }); ok(r.json.ok, JSON.stringify(r.json));
const a = { agentId: r.json.agentId, secret: r.json.secret, serverId: r.json.serverId, hb: (body = {}) => call(app, `agent:${r.json.agentId}`, r.json.secret, 'POST', '/agent/v1/heartbeat', { version: '0.1.1', metrics: { cpu: 5, mem: { total: 100, used: 50 }, disk: { total: 1000, used: 100 }, load1: 0.1 }, modules: {}, clockSkewMs: 0, ...body }), next: () => call(app, `agent:${r.json.agentId}`, r.json.secret, 'GET', '/agent/v1/jobs/next?wait=0'), result: (id, body) => call(app, `agent:${r.json.agentId}`, r.json.secret, 'POST', `/agent/v1/jobs/${id}/result`, body), progress: (id, body) => call(app, `agent:${r.json.agentId}`, r.json.secret, 'POST', `/agent/v1/jobs/${id}/progress`, body) };
await a.hb(); if (srv) core.approveServer(d, srv.id, 'op'); return a;
}
/** 부트스트랩 완료 상태로 만들기 */
function bootstrapDone(d, version = '0.1.1') { core.claimLive(d, { instanceId: d.instanceId, version }); core.setBootstrap(d, { release_version: version, main_commit: 'abc', artifact_sha256: 'x' }); }
/** Agent 역할을 흉내: 다음 작업을 받아 결과를 보낸다 */
async function actAgent(a, fn) { const n = await a.next(); const job = n.json.job; if (!job) return null; const out = fn ? await fn(job) : { status: 'done', result: { ok: true } }; await a.result(job.id, out); return job; }
const portalSecret = 'portal-secret-example';
/** 작업이 끝날 때까지 tick 을 기다리지 않고 계속 던진다(부모 작업이 자식을 기다리는 동안 자식도 실행되게) */
async function pump(h, jobId, maxMs = 8000) { const ps = []; const t0 = Date.now(); while (Date.now() - t0 < maxMs) { const st = h.d.jobs.get(jobId).status; if (['done', 'failed', 'cancelled', 'needs_user', 'awaiting_external'].includes(st) && !h.d.jobs.list({ status: ['queued', 'running'] }).length) break; ps.push(h.d.jobs.tick().catch(() => {})); await sleep(10); } await Promise.all(ps); return h.d.jobs.get(jobId); }
/** Agent 가 다음 작업이 올 때까지 기다렸다가 처리 */
async function serveOne(a, fn, maxMs = 4000) { const t0 = Date.now(); while (Date.now() - t0 < maxMs) { const n = await a.next(); if (n.json.job) { const out = fn(n.json.job); await a.result(n.json.job.id, out); return n.json.job; } await sleep(10); } return null; }
await test('등록 토큰: 1회만, 만료 거부, 잘못된 토큰 거부', async () => {
const h = makeHal(); const srv = core.createServer(h.d, { name: 'a', role: 'web' }); const t = core.issueRegistrationToken(h.d, { serverId: srv.id });
let r = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: t.token, facts: {}, version: '0.1.1' } }); ok(r.json.ok); ok(r.json.secret && r.json.secret.length > 20);
r = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: t.token } }); eq(r.json.error, 'token_used');
r = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: 'nope' } }); eq(r.json.error, 'bad_token');
const t2 = core.issueRegistrationToken(h.d, { serverId: srv.id, ttlMs: -1 }); r = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: t2.token } }); eq(r.json.error, 'token_expired');
ok(h.d.vault.has(`service:agent:${core.getServer(h.d, srv.id).agent_id}`), '비밀은 보관소에');
});
await test('Agent HMAC 인증: 정상·서명 오류·시계 차이·해지', async () => {
const h = makeHal(); const a = await registerAgent(h);
let r = await a.hb(); eq(r.status, 200, JSON.stringify(r.json));
r = await call(h.app, `agent:${a.agentId}`, 'wrong-secret', 'POST', '/agent/v1/heartbeat', {}); eq(r.status, 401); ok(r.json.message.includes('bad_signature'));
const hdr = sig(`agent:${a.agentId}`, a.secret, 'POST', '/agent/v1/heartbeat', {}); hdr['x-service-ts'] = String(Date.now() - 10 * 60000); r = await invoke(h.app, 'POST', '/agent/v1/heartbeat', { body: {}, headers: hdr }); eq(r.status, 401); ok(r.json.message.includes('clock_skew'));
core.revokeAgent(h.d, a.agentId); r = await a.hb(); eq(r.status, 401);
});
await test('heartbeat → 지표·용량·모듈 버전 저장, 무응답 → offline, 지표 90일 보존', async () => {
const h = makeHal({ offlineAfterMs: 50 }); const a = await registerAgent(h);
await a.hb({ modules: { webmaker: { version: '1.7.3', active: true, health: 'ok' } }, clockSkewMs: 120000 });
const s = core.publicServer(h.d, core.getServer(h.d, a.serverId)); eq(s.modules.webmaker.actual, '1.7.3'); eq(s.capacity.diskTotalBytes, 1000); ok(core.metrics(h.d, a.serverId).length >= 1);
ok(h.d.store.get('SELECT id FROM notifications WHERE incident_key=?', `server:${a.serverId}:clock`), '시계 차이 경고');
await sleep(80); eq(core.markOffline(h.d).length, 1); eq(core.listServers(h.d)[0].status, 'offline'); await a.hb(); eq(core.listServers(h.d)[0].status, 'ready', 'heartbeat 복귀');
h.d.store.run('INSERT INTO metrics_samples(server_id, at, cpu) VALUES(?,?,?)', a.serverId, new Date(Date.now() - 100 * 86400000).toISOString(), 1); eq(core.purgeMetrics(h.d), 1); ok(core.metrics(h.d, a.serverId, { since: new Date(0).toISOString() }).every((m) => Date.now() - Date.parse(m.at) < 91 * 86400000));
});
await test('부트스트랩 관문: 첫 Agent 승인 전 포털 API 503, 승인·HAL 실행 버전 뒤 완료·수동 설치 잠김', async () => {
const h = makeHal(); h.d.vault.set('service:portal', portalSecret);
let r = await call(h.app, 'portal', portalSecret, 'GET', '/portal/v1/servers'); eq(r.status, 503); eq(r.json.error, 'bootstrap_incomplete');
core.claimLive(h.d, { instanceId: h.d.instanceId, version: '0.1.1' }); const a = await registerAgent(h, { role: 'control' });
ok(core.bootstrapComplete(h.d)); eq(core.bootstrapState(h.d).manual_install_locked, 1); eq(core.bootstrapState(h.d).first_agent_id, a.agentId);
r = await call(h.app, 'portal', portalSecret, 'GET', '/portal/v1/servers'); eq(r.status, 200); eq(r.json.servers.length, 1); ok(!('facts' in r.json.servers[0]));
r = await call(h.app, 'portal', 'bad', 'GET', '/portal/v1/servers'); eq(r.status, 401);
});
await test('하나의 HAL: 다른 인스턴스의 임대가 살아 있으면 기동 거부, takeover 허용', () => {
const h = makeHal(); core.claimLive(h.d, { instanceId: 'hal_A', version: '0.1.1' });
const e = (() => { try { core.claimLive(h.d, { instanceId: 'hal_B', version: '0.1.1' }); } catch (x) { return x; } })(); eq(e.code, 'another_hal_live');
ok(core.claimLive(h.d, { instanceId: 'hal_B', version: '0.1.1', takeover: true })); ok(core.claimLive(h.d, { instanceId: 'hal_C', version: '0.1.1', now: Date.now() + 120000 }), '임대 만료 뒤 허용');
});
await test('포털 작업: 멱등 생성, Agent 배정·long-poll·결과 → HAL 작업 완료(외부 확인 포함), 이벤트 outbox', async () => {
const h = makeHal(); h.d.vault.set('service:portal', portalSecret); bootstrapDone(h.d); const a = await registerAgent(h);
let r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/jobs', { kind: 'site.deploy', target: a.serverId, payload: { siteId: 's1', externalCheckUrl: 'https://s1.example.test/', expectMarker: 'marker' }, idempotencyKey: 'dep-1', groupId: 'g', subjectType: 'site', subjectId: 's1' }); eq(r.status, 200, JSON.stringify(r.json)); ok(r.json.created); const id = r.json.halJobId;
r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/jobs', { kind: 'site.deploy', target: a.serverId, payload: {}, idempotencyKey: 'dep-1' }); eq(r.json.created, false); eq(r.json.halJobId, id);
r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/jobs', { kind: 'module.deploy', target: a.serverId }); eq(r.status, 400, '포털은 module.* 요청 불가');
const p = h.d.jobs.tick(); await sleep(30);
let n = await a.next(); eq(n.json.job.kind, 'site.deploy'); eq(n.json.job.payload.siteId, 's1'); ok(!('_dedupe' in n.json.job.payload));
await a.progress(n.json.job.id, { progress: 40, steps: [{ id: 'x', state: 'running' }] }); await sleep(40); ok(h.d.jobs.get(id).progress >= 40);
await a.result(n.json.job.id, { status: 'done', result: { version: 'v1', siteUrl: 'https://s1.example.test/' } }); await p;
const j = h.d.jobs.get(id); eq(j.status, 'done'); eq(j.result.version, 'v1'); ok(j.result.externalCheck.ok); eq(j.steps.map((s) => s.state), ['done', 'done', 'done']);
r = await call(h.app, 'portal', portalSecret, 'GET', `/portal/v1/jobs/${id}`); eq(r.json.status, 'done'); eq(r.json.result.version, 'v1');
ok(h.d.store.get('SELECT COUNT(*) AS n FROM portal_events_outbox').n >= 2, '상태 변화마다 outbox');
// 실패 결과 → failed, 재시도 API → queued
r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/jobs', { kind: 'site.check', target: a.serverId, payload: { siteId: 's1', urls: [] } }); const id2 = r.json.halJobId; const p2 = h.d.jobs.tick(); await sleep(30); await actAgent(a, () => ({ status: 'failed', error: 'disk full', errorCode: 'enospc' })); await p2; eq(h.d.jobs.get(id2).status, 'failed'); eq(h.d.jobs.get(id2).error_code, 'enospc');
r = await call(h.app, 'portal', portalSecret, 'POST', `/portal/v1/jobs/${id2}/retry`); eq(r.json.status, 'queued');
// needs_user 결과 → HAL 작업 needs_user
const p3 = h.d.jobs.tick(); await sleep(30); await actAgent(a, () => ({ status: 'needs_user', userMessage: 'sudo 필요' })); await p3; eq(h.d.jobs.get(id2).status, 'needs_user'); ok(h.d.jobs.get(id2).user_message.includes('sudo'));
});
await test('모듈 배포: 릴리스 승인·HAL 먼저·대상별 자식 작업·일부 실패는 partial, 실제 버전 불일치는 실패, 되돌리기', async () => {
const h = makeHal(); bootstrapDone(h.d); const a1 = await registerAgent(h, { name: 'w1', role: 'control' }); const a2 = await registerAgent(h, { name: 'w2', role: 'web' });
const art = join(h.dataDir, 'artifacts', '0.1.1'); mkdirSync(art, { recursive: true }); writeFileSync(join(art, 'portal-0.1.1.tar.gz'), 'x'); writeFileSync(join(art, 'hal-0.1.1.tar.gz'), 'y');
const sum = (f) => createHash('sha256').update(f).digest('hex');
core.registerRelease(h.d, { version: '0.1.1', mainCommit: 'abc', builtAt: new Date().toISOString(), artifacts: { portal: { file: 'portal-0.1.1.tar.gz', sha256: sum('x') }, hal: { file: 'hal-0.1.1.tar.gz', sha256: sum('y') } } });
const enq = (module, version = '0.1.1', ids = [a1.serverId, a2.serverId]) => h.d.jobs.enqueue({ kind: 'module.deploy', subject: { module, version, serverIds: ids }, createdBy: 'op', actorType: 'operator' }).job;
let j = enq('portal'); await h.d.jobs.drain(); eq(h.d.jobs.get(j.id).error_code, 'release_not_approved');
core.approveRelease(h.d, '0.1.1', 'op'); core.setBootstrap(h.d, { hal_running_version: '0.1.0' });
j = enq('portal'); await h.d.jobs.drain(); eq(h.d.jobs.get(j.id).error_code, 'hal_first', 'HAL 이 먼저');
// HAL 자체 배포 → 실행 버전 갱신
j = enq('hal', '0.1.1', [a1.serverId]); let pp = pump(h, j.id); await serveOne(a1, () => ({ status: 'done', result: { actualVersion: '0.1.1', health: { ok: true }, active: true, previousVersion: '0.1.0' } })); await pp;
eq(h.d.jobs.get(j.id).status, 'done', JSON.stringify(h.d.jobs.get(j.id).error)); eq(core.bootstrapState(h.d).hal_running_version, '0.1.1'); ok(core.getRelease(h.d, '0.1.1').hal_deployed_at);
// 포털을 두 서버에: w1 성공, w2 실패 → partial
j = enq('portal'); pp = pump(h, j.id); await serveOne(a1, () => ({ status: 'done', result: { actualVersion: '0.1.1', health: { ok: true }, active: true } })); await serveOne(a2, () => ({ status: 'failed', error: 'health failed', errorCode: 'health_failed' })); await pp;
const parent = h.d.jobs.get(j.id); eq(parent.status, 'failed', JSON.stringify(parent.error)); eq(parent.error_code, 'partial_failure'); const sum2 = h.d.jobs.childrenSummary(j.id); eq(sum2.overall, 'partial'); eq(sum2.total, 2);
eq(h.d.store.get('SELECT health, actual_version FROM module_targets WHERE module=? AND server_id=?', 'portal', a1.serverId).actual_version, '0.1.1');
// 버전 불일치(Agent 가 다른 버전 보고) → 실패
j = enq('portal', '0.1.1', [a2.serverId]); pp = pump(h, j.id); await serveOne(a2, () => ({ status: 'done', result: { actualVersion: '0.1.0', health: { ok: true } } })); await pp; eq(h.d.jobs.childrenSummary(j.id).children[0].status, 'failed');
ok(h.d.store.get('SELECT 1 AS x FROM module_targets WHERE module=? AND server_id=? AND target_version<>actual_version', 'portal', a2.serverId));
// 되돌리기: 이전 버전으로
j = h.d.jobs.enqueue({ kind: 'module.rollback', subject: { module: 'portal', toVersion: '0.1.0', serverIds: [a1.serverId] }, createdBy: 'op', actorType: 'operator' }).job; pp = pump(h, j.id); const rb = await serveOne(a1, () => ({ status: 'done', result: { actualVersion: '0.1.0', health: { ok: true }, active: true } })); await pp; eq(rb.kind, 'module.rollback'); eq(h.d.jobs.get(j.id).status, 'done', JSON.stringify(h.d.jobs.get(j.id).error));
});
await test('외부 Agent: 허용 작업만, 다른 사이트 거부, 상태 조회', async () => {
const h = makeHal(); bootstrapDone(h.d); await registerAgent(h, { role: 'control' }); const ext = await registerAgent(h, { kind: 'external', siteId: 's9', groupId: 'g1', name: '고객서버' });
const e = (() => { try { core.dispatchAgentJob(h.d, { halJobId: 'x', agentId: ext.agentId, kind: 'backup.create', payload: {} }); } catch (x) { return x; } })(); eq(e.code, 'kind_not_allowed_external');
const e2 = (() => { try { core.dispatchAgentJob(h.d, { halJobId: 'x', agentId: ext.agentId, kind: 'site.deploy', payload: { siteId: 'other' } }); } catch (x) { return x; } })(); eq(e2.code, 'site_mismatch');
ok(core.dispatchAgentJob(h.d, { halJobId: 'x', agentId: ext.agentId, kind: 'site.deploy', payload: { siteId: 's9' } }));
const st = core.externalAgentStatus(h.d, ext.agentId); eq(st.status, 'connected'); eq(st.siteId, 's9');
h.d.vault.set('service:portal', portalSecret); const r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/external-agents/token', { siteId: 's10', groupId: 'g1' }); ok(r.json.token); ok(r.json.installCommand.includes('--external --site s10'));
});
await test('운영자: 최초 설정 토큰, 로그인·잠금, 네트워크 관문·접근권, 재인증 필요, CSRF, MFA', async () => {
const h = makeHal(); writeFileSync(join(h.dataDir, 'setup.token'), 'setup-example-token');
let r = await invoke(h.app, 'POST', '/ops/api/setup', { body: { setupToken: 'wrong', login: 'ozops', password: 'password-example-1', displayName: '오즈' } }); eq(r.status, 403);
r = await invoke(h.app, 'POST', '/ops/api/setup', { body: { setupToken: 'setup-example-token', login: 'ozops', password: 'password-example-1', displayName: '오즈' } }); eq(r.status, 200, JSON.stringify(r.json)); ok(!existsSync(join(h.dataDir, 'setup.token')));
r = await invoke(h.app, 'POST', '/ops/api/setup', { body: { setupToken: 'setup-example-token', login: 'ozops2', password: 'password-example-1' } }); eq(r.status, 403, '두 번째 설정 거부');
// 허용 네트워크 밖: 로그인도 403
r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' }, ip: '203.0.113.5' }); eq(r.status, 403); eq(r.json.error, 'access_denied');
for (let i = 0; i < 5; i++) { r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'bad' } }); } eq(r.status, 400); r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' } }); eq(r.json.error, 'locked');
h.d.store.run('UPDATE ops_accounts SET failed_logins=0, lock_until=NULL');
r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' } }); eq(r.status, 200); const cookie = r.cookies.aw_ops; const csrf = r.json.csrf; ok(cookie && csrf);
const asOp = (method, path, body, extra = {}) => invoke(h.app, method, path, { body, cookies: { aw_ops: cookie }, headers: { 'x-csrf': csrf, ...extra } });
r = await asOp('GET', '/ops/api/me'); eq(r.json.operator.login, 'ozops'); ok(r.json.recentlyAuthed);
r = await invoke(h.app, 'POST', '/ops/api/servers', { body: { name: 'x', role: 'web' }, cookies: { aw_ops: cookie } }); eq(r.status, 403); eq(r.json.error, 'csrf');
r = await asOp('POST', '/ops/api/servers', { name: 'x', role: 'web' }); eq(r.status, 200); const sid = r.json.server.id;
h.d.store.run('UPDATE sessions SET reauth_at=?', new Date(Date.now() - 11 * 60000).toISOString());
r = await asOp('PATCH', `/ops/api/servers/${sid}`, { status: 'isolated' }); eq(r.json.error, 'reauth_required'); r = await asOp('POST', '/ops/api/reauth', { password: 'password-example-1' }); eq(r.status, 200); r = await asOp('PATCH', `/ops/api/servers/${sid}`, { status: 'isolated' }); eq(r.status, 200);
// 접근권: 밖에서 URL 로 기기 묶기 → 쿠키로 로그인 화면 허용
r = await asOp('POST', '/ops/api/access-grants', { hours: 2 }); const url = new URL(r.json.grant.url); r = await invoke(h.app, 'GET', '/ops/access?code=' + url.searchParams.get('code'), { ip: '203.0.113.5', headers: { 'user-agent': 'UA-1' } }); eq(r.status, 302); const gc = r.cookies.aw_ops_grant; ok(gc);
r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' }, ip: '203.0.113.5', cookies: { aw_ops_grant: gc }, headers: { 'user-agent': 'UA-1' } }); eq(r.status, 200); eq(r.json.via, 'grant');
r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' }, ip: '203.0.113.5', cookies: { aw_ops_grant: gc }, headers: { 'user-agent': 'UA-2' } }); eq(r.status, 403, '다른 기기(UA) 거부');
// MFA
r = await asOp('POST', '/ops/api/mfa/begin'); const secret = r.json.secret; r = await asOp('POST', '/ops/api/mfa/confirm', { code: '000000' }); eq(r.json.error, 'bad_totp'); r = await asOp('POST', '/ops/api/mfa/confirm', { code: totpCode(secret) }); eq(r.json.recoveryCodes.length, 8); const rc = r.json.recoveryCodes[0];
r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' } }); ok(r.json.mfaRequired); r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1', totp: rc } }); eq(r.status, 200); r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1', totp: rc } }); eq(r.status, 400, '복구 코드는 1회');
ok(ipAllowed('192.168.20.5', ['192.168.0.0/16']) && !ipAllowed('10.1.1.1', ['192.168.0.0/16']) && ipAllowed('::ffff:127.0.0.1', ['127.0.0.1/32']));
});
await test('긴급 복구: 정상 상태 선언 거부, 3단계 서로 다른 확인, 승인 부족·만료 자격·허용 범위 밖·일반 배포 거부, 정상 복구·조정·종료', async () => {
const h = makeHal(); let e = (() => { try { emergency.declare(h.d, { incidentKey: 'INC-1', reason: 'HAL DB 손상', scope: ['hal', 'hal_data'], by: 'op' }); } catch (x) { return x; } })(); eq(e.code, 'hal_healthy');
h.d.store.run('INSERT INTO incidents(id, key, severity, title, started_at, updated_at) VALUES(?,?,?,?,?,?)', 'inc1', 'INC-1', 'critical', 'HAL 장애', new Date().toISOString(), new Date().toISOString());
const s = emergency.declare(h.d, { incidentKey: 'INC-1', reason: 'HAL DB 손상으로 작업 엔진 사용 불가', scope: ['hal', 'hal_data'], by: 'op' }); eq(s.status, 'declared');
e = (() => { try { emergency.issueCredential(h.d, s.id, 'op'); } catch (x) { return x; } })(); eq(e.code, 'approvals_insufficient');
e = (() => { try { emergency.approve(h.d, s.id, { step: 2, by: 'op', identifiers: ['hal'] }); } catch (x) { return x; } })(); eq(e.code, 'wrong_step');
emergency.approve(h.d, s.id, { step: 1, by: 'op', impactText: 'HAL 제어 평면 중단, 고객 서비스는 유지' });
e = (() => { try { emergency.approve(h.d, s.id, { step: 2, by: 'op', identifiers: ['hal'] }); } catch (x) { return x; } })(); eq(e.code, 'identifiers_mismatch');
emergency.approve(h.d, s.id, { step: 2, by: 'op', identifiers: ['hal_data', 'hal'] });
e = (() => { try { emergency.approve(h.d, s.id, { step: 3, by: 'op', finalConfirm: '아무거나', revertible: true }); } catch (x) { return x; } })(); eq(e.code, 'final_confirm_mismatch');
const s3 = emergency.approve(h.d, s.id, { step: 3, by: 'op', finalConfirm: '복구 실행 INC-1', revertible: true }); eq(s3.status, 'approved'); eq(s3.approvals.length, 3); ok(s3.approvals.every((a) => a.at));
const cred = emergency.issueCredential(h.d, s.id, 'op'); ok(cred.credential);
let actions = []; h.d.emergencyRunner = async (action, target) => { actions.push(action); return { ok: true, action }; };
e = await rejects(() => emergency.perform(h.d, s.id, { credential: cred.credential, action: 'module.deploy', by: 'op' }), 'action_forbidden');
e = await rejects(() => emergency.perform(h.d, s.id, { credential: cred.credential, action: 'cert.reissue_internal', by: 'op' }), 'out_of_scope');
e = await rejects(() => emergency.perform(h.d, s.id, { credential: 'bad', action: 'hal.restart', by: 'op' }), 'bad_credential');
ok((await emergency.perform(h.d, s.id, { credential: cred.credential, action: 'hal.restart', by: 'op' })).ok); eq(actions, ['hal.restart']);
const rejectedLogged = emergency.getSession(h.d, s.id).actions.filter((a) => a.result === 'rejected').length; eq(rejectedLogged, 2, '거부도 기록');
h.d.store.run('UPDATE emergency_sessions SET credential_expires_at=? WHERE id=?', new Date(Date.now() - 1000).toISOString(), s.id);
await rejects(() => emergency.perform(h.d, s.id, { credential: cred.credential, action: 'hal.restart', by: 'op' }), 'credential_expired');
h.d.store.run('UPDATE emergency_sessions SET credential_expires_at=? WHERE id=?', new Date(Date.now() + 60000).toISOString(), s.id);
const v = emergency.verifyAndRevoke(h.d, s.id, { by: 'op', verify: { halRestarted: true } }); ok(v.credential_revoked_at); await rejects(() => emergency.perform(h.d, s.id, { credential: cred.credential, action: 'hal.restart', by: 'op' }), 'credential_revoked');
const rc = emergency.reconcile(h.d, s.id, 'op'); ok(Array.isArray(rc.drift)); eq(emergency.close(h.d, s.id, 'op').status, 'closed');
ok(h.d.store.all('SELECT * FROM audit_log WHERE action LIKE \'emergency.%\'').every((a) => a.immutable === 1));
});
await test('재해복구: 설정 검증·복구 사본 생성·검증·보존, 복원 승인·기존 제어 중지 확인, 완료 확인 항목, 훈련 → 준비 상태', async () => {
const h = makeHal(); h.d.store.exec('CREATE TABLE t(x)'); eq(dr.readiness(h.d).ready, false); eq(dr.readiness(h.d).missingConfig.length, 4);
const ext = join(tmpDir(), 'ext'); dr.setConfig(h.d, { rto_minutes: 60, rpo_minutes: 30, snapshot_interval_minutes: 60, retention_count: 2, external_store: ext }, 'op');
// 메모리 DB 는 VACUUM INTO 가 가능하다
const s1 = dr.snapshot(h.d, { kind: 'manual', by: 'op' }); eq(s1.verify_state, 'ok'); ok(existsSync(s1.location)); ok(s1.external_copied_at, '외부 보관'); ok(s1.sha256);
const list = execFileSync('tar', ['-tzf', s1.location]).toString(); ok(list.includes('hal.sqlite') && list.includes('meta.json') && !list.includes('vault.key'));
dr.snapshot(h.d); dr.snapshot(h.d); eq(dr.listSnapshots(h.d).filter((s) => !s.deleted_at).length, 2, '보존 개수');
const latest = dr.listSnapshots(h.d).find((x) => !x.deleted_at); const rq = dr.requestRestore(h.d, { snapshotId: latest.id, by: 'op' }); let e = (() => { try { dr.startRestore(h.d, rq.id, { by: 'op', oldControlStopped: true, confirmText: '기존 제어 위치를 중지했습니다' }); } catch (x) { return x; } })(); eq(e.code, 'restore_not_approved');
dr.approveRestore(h.d, rq.id, 'op2'); e = (() => { try { dr.startRestore(h.d, rq.id, { by: 'op', oldControlStopped: false, confirmText: '' }); } catch (x) { return x; } })(); eq(e.code, 'old_control_not_confirmed');
const st = dr.startRestore(h.d, rq.id, { by: 'op', oldControlStopped: true, confirmText: '기존 제어 위치를 중지했습니다' }); ok(st.command.includes('hal-restore.sh'));
e = (() => { try { dr.finishRestore(h.d, rq.id, { by: 'op', checks: { operatorLogin: true } }); } catch (x) { return x; } })(); eq(e.code, 'checks_incomplete');
eq(dr.finishRestore(h.d, rq.id, { by: 'op', checks: { operatorLogin: true, halControl: true, https: true, customerService: true } }).status, 'completed');
const dl = dr.startDrill(h.d, { by: 'op', snapshotId: latest.id }); eq(dr.finishDrill(h.d, dl.id, { by: 'op', steps: { hal: true, operatorLogin: true, vault: true, certBase: true, httpsCustomer: false }, measuredRtoMinutes: 20, measuredRpoMinutes: 10 }).result, 'failed');
const dl2 = dr.startDrill(h.d, { by: 'op' }); eq(dr.finishDrill(h.d, dl2.id, { by: 'op', steps: { hal: true, operatorLogin: true, vault: true, certBase: true, httpsCustomer: true }, measuredRtoMinutes: 20, measuredRpoMinutes: 10 }).result, 'success'); ok(dr.readiness(h.d).ready);
});
await test('공개 상태·운영 현황·전송 업로드·artifact 내려받기', async () => {
const h = makeHal(); bootstrapDone(h.d); const a = await registerAgent(h);
let r = await invoke(h.app, 'GET', '/api/status'); ok(r.json.components.length >= 3); ok(!JSON.stringify(r.json).includes(a.serverId), '내부 id 없음');
r = await invoke(h.app, 'GET', '/status'); ok(r.text.includes('Ariweb 서비스 상태'));
const body = Buffer.from('hello-transfer'); const sum = createHash('sha256').update(body).digest('hex');
const low = (o) => Object.fromEntries(Object.entries(o).map(([k, v]) => [k.toLowerCase(), v])); const hdr = low(signHeaders(`agent:${a.agentId}`, a.secret, 'POST', '/agent/v1/transfers/tr_t1', '')); r = await invoke(h.app, 'POST', '/agent/v1/transfers/tr_t1', { body: body.toString(), headers: { ...hdr, 'content-type': 'application/octet-stream', 'x-transfer-sha256': sum } }); eq(r.status, 200, JSON.stringify(r.json)); eq(r.json.sha256, sum);
ok(core.getTransfer(h.d, 'tr_t1'));
r = await invoke(h.app, 'POST', '/agent/v1/transfers/tr_t2', { body: 'x', headers: { ...low(signHeaders(`agent:${a.agentId}`, a.secret, 'POST', '/agent/v1/transfers/tr_t2', '')), 'content-type': 'application/octet-stream', 'x-transfer-sha256': 'bad' } }); eq(r.status, 400);
const art = join(h.dataDir, 'artifacts', '0.1.1'); mkdirSync(art, { recursive: true }); writeFileSync(join(art, 'agent-0.1.1.tar.gz'), 'tgz'); core.registerRelease(h.d, { version: '0.1.1', mainCommit: 'abc', builtAt: 'now', artifacts: { agent: { file: 'agent-0.1.1.tar.gz', sha256: createHash('sha256').update('tgz').digest('hex') } } });
r = await call(h.app, `agent:${a.agentId}`, a.secret, 'GET', '/agent/v1/releases/0.1.1/artifact?module=agent'); eq(r.status, 200); eq(r.text, 'tgz');
const t = core.issueRegistrationToken(h.d, { kind: 'managed' }); r = await invoke(h.app, 'GET', '/agent/v1/bootstrap-artifact?module=agent&version=0.1.1', { headers: { 'x-registration-token': t.token } }); eq(r.status, 200); eq(r.text, 'tgz');
});
await test('서버 이전 작업: 단계 순서와 verify_new 실패 시 전환하지 않음', async () => {
const h = makeHal(); bootstrapDone(h.d); const a1 = await registerAgent(h, { name: 'from' }); const a2 = await registerAgent(h, { name: 'to' });
const j = h.d.jobs.enqueue({ kind: 'site.relocate', subject: { payload: { siteId: 's1', fromServerId: a1.serverId, toServerId: a2.serverId, kind: 'static' } }, subjectType: 'site', subjectId: 's1' }).job;
const seen = []; const pp = pump(h, j.id, 10000); const serve = async (a, fn) => { const job = await serveOne(a, fn); if (job) seen.push(job.kind); return job; };
await serve(a2, () => ({ status: 'done', result: { prepared: true } })); await serve(a1, () => ({ status: 'done', result: { transferId: 'tr_rl1', checksum: 'c1' } })); await serve(a2, () => ({ status: 'done', result: { ok: true } })); await serve(a1, () => ({ status: 'done', result: { transferId: 'tr_rl2', checksum: 'c2' } })); await serve(a2, () => ({ status: 'done', result: { ok: true, checksum: 'c2' } })); await serve(a2, () => ({ status: 'done', result: { ok: false, reason: '체크섬 다름' } }));
await pp;
eq(seen, ['site.prepare', 'site.export', 'site.import', 'site.export', 'site.import', 'site.stage_check']); eq(h.d.jobs.get(j.id).status, 'failed'); eq(h.d.jobs.get(j.id).error_code, 'verify_new_failed'); ok(!h.d.store.get('SELECT 1 AS x FROM agent_jobs WHERE hal_job_id=? AND kind=\'site.activate\'', j.id), '전환하지 않음');
});
done();