OZ-Group/AriWeb-Suite
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(hal): inventory existing Webmaker sites

Browse Source
This commit is contained in:
oz
2026-10-27 08:57:21 +00:00
parent 4bd698452b
commit 12958ab695
10 changed files with 246 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/agent.test.mjs
+31 -2
View File
@@ -6,7 +6,8 @@ import { createRunner } from '../agent/runner.mjs';
import { KINDS } from '../agent/jobs/index.mjs';
import { ingestLog } from '../agent/jobs/site.mjs';
import { sample } from '../agent/metrics.mjs';
import { mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync, readlinkSync, copyFileSync, statSync, appendFileSync, chmodSync } from 'node:fs';
import { scanWebmakerInventory, webmakerInventoryBatches, configuredWebmakerInventory } from '../agent/webmaker-inventory.mjs';
import { mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync, readlinkSync, copyFileSync, statSync, appendFileSync, chmodSync, symlinkSync } from 'node:fs';
import { join } from 'node:path';
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
@@ -32,6 +33,33 @@ function env(over = {}) {
/** 사이트 번들 디렉터리 만들기 */
function bundle(base, name, files = { 'index.html': '<h1>hello marker-1</h1>' }) { const d = join(base, 'bundles', name); mkdirSync(d, { recursive: true }); for (const [f, c] of Object.entries(files)) { mkdirSync(join(d, f, '..'), { recursive: true }); writeFileSync(join(d, f), c); } return d; }
await test('Webmaker 사이트 인벤토리: 모든 디렉터리·배포 상태·메타데이터를 비밀 없이 HAL에 전달', async () => {
const base = tmpDir(); const dataRoot = join(base, 'data'); const dataDir = join(base, 'custom-meta'); const sitesRoot = join(base, 'published');
mkdirSync(dataDir, { recursive: true }); for (const id of ['a', 'b', 'c', 'e'].map((x) => x.repeat(32))) { mkdirSync(join(sitesRoot, id), { recursive: true }); writeFileSync(join(sitesRoot, id, 'index.html'), id[0]); }
writeFileSync(join(dataDir, 'a'.repeat(32) + '.json'), JSON.stringify({ id: 'a'.repeat(32), title: '사이트 A', status: 'idle', ownerId: 'owner-a', domains: [{ name: 'a.example.test' }], deployments: [{ ok: true, kind: 'deploy' }], content: 'HAL에 보내면 안 되는 본문', commands: [{ prompt: '비밀' }] }));
writeFileSync(join(dataDir, 'b'.repeat(32) + '.json'), JSON.stringify({ id: 'b'.repeat(32), title: '초안', deployments: [] }));
writeFileSync(join(dataDir, 'c'.repeat(32) + '.json'), JSON.stringify({ id: 'c'.repeat(32), title: '중지', deployments: [{ ok: true, kind: 'deploy' }, { ok: true, kind: 'unpublish' }], unpublishedAt: '2026-01-01T00:00:00Z' }));
writeFileSync(join(dataDir, 'd'.repeat(32) + '.json'), JSON.stringify({ id: 'd'.repeat(32), title: '배포 전 초안', deployments: [] }));
writeFileSync(join(dataDir, 'f'.repeat(32) + '.json'), JSON.stringify([]));
const waitingId = 'f'.repeat(31) + 'd'; const legacyId = 'f'.repeat(31) + 'e';
writeFileSync(join(dataDir, waitingId + '.json'), JSON.stringify({ title: 'DNS 대기', deployments: [{ ok: true, phase: 'dns-wait' }] }));
writeFileSync(join(dataDir, legacyId + '.json'), JSON.stringify({ title: '구형 배포 기록', deployments: [{ ok: true }] }));
writeFileSync(join(dataDir, 'A'.repeat(32) + '.json'), JSON.stringify({ title: '대문자 ID는 비정상' }));
writeFileSync(join(dataDir, 'g'.repeat(32) + '.json.swp'), JSON.stringify({ title: '임시 파일은 제외' }));
const inv = scanWebmakerInventory({ dataRoot, dataDir, sitesRoot });
eq(inv.total, 8); eq(inv.returned, 8); eq(inv.truncated, false); eq(inv.complete, true); eq(inv.sites[0].id, 'a'.repeat(32)); eq(inv.sites[0].title, '사이트 A'); eq(inv.sites[0].ownerId, 'owner-a'); eq(inv.sites[0].domains, ['a.example.test']); eq(inv.sites[0].deploymentState, 'deployed'); ok(inv.sites[0].bytes > 0);
eq(inv.sites[1].deploymentState, 'draft'); eq(inv.sites[2].deploymentState, 'unpublished'); eq(inv.sites[3].deploymentState, 'draft'); eq(inv.sites[3].files, 0, '메타데이터에만 있는 미배포 초안도 포함'); eq(inv.sites[4].metadata, 'missing', '배포 디렉터리에만 있는 사이트도 포함'); eq(inv.sites.find((x) => x.id === 'f'.repeat(32)).metadata, 'invalid', '배열·원시 JSON은 유효한 사이트 메타데이터가 아님'); eq(inv.sites.find((x) => x.id === waitingId).deploymentState, 'draft', 'DNS 대기 단계는 운영 배포 완료가 아님'); eq(inv.sites.find((x) => x.id === legacyId).deploymentState, 'deployed', 'phase 도입 전 성공 기록은 배포됨'); ok(inv.errors.some((x) => x.includes('invalid_shape'))); ok(!JSON.stringify(inv).includes('대문자 ID') && !JSON.stringify(inv).includes('임시 파일은 제외')); ok(!JSON.stringify(inv).includes('본문') && !JSON.stringify(inv).includes('비밀'));
const limited = scanWebmakerInventory({ dataRoot, dataDir, sitesRoot, maxSites: 2 }); eq(limited.total, 8); eq(limited.returned, 2); eq(limited.truncated, true); eq(limited.complete, false);
const broken = scanWebmakerInventory({ dataRoot, dataDir, sitesRoot: join(base, 'missing-root') }); eq(broken.total, 7); eq(broken.complete, false); ok(broken.errors.some((x) => x.startsWith('sites:')), '배포 루트 오류를 숨기지 않으면서 메타데이터 프로젝트는 보존');
const missingMeta = scanWebmakerInventory({ dataRoot, dataDir: join(base, 'missing-meta'), sitesRoot }); eq(missingMeta.complete, false); ok(missingMeta.errors.some((x) => x.startsWith('metadata:')), '메타데이터 루트 읽기 실패 표시'); eq(missingMeta.sites[0].metadata, 'missing');
const aliasRoot = join(base, 'alias-root'); symlinkSync(dataRoot, aliasRoot, 'dir'); const aliasInv = scanWebmakerInventory({ dataRoot: aliasRoot, dataDir, sitesRoot }); eq(aliasInv.total, 8); eq(aliasInv.errors.filter((x) => x.includes('invalid_shape')).length, 1, '실경로가 같은 metadata root는 한 번만 스캔');
const largeSites = Array.from({ length: 100 }, (_, i) => ({ id: String(i).padStart(32, 'a'), title: 'x'.repeat(200), domains: Array.from({ length: 100 }, (_, n) => `${String(n).padStart(240, 'd')}.test`) })); const batches = webmakerInventoryBatches({ ...inv, sites: largeSites, returned: 100 }); ok(Buffer.byteLength(JSON.stringify({ facts: { webmakerInventory: { ...inv, sites: largeSites } } })) > 2 * 1024 * 1024, '단일 heartbeat면 2MiB 초과 fixture'); ok(batches.length > 1 && batches.every((b) => b.sites.length <= 1000 && Buffer.byteLength(JSON.stringify(b)) <= 512 * 1024)); eq(batches.flatMap((b) => b.sites).map((x) => x.id), largeSites.map((x) => x.id), 'byte 배치 후 전체 사이트 무누락');
const manySites = Array.from({ length: 1500 }, (_, i) => ({ id: `s${i}` })); const countBatches = webmakerInventoryBatches({ ...inv, sites: manySites, returned: 1500 }); ok(countBatches.length === 2 && countBatches.every((b) => b.sites.length <= 1000)); eq(countBatches.flatMap((b) => b.sites).length, 1500, 'HAL 항목 제한 배치 후 무누락');
const noisy = webmakerInventoryBatches({ ...inv, sites: [], errors: Array.from({ length: 100000 }, () => 'scan:EACCES'.repeat(40)) }); eq(noisy.length, 1); ok(Buffer.byteLength(JSON.stringify(noisy[0])) <= 512 * 1024 && noisy[0].meta.errorsTruncated, '빈 사이트·대량 오류도 HTTP 한도 내 명시적 절단');
const unconfigured = configuredWebmakerInventory({ units: { webmaker: 'webmaker' } }, {}); ok(unconfigured.webmakerInventory.errors[0].startsWith('configuration:') && unconfigured.webmakerInventory.complete === false, '명시적 운영 경로 없이는 추측 스캔하지 않음'); let incompleteRejected = false; try { webmakerInventoryBatches(unconfigured.webmakerInventory); } catch (e) { incompleteRejected = e.message === 'inventory_incomplete'; } ok(incompleteRejected, '불완전 스캔은 HAL의 마지막 정상 인벤토리를 덮어쓰지 않음');
const configured = configuredWebmakerInventory({ units: { webmaker: 'webmaker' } }, { ARIWEB_WEBMAKER_DATA_ROOT: dataRoot, ARIWEB_WEBMAKER_DATA_DIR: dataDir, ARIWEB_WEBMAKER_SITES_ROOT: sitesRoot }); eq(configured.webmakerInventory.total, 8); eq(configured.webmakerInventory.complete, true);
});
await test('site.deploy: 레이아웃(releases/current)·버전 기록·vhost·내부 확인·previous, rollback, 실패 시 이전 버전 유지와 leftovers, remove', async () => {
const e = env(); const b1 = bundle(e.base, 'b1'); mkdirSync(e.roots.wildcardCert, { recursive: true }); writeFileSync(join(e.roots.wildcardCert, 'fullchain.pem'), 'x');
let r = await e.run('site.deploy', { siteId: 'site1', versionRef: 'v1', bundlePath: b1, domains: ['site1.ariweb.net'], primaryDomain: 'site1.ariweb.net', expectMarker: 'marker-1' }); eq(r.status, 'done', JSON.stringify(r));
@@ -152,8 +180,9 @@ await test('module.install(실제 환경 결함 수정): REVISION.json 의 Suite
const wmDir = join(e.roots.srv, 'webmaker'); mkdirSync(join(wmDir, 'releases', 'old'), { recursive: true }); execFileSync('ln', ['-sfn', join(wmDir, 'releases', 'old'), join(wmDir, 'current')]);
const wmData = join(e.base, 'wmdata'); mkdirSync(wmData, { recursive: true }); writeFileSync(join(wmData, 'ok.json'), '{}'); writeFileSync(join(wmData, 'unreadable.json'), '{}'); chmodSync(join(wmData, 'unreadable.json'), 0o000); // root 가 남긴 파일처럼 읽을 수 없는 항목
e.sys.unitText = async (u) => (u === 'webmaker' ? `[Service]\nUser=${(await import('node:os')).userInfo().username}\nWorkingDirectory=${wmDir}/current\nProtectSystem=strict\nEnvironment=DATA_ROOT=${wmData}\n` : '');
const before = e.sys.calls.length; const runner2 = createRunner({ kinds: KINDS, roots: e.roots, sys: e.sys, client: fakeClient({ artifacts: { '/art/wm': wtgz } }), dataDir: e.dataDir });
const before = e.sys.calls.length; const runner2 = createRunner({ kinds: KINDS, roots: e.roots, sys: e.sys, client: fakeClient({ artifacts: { '/art/wm': wtgz } }), dataDir: e.dataDir }); const oldLocale = process.env.LC_ALL; process.env.LC_ALL = 'C';
r = await runner2.execute({ id: 'u2', kind: 'module.install', payload: { module: 'webmaker', version: '0.1.1', artifactUrl: '/art/wm', sha256: wsum, descriptor: { name: 'webmaker', unit: 'webmaker', unitTemplate: 'webmaker.service.tmpl', keepExistingUnit: true, health: { url: 'http://127.0.0.1:1/x', expect: 'any' }, version: { file: 'package.json' }, layout: 'releases', preStop: ['webmaker.backupData'], postStart: [], extraUnits: ['webmaker-setup'] }, appDir: wmDir, unitTemplate: 'WorkingDirectory={{APP_DIR}}/current\n{{ENV_LINES}}\nExecStart={{NODE}} server.js' } });
if (oldLocale == null) delete process.env.LC_ALL; else process.env.LC_ALL = oldLocale;
eq(r.status, 'done', JSON.stringify(r)); eq(r.result.actualVersion, '0.1.1'); eq(r.result.upstreamVersion, '1.7.3'); eq(r.steps.find((s) => s.id === 'unit').result.skipped, 'existing_unit_kept');
{ const pre = r.steps.find((s) => s.id === 'pre:webmaker.backupData').result; ok(pre.backup && existsSync(pre.backup), '데이터 백업 파일'); ok(pre.unreadable && pre.unreadable.length === 1 && /unreadable\.json/.test(pre.unreadable[0]), '읽지 못한 파일은 경고로: ' + JSON.stringify(pre.unreadable)); }
ok(!e.sys.calls.slice(before).some((c) => c.cmd === 'sudo' && c.args.includes('tee')), '유닛 파일을 다시 쓰지 않는다'); ok(e.sys.calls.slice(before).some((c) => c.args.join(' ').includes('systemctl restart webmaker-setup')));
test/hal-ui.test.mjs
+4 -2
View File
@@ -42,13 +42,15 @@ await test('화면 접근: 허용 네트워크 밖 403, 세션 없음/고객 쿠
await test('주요 화면 렌더: 서버(등록·용량·모듈 차이)·서버 상세·릴리스(HAL 먼저)·작업(부분 실패)·사건·감사·복구·계정', async () => {
const h = makeHal(); const op = await loginOp(h);
const s = core.createServer(h.d, { name: '웹1', role: 'web', location: '서울' }); const t = core.issueRegistrationToken(h.d, { serverId: s.id });
const reg = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: t.token, facts: { hostname: 'web1' }, version: '0.1.1' } }); ok(reg.json.ok);
const reg = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: t.token, facts: { hostname: 'web1', webmakerInventory: { total: 1, returned: 1, truncated: false, sites: [{ id: 'a'.repeat(32), title: '기존 사이트 <A>', ownerId: 'owner-a', status: '<idle>', domains: ['a.example.test'], deploymentState: 'deployed', bytes: 1234, files: 3, updatedAt: '2026-10-27T00:00:00Z', metadata: 'ok' }] } }, version: '0.1.1' } }); ok(reg.json.ok);
h.d.store.run('UPDATE servers SET last_seen_at=?, capacity_json=?, metrics_json=? WHERE id=?', new Date().toISOString(), JSON.stringify({ diskFreeBytes: 10e9, diskTotalBytes: 100e9, sitesHosted: 3, reservedBytes: 1e9 }), JSON.stringify({ cpu: 12, memUsed: 50, memTotal: 100, netIn: 1000, netOut: 2000 }), s.id);
core.approveServer(h.d, s.id, 'ozops');
h.d.store.run('INSERT INTO module_targets(module, server_id, target_version, actual_version, health, active, last_check_at, updated_at) VALUES(?,?,?,?,?,?,?,?)', 'portal', s.id, '0.1.1', '0.1.0', 'ok', 1, new Date().toISOString(), new Date().toISOString());
core.registerRelease(h.d, { version: '0.1.1', mainCommit: 'abcdef1234567890', builtAt: new Date().toISOString(), artifacts: { hal: { file: 'hal.tgz', sha256: 'sha256:x', size: 1 } }, checks: { testsPassed: true } });
let r = await op.as('GET', '/ops/servers'); eq(r.status, 200); ok(r.text.includes('웹1') && r.text.includes('배치 가능') && r.text.includes('0.1.0') && r.text.includes('0.1.1'), '서버 표에 상태·모듈 차이');
r = await op.as('GET', `/ops/servers/${s.id}`); ok(r.text.includes('모듈(목표 vs 실제)') && r.text.includes('서버 삭제') && r.text.includes('id="metrics"'));
r = await op.as('GET', `/ops/servers/${s.id}`); ok(r.text.includes('모듈(목표 vs 실제)') && r.text.includes('서버 삭제') && r.text.includes('id="metrics"')); ok(r.text.includes('Webmaker 사이트 디렉터리 인벤토리') && r.text.includes('owner-a') && r.text.includes('a.example.test') && r.text.includes('기존 사이트 &lt;A&gt;') && !r.text.includes('기존 사이트 <A>') && r.text.includes('&lt;idle&gt;') && !r.text.includes('<idle>'));
h.d.store.run('UPDATE servers SET facts_json=? WHERE id=?', JSON.stringify({ hostname: 'web1', webmakerInventory: { total: 0, returned: 0, truncated: false, errors: ['sites:EACCES<script>'], errorsTruncated: true, sites: [] } }), s.id); r = await op.as('GET', `/ops/servers/${s.id}`); ok(r.text.includes('Webmaker 사이트 디렉터리 인벤토리') && r.text.includes('인벤토리 스캔 오류 1건') && r.text.includes('EACCES&lt;script&gt;') && !r.text.includes('EACCES<script>'));
h.d.store.run('UPDATE servers SET facts_json=? WHERE id=?', JSON.stringify({ hostname: 'web1', webmakerInventory: { total: 'bad', returned: {}, errors: '<script>', sites: [null, { id: '<id>', domains: '<img onerror=1>', files: {} }] } }), s.id); r = await op.as('GET', `/ops/servers/${s.id}`); eq(r.status, 200); ok(r.text.includes('&lt;id&gt;') && !r.text.includes('<img onerror=1>'), 'null 등 malformed agent facts도 렌더러를 깨거나 HTML로 실행하지 않음');
r = await op.as('GET', '/ops/releases'); ok(r.text.includes('HAL 을 먼저 배포') && r.text.includes('abcdef123456') && r.text.includes('테스트 통과') && r.text.includes('승인 대기'));
// 작업: 부모 + 자식(하나 실패) → 부분 실패 표시
h.d.jobs.register('ui.demo', { title: '데모', steps: [{ id: 'a', title: 'A' }], async run(job, ctx) { await ctx.step('a', () => { if (job.subject.fail) throw new Error('대상 실패'); return 1; }); } });
test/hal.test.mjs
+12
View File
@@ -64,6 +64,18 @@ await test('heartbeat → 지표·용량·모듈 버전 저장, 무응답 → of
await sleep(80); eq(core.markOffline(h.d).length, 1); eq(core.listServers(h.d)[0].status, 'offline'); await a.hb(); eq(core.listServers(h.d)[0].status, 'ready', 'heartbeat 복귀');
h.d.store.run('INSERT INTO metrics_samples(server_id, at, cpu) VALUES(?,?,?)', a.serverId, new Date(Date.now() - 100 * 86400000).toISOString(), 1); eq(core.purgeMetrics(h.d), 1); ok(core.metrics(h.d, a.serverId, { since: new Date(0).toISOString() }).every((m) => Date.now() - Date.parse(m.at) < 91 * 86400000));
});
await test('Webmaker 인벤토리 배치: 2MiB heartbeat와 분리·순서 검증·완료 시 원자 반영·후속 heartbeat 보존', async () => {
const h = makeHal(); const a = await registerAgent(h); const path = '/agent/v1/inventory/webmaker'; const send = (body) => call(h.app, `agent:${a.agentId}`, a.secret, 'POST', path, body);
const sites = ['a', 'b', 'c'].map((x) => ({ id: x.repeat(32), title: `site-${x}`, domains: [`${x}.test`], files: 1, bytes: 2 })); sites[0].privateContact = 'do-not-store@example.test'; sites[0].nested = { secret: 'do-not-store' }; const meta = { total: 3, errors: [], truncated: false, scannedAt: '2026-10-27T00:00:00.000Z' };
let r = await send({ scanId: 'scan-1', offset: 1, itemTotal: 3, meta, sites: [sites[1]] }); eq(r.status, 400); eq(r.json.error, 'inventory_batch_order');
r = await send({ scanId: 'scan-1', offset: 0, itemTotal: 3, meta, sites: sites.slice(0, 2) }); eq(r.status, 200); eq(r.json.complete, false); eq(core.getServer(h.d, a.serverId).facts.webmakerInventory, undefined, '중간 배치는 공개하지 않음');
r = await send({ scanId: 'scan-1', offset: 2, itemTotal: 3, meta, sites: sites.slice(2) }); eq(r.status, 200); eq(r.json.complete, true); const stored = core.getServer(h.d, a.serverId).facts.webmakerInventory; eq(stored.sites.map((x) => x.id), sites.map((x) => x.id)); eq(stored.truncated, false); ok(!JSON.stringify(stored).includes('privateContact') && !JSON.stringify(stored).includes('do-not-store'), '허용되지 않은 개인정보·중첩 필드 저장 금지'); eq(h.d.store.get('SELECT COUNT(*) AS n FROM agent_inventory_staging').n, 0);
r = await send({ scanId: 'scan-partial', offset: 0, itemTotal: 1, meta: { ...meta, total: 2, truncated: true }, sites: sites.slice(0, 1) }); eq(r.status, 400); eq(r.json.error, 'inventory_incomplete');
r = await send({ scanId: 'scan-bad', offset: 0, itemTotal: 1, meta, sites: [{ id: '../bad', title: 'bad' }] }); eq(r.status, 400); eq(r.json.error, 'bad_inventory_batch');
r = await send({ scanId: 'scan-too-many', offset: 0, itemTotal: 10001, meta, sites: [] }); eq(r.status, 400); eq(r.json.error, 'bad_inventory_batch');
await a.hb({ facts: { hostname: 'web-after' } }); const facts = core.getServer(h.d, a.serverId).facts; eq(facts.hostname, 'web-after'); eq(facts.webmakerInventory.returned, 3, '작은 heartbeat가 완성된 인벤토리를 지우지 않음');
r = await send({ scanId: 'scan-cascade', offset: 0, itemTotal: 2, meta: { ...meta, total: 2 }, sites: sites.slice(0, 1) }); eq(r.status, 200); eq(h.d.store.get('SELECT COUNT(*) AS n FROM agent_inventory_staging').n, 1); ok(h.d.store.get("SELECT name FROM sqlite_master WHERE type='trigger' AND name='agent_inventory_staging_agent_cleanup'"), '이전 스키마에도 적용 가능한 Agent 삭제 cleanup trigger'); h.d.store.run('DELETE FROM agents WHERE id=?', a.agentId); eq(h.d.store.get('SELECT COUNT(*) AS n FROM agent_inventory_staging').n, 0, 'Agent 삭제 시 미완료 staging cascade');
});
await test('setBootstrap: HAL 기동 전(행 없음) CLI bootstrap-release 의 기록이 남는다(실제 환경 결함 수정)', async () => {
const h = makeHal(); eq(h.d.store.get('SELECT COUNT(*) AS n FROM bootstrap_state').n, 0); core.setBootstrap(h.d, { release_version: '0.1.1', main_commit: 'abc', artifact_sha256: 'def' });
eq(core.bootstrapState(h.d).release_version, '0.1.1'); eq(core.bootstrapState(h.d).main_commit, 'abc');