diff --git a/VERSION_HISTORY.md b/VERSION_HISTORY.md index 8ec3ebf..e12fb23 100644 --- a/VERSION_HISTORY.md +++ b/VERSION_HISTORY.md @@ -118,6 +118,7 @@ - **`deploy/bootstrap-hal.sh --portal-host none`**: 포털을 다른 호스트에서 운영하는 HAL 전용 호스트에는 `my.ariweb.net` nginx 접근 경로를 만들지 않는다(이전에는 항상 만들어 포털이 없는 호스트에도 죽은 vhost 가 남았다). 기본값(`my.ariweb.net`)의 동작은 그대로다. - bear(Ubuntu 24.04, nginx 1.24, Node 24.22, certbot nginx 플러그인 운영)에 HAL 만 최초 부트스트랩: 인증서는 호스트별 certbot(webroot `/var/www/html`, ECDSA, 갱신 훅 `systemctl reload nginx`) — `--cert /etc/letsencrypt/live/hal21.ariweb.net --acme /var/www/html`. 기존 Webmaker·Gitea·Matrix·기본 서버는 건드리지 않았다. 절차·검증·되돌리기는 `docs/RUNBOOK-0.1.1.md` §7. 배포 artifact: main `ab75b7d`, hal sha256 `f574d875…`(deploy/ 변경으로 ariweb.net 호스트의 `e7741899…` 와 다름). 알려진 사소한 결함: 부트스트랩 5단계의 HTTPS 자체 확인이 nginx reload 직후 실행되어 거짓 경고가 날 수 있다(결과에 영향 없음). - **HAL 기본 주소 안내**(`hal/app.mjs`): `https://hal21.ariweb.net/` 접속이 JSON 404를 반환하던 동작을 운영자 콘솔 표준 경로 `/ops/`로 이동하도록 수정했다. `/ops`의 기존 사설망/VPN·기기 접근권 관문과 별도 운영자 로그인은 그대로 적용된다. +- **기존 사이트 인벤토리**(`agent/webmaker-inventory.mjs`, HAL 서버 상세): Node Agent가 사이트 디렉터리와 Webmaker 메타데이터·배포 이력을 대조해 모든 디렉터리의 ID·제목·계정 식별자·배포 상태(`deployed`/`unpublished`/`draft`/`unknown`)·서비스 상태·도메인·크기·파일 수를 HAL에 연결한다. 메타데이터가 없는 디렉터리도 `missing`으로 남기며 총 수·수집 수·잘림 여부·스캔 오류를 함께 기록한다. 2 MiB heartbeat 제한과 1,000항목 제한을 넘는 인벤토리는 512 KiB 이하 인증 배치로 전송하고 마지막 배치에서만 원자 반영한다. 페이지 본문·프롬프트·연락처·인증정보는 전송하지 않는다. ### 운영 Webmaker ↔ HAL 연동 점검 도구(2026-10-27, webmaker.ariweb.net) diff --git a/agent/index.mjs b/agent/index.mjs index d61d2fe..5a6e3a9 100644 --- a/agent/index.mjs +++ b/agent/index.mjs @@ -12,6 +12,7 @@ import { createSys } from './sys.mjs'; import { createRunner } from './runner.mjs'; import { KINDS } from './jobs/index.mjs'; import { sample, facts } from './metrics.mjs'; +import { configuredWebmakerInventory, webmakerInventoryBatches } from './webmaker-inventory.mjs'; import { readFileSync, existsSync, readdirSync } from 'node:fs'; import { join } from 'node:path'; import { hostname } from 'node:os'; @@ -36,9 +37,27 @@ if (cmd === 'register') { const client = createClient({ halUrl: cfg.halUrl, agentId: cfg.id, secret: cfg.vault().get('agent-secret') }); const runner = createRunner({ kinds: KINDS, roots: cfg.roots, sys, client, dataDir: cfg.dataDir, log }); let stopping = false; let running = 0; let lastFacts = null; let restartUnit = null; + /** 기존 Webmaker 사이트: 운영에 필요한 식별자·상태·도메인·배포 용량만 facts에 싣고 본문·프롬프트·연락처·비밀은 제외한다. */ + function inventoryFacts(f) { + return configuredWebmakerInventory(f); + } + /** 2 MiB 요청 한도와 무관하게 전체 인벤토리를 512 KiB 이하 배치로 순차 전송한다. */ + async function syncWebmakerInventory(inv) { + if (!inv) return; const batches = webmakerInventoryBatches(inv); let offset = 0; + for (const body of batches) { const r = await client.request('POST', '/agent/v1/inventory/webmaker', body); offset = Number(r.nextOffset); } + if (offset !== inv.sites.length) throw new Error(`Webmaker 인벤토리 전송 수 불일치(${offset}/${inv.sites.length})`); + } /** 모듈 상태(heartbeat): facts 의 유닛이 살아 있는지 + current/REVISION.json(없으면 package.json) 버전 */ async function moduleStatus(f) { const out = {}; for (const [m, unit] of Object.entries(f.units || {})) { const appDir = (f.appDirs || {})[m] || join(cfg.roots.srv, m); let version = null; try { version = JSON.parse(readFileSync(join(appDir, 'current', 'REVISION.json'), 'utf8')).version; } catch (_) {} if (!version) { try { version = JSON.parse(readFileSync(join(appDir, 'current', 'package.json'), 'utf8')).version; } catch (_) {} } /* 도입 모듈은 REVISION.json 의 Suite 버전(상류 package.json 버전은 목표와 다르다 — 실제 환경 결함 수정) */ const active = await sys.isActive(unit); out[m] = { version, active, health: active ? 'ok' : 'down' }; } out.agent = { version: VERSION, active: true, health: 'ok' }; out.observability = { version: VERSION, active: true, health: 'ok' }; if (existsSync(cfg.roots.backups)) out.backup = { version: VERSION, active: true, health: 'ok' }; return out; } - async function heartbeat() { try { if (!lastFacts || Date.now() - lastFacts.at > 10 * 60000) lastFacts = { at: Date.now(), f: facts(sys) }; const sites = existsSync(cfg.roots.sites) ? readdirSync(cfg.roots.sites).filter((x) => existsSync(join(cfg.roots.sites, x, 'current'))).length : 0; await client.request('POST', '/agent/v1/heartbeat', { version: VERSION, metrics: sample(cfg.roots), modules: await moduleStatus(lastFacts.f), clockSkewMs: client.clockSkewMs, facts: lastFacts.f, sitesHosted: sites, jobs: { running } }); } catch (e) { log(`[agent] heartbeat 실패: ${e.message}`); } } + async function heartbeat() { try { + if (!lastFacts || Date.now() - lastFacts.at > 10 * 60000) lastFacts = { at: Date.now(), f: inventoryFacts(facts(sys)), inventorySynced: false }; + const managed = existsSync(cfg.roots.sites) ? readdirSync(cfg.roots.sites).filter((x) => existsSync(join(cfg.roots.sites, x, 'current'))) : []; + const legacy = lastFacts.f.webmakerInventory?.sites?.map((x) => x.id) || []; + const sites = new Set([...managed, ...legacy]).size; + if (lastFacts.f.webmakerInventory && !lastFacts.inventorySynced) { try { await syncWebmakerInventory(lastFacts.f.webmakerInventory); lastFacts.inventorySynced = true; } catch (e) { log(`[agent] Webmaker 인벤토리 전송 실패: ${e.message}`); } } + const heartbeatFacts = { ...lastFacts.f }; delete heartbeatFacts.webmakerInventory; + await client.request('POST', '/agent/v1/heartbeat', { version: VERSION, metrics: sample(cfg.roots), modules: await moduleStatus(lastFacts.f), clockSkewMs: client.clockSkewMs, facts: heartbeatFacts, sitesHosted: sites, jobs: { running } }); + } catch (e) { log(`[agent] heartbeat 실패: ${e.message}`); } } async function execute(job) { running++; try { const out = await runner.execute(job); if (out?.result?.restartAfterReport) restartUnit = out.result.restartAfterReport; } finally { running--; } if (restartUnit) { const u = restartUnit; restartUnit = null; log(`[agent] 자기 갱신 — ${u} 재시작`); setTimeout(() => sys.systemctl('restart', u, { optional: true }), 1500); } } // 재시작 뒤: 보고 못 한 결과 전송, 미완료 작업 재개 await runner.flushPending(); try { const u = await client.request('GET', '/agent/v1/jobs/unfinished'); for (const j of u.jobs || []) { log(`[agent] 미완료 작업 재개 ${j.id} ${j.kind}`); execute(j); } } catch (e) { log(`[agent] 미완료 작업 조회 실패: ${e.message}`); } diff --git a/agent/webmaker-inventory.mjs b/agent/webmaker-inventory.mjs new file mode 100644 index 0000000..cbdd974 --- /dev/null +++ b/agent/webmaker-inventory.mjs @@ -0,0 +1,116 @@ +/** + * agent/webmaker-inventory.mjs — 기존 Webmaker 사이트 디렉터리를 HAL 서버 인벤토리로 정리한다. + * + * 운영 정보만 보낸다. 페이지 본문, 프롬프트, 사용자 연락처, 인증정보와 비밀값은 결과에 + * 넣지 않는다. 사이트 작업 디렉터리도 존재할 수 있으므로 배포 이력을 함께 읽어 상태를 + * deployed/unpublished/draft/unknown으로 구분한다. 메타데이터 없는 디렉터리도 누락하지 않는다. + */ +import { existsSync, lstatSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const SITE_ID = /^[a-f0-9]{32}$/; +const text = (v, max = 240) => typeof v === 'string' ? v.replace(/[\u0000-\u001f\u007f]/g, '').slice(0, max) : null; +function domainsOf(site) { + const src = Array.isArray(site?.domains) ? site.domains : []; + return [...new Set(src.map((d) => typeof d === 'string' ? d : d?.domain || d?.hostname || d?.host || d?.name) + .map((d) => text(d, 253)?.toLowerCase()).filter(Boolean))].slice(0, 100); +} +function treeStat(root, errors, siteId) { + let bytes = 0; let files = 0; let complete = true; const stack = [root]; + const note = (e) => { complete = false; const code = e?.code || 'read_failed'; const marker = `site:${siteId}:${code}`; if (!errors.includes(marker)) errors.push(marker); }; + while (stack.length) { + const dir = stack.pop(); let entries = []; + try { entries = readdirSync(dir, { withFileTypes: true }); } catch (e) { note(e); continue; } + for (const e of entries) { + const p = join(dir, e.name); try { + if (e.isSymbolicLink()) continue; + if (e.isDirectory()) stack.push(p); + else if (e.isFile()) { bytes += lstatSync(p).size; files += 1; } + } catch (error) { note(error); } + } + } + return { bytes, files, complete }; +} +function deploymentState(site) { + if (!site) return 'unknown'; + if (site.unpublishedAt) return 'unpublished'; + const history = Array.isArray(site.deployments) ? site.deployments : []; + const lastLifecycle = [...history].reverse().find((d) => d && ((d.kind === 'unpublish' && d.ok !== false) || (d.ok && d.kind !== 'unpublish' && (d.ready === true || d.phase === 'ready' || d.phase === 'transferred' || !d.phase)))); + if (lastLifecycle?.kind === 'unpublish') return 'unpublished'; + if (lastLifecycle || site.deployedAt || site.publishedAt) return 'deployed'; + return 'draft'; +} +export function scanWebmakerInventory({ dataRoot, dataDir = null, sitesRoot, maxSites = Infinity } = {}) { + const metaRoot = dataDir || join(dataRoot || '', 'sites'); + let dirs = []; let metadataIds = []; let complete = true; const errors = []; + try { dirs = readdirSync(sitesRoot, { withFileTypes: true }).filter((e) => e.isDirectory() && SITE_ID.test(e.name)).map((e) => e.name); } + catch (e) { complete = false; errors.push(`sites:${e.code || 'read_failed'}`); } + try { metadataIds = readdirSync(metaRoot, { withFileTypes: true }).filter((e) => e.isFile() && SITE_ID.test(e.name.replace(/\.json$/i, '')) && /\.json$/i.test(e.name)).map((e) => e.name.slice(0, -5)); } catch (e) { complete = false; errors.push(`metadata:${e.code || 'read_failed'}`); } + const deployed = new Set(dirs); const ids = [...new Set([...dirs, ...metadataIds])].sort(); + const limit = Number.isFinite(maxSites) && maxSites >= 0 ? Math.floor(maxSites) : ids.length; + const selected = ids.slice(0, limit); + const sites = selected.map((id) => { + let site = null; let metadata = 'missing'; + const metadataFile = join(metaRoot, `${id}.json`); + try { + site = JSON.parse(readFileSync(metadataFile, 'utf8')); + if (!site || typeof site !== 'object' || Array.isArray(site)) { site = null; metadata = 'invalid'; const marker = `metadata:${id}:invalid_shape`; if (!errors.includes(marker)) errors.push(marker); } + else metadata = 'ok'; + } catch (e) { + if (e?.code === 'ENOENT' && !existsSync(metadataFile)) metadata = 'missing'; + else { metadata = 'invalid'; const marker = `metadata:${id}:${e?.code || 'invalid_json'}`; if (!errors.includes(marker)) errors.push(marker); } + } + const size = deployed.has(id) ? treeStat(join(sitesRoot, id), errors, id) : { bytes: 0, files: 0, complete: true }; + if (!size.complete) complete = false; + return { + id, + title: text(site?.title || site?.name || id, 200), + ownerId: text(site?.ownerId, 128), + status: text(site?.status, 80), + deploymentState: deploymentState(site), + domains: domainsOf(site), + updatedAt: text(site?.updatedAt || site?.deployedAt || site?.publishedAt, 40), + metadata, + bytes: size.bytes, + files: size.files, + }; + }); + if (sites.length < ids.length) complete = false; + return { + dataRoot: dataRoot || null, + dataDir: metaRoot || null, + sitesRoot: sitesRoot || null, + total: ids.length, + returned: sites.length, + truncated: sites.length < ids.length, + errors, + complete, + scannedAt: new Date().toISOString(), + sites, + }; +} + +/** 운영 경로는 Agent unit에 명시적으로 주입한다. systemd 설정을 추측·재해석해 다른 데이터 루트를 스캔하지 않는다. */ +export function configuredWebmakerInventory(facts, env = process.env) { + if (!facts?.units?.webmaker) return facts; + const dataRoot = env.ARIWEB_WEBMAKER_DATA_ROOT; const dataDir = env.ARIWEB_WEBMAKER_DATA_DIR; const sitesRoot = env.ARIWEB_WEBMAKER_SITES_ROOT; + if (![dataRoot, dataDir, sitesRoot].every((p) => typeof p === 'string' && p.startsWith('/'))) return { ...facts, webmakerInventory: { dataRoot: null, dataDir: null, sitesRoot: null, total: 0, returned: 0, truncated: false, errors: ['configuration:ARIWEB_WEBMAKER_DATA_ROOT_DATA_DIR_SITES_ROOT_required'], complete: false, scannedAt: new Date().toISOString(), sites: [] } }; + return { ...facts, webmakerInventory: scanWebmakerInventory({ dataRoot, dataDir, sitesRoot }) }; +} + +/** HAL JSON 본문 상한과 무관하게 전체 사이트를 byte·항목 수 기준 배치로 나눈다. */ +export function webmakerInventoryBatches(inv, maxBytes = 512 * 1024) { + if (!Number.isInteger(maxBytes) || maxBytes < 64 * 1024) throw new Error('인벤토리 배치 한도가 너무 작습니다'); + if (inv?.complete === false) throw new Error('inventory_incomplete'); + const sites = Array.isArray(inv?.sites) ? inv.sites : []; const rawErrors = Array.isArray(inv?.errors) ? inv.errors : []; const meta = { + dataRoot: String(inv?.dataRoot || '').slice(0, 4096), dataDir: String(inv?.dataDir || '').slice(0, 4096), sitesRoot: String(inv?.sitesRoot || '').slice(0, 4096), total: Number.isFinite(Number(inv?.total)) ? Number(inv.total) : sites.length, truncated: !!inv?.truncated, + errors: rawErrors.slice(0, 1000).map((x) => String(x).slice(0, 500)), errorsTruncated: rawErrors.length > 1000, scannedAt: String(inv?.scannedAt || new Date().toISOString()).slice(0, 80), + }; const scanId = meta.scannedAt || String(Date.now()); const out = []; let offset = 0; let part = []; + const body = (items) => ({ scanId, offset, itemTotal: sites.length, meta, sites: items }); + for (const site of sites) { + const candidate = [...part, site]; if (candidate.length > 1000 || Buffer.byteLength(JSON.stringify(body(candidate))) > maxBytes) { + if (!part.length) throw new Error('Webmaker 인벤토리 단일 항목이 배치 한도를 넘습니다'); out.push(body(part)); offset += part.length; part = [site]; if (Buffer.byteLength(JSON.stringify(body(part))) > maxBytes) throw new Error('Webmaker 인벤토리 단일 항목이 배치 한도를 넘습니다'); + } else part = candidate; + } + if (part.length || !sites.length) { const last = body(part); if (Buffer.byteLength(JSON.stringify(last)) > maxBytes) throw new Error('Webmaker 인벤토리 메타데이터가 배치 한도를 넘습니다'); out.push(last); } return out; +} diff --git a/hal/app.mjs b/hal/app.mjs index dcabd06..442e654 100644 --- a/hal/app.mjs +++ b/hal/app.mjs @@ -59,6 +59,7 @@ export function buildApp(d) { app.get('/agent/v1/bootstrap-artifact', wrap((ctx) => { const tok = String(ctx.headers['x-registration-token'] || ''); const t = d.store.get('SELECT * FROM registration_tokens WHERE token_hash=?', sha256(tok)); if (!t || t.used_at || Date.parse(t.expires_at) < Date.now()) throw new HttpError(401, 'bad_token', '등록 토큰이 올바르지 않거나 만료되었습니다'); const version = ctx.query.version || core.bootstrapState(d).release_version; const a = core.artifactPath(d, version, ctx.query.module || 'agent'); if (!a) throw new HttpError(404, 'artifact_not_found', 'artifact 가 없습니다'); ctx.ended = true; ctx.res.writeHead(200, { 'Content-Type': 'application/gzip', 'Content-Length': a.size, 'X-Sha256': a.sha256, 'Cache-Control': 'no-store' }); createReadStream(a.path).pipe(ctx.res); })); app.use(async (ctx) => { if (ctx.path.startsWith('/agent/v1/') && !['/agent/v1/register', '/agent/v1/install.sh', '/agent/v1/bootstrap-artifact'].includes(ctx.path)) await agentAuth(ctx); }); app.post('/agent/v1/heartbeat', wrap(async (ctx) => core.heartbeat(d, ctx.agent, await ctx.body()))); + app.post('/agent/v1/inventory/webmaker', wrap(async (ctx) => core.webmakerInventoryBatch(d, ctx.agent, await ctx.body()))); app.get('/agent/v1/jobs/next', wrap(async (ctx) => { const wait = Math.min(25, Number(ctx.query.wait || 0)); const until = Date.now() + wait * 1000; while (true) { const j = core.nextAgentJob(d, ctx.agent.id); if (j || Date.now() >= until || d.stopping) return { ok: true, job: j || null, serverTime: nowIso() }; await new Promise((r) => setTimeout(r, 500)); } })); app.get('/agent/v1/jobs/unfinished', wrap((ctx) => ({ ok: true, jobs: core.unfinishedAgentJobs(d, ctx.agent.id) }))); app.post('/agent/v1/jobs/:id/progress', wrap(async (ctx) => { const b = await ctx.body(); const ok = core.agentJobProgress(d, ctx.params.id, ctx.agent.id, b); return { ok, cancelRequested: core.isAgentJobCancelled(d, ctx.params.id) }; })); diff --git a/hal/core.mjs b/hal/core.mjs index 68321b4..d9537a1 100644 --- a/hal/core.mjs +++ b/hal/core.mjs @@ -136,13 +136,44 @@ export function externalAgentStatus(d, id) { lastDeploy: last ? { id: last.id, status: last.status, at: last.ended_at || last.started_at || last.created_at, result: parseJson(last.result_json, null), error: last.error } : null, lastCheck: lastCheck ? { at: lastCheck.ended_at, result: parseJson(lastCheck.result_json, null) } : null }; } +/** Webmaker 인벤토리를 2 MiB HTTP 한도 아래의 순차 배치로 받아 마지막 배치에서만 공개 facts에 원자 반영한다. */ +export function webmakerInventoryBatch(d, agent, body = {}) { + if (!agent.server_id || agent.kind !== 'managed') throw new JobError('inventory_not_allowed', '관리 서버 Agent만 인벤토리를 제출할 수 있습니다'); + const scanId = String(body.scanId || ''); const offset = Number(body.offset); const itemTotal = Number(body.itemTotal); const sites = body.sites; + if (!scanId || scanId.length > 100 || !Number.isInteger(offset) || offset < 0 || !Number.isInteger(itemTotal) || itemTotal < 0 || itemTotal > 10000 || !Array.isArray(sites) || sites.length > 1000) throw new JobError('bad_inventory_batch', '인벤토리 배치 형식이 올바르지 않습니다'); + const safeText = (v, n) => typeof v === 'string' ? v.replace(/[\u0000-\u001f\u007f]/g, '').slice(0, n) : null; + const safeCount = (v) => Number.isSafeInteger(Number(v)) && Number(v) >= 0 ? Number(v) : 0; + const states = new Set(['deployed', 'unpublished', 'draft', 'unknown']); const metadataStates = new Set(['ok', 'missing', 'invalid']); + const cleanSites = sites.map((site) => { + if (!site || typeof site !== 'object' || Array.isArray(site) || !/^[a-f0-9]{32}$/.test(String(site.id || ''))) throw new JobError('bad_inventory_batch', '사이트 항목 형식이 올바르지 않습니다'); + const deploymentState = states.has(site.deploymentState) ? site.deploymentState : 'unknown'; const metadata = metadataStates.has(site.metadata) ? site.metadata : 'missing'; + const domains = Array.isArray(site.domains) ? [...new Set(site.domains.map((x) => safeText(x, 253)?.toLowerCase()).filter(Boolean))].slice(0, 100) : []; + return { id: site.id, title: safeText(site.title, 200), ownerId: safeText(site.ownerId, 128), status: safeText(site.status, 80), deploymentState, domains, updatedAt: safeText(site.updatedAt, 40), metadata, bytes: safeCount(site.bytes), files: safeCount(site.files) }; + }); + const metaIn = body.meta && typeof body.meta === 'object' && !Array.isArray(body.meta) ? body.meta : {}; const safeTotal = Number(metaIn.total); const meta = { dataRoot: safeText(metaIn.dataRoot, 4096), dataDir: safeText(metaIn.dataDir, 4096), sitesRoot: safeText(metaIn.sitesRoot, 4096), total: Number.isSafeInteger(safeTotal) && safeTotal >= 0 && safeTotal <= 10000 ? safeTotal : itemTotal, truncated: !!metaIn.truncated, errors: Array.isArray(metaIn.errors) ? metaIn.errors.slice(0, 1000).map((x) => safeText(x, 500)).filter(Boolean) : [], errorsTruncated: !!metaIn.errorsTruncated, scannedAt: safeText(metaIn.scannedAt, 100) }; + if (meta.total !== itemTotal || meta.truncated) throw new JobError('inventory_incomplete', '완전한 Webmaker 인벤토리만 반영할 수 있습니다'); + return d.store.tx(() => { + d.store.run('DELETE FROM agent_inventory_staging WHERE updated_at < ?', new Date(Date.now() - 24 * 3600 * 1000).toISOString()); + let row = d.store.get('SELECT * FROM agent_inventory_staging WHERE agent_id=? AND kind=?', agent.id, 'webmaker'); + if (offset === 0) { d.store.run('INSERT INTO agent_inventory_staging(agent_id, kind, scan_id, total, next_offset, meta_json, items_json, updated_at) VALUES(?,?,?,?,0,?,?,?) ON CONFLICT(agent_id, kind) DO UPDATE SET scan_id=excluded.scan_id,total=excluded.total,next_offset=0,meta_json=excluded.meta_json,items_json=\'[]\',updated_at=excluded.updated_at', agent.id, 'webmaker', scanId, itemTotal, toJson(meta), '[]', nowIso()); row = d.store.get('SELECT * FROM agent_inventory_staging WHERE agent_id=? AND kind=?', agent.id, 'webmaker'); } + if (!row || row.scan_id !== scanId || row.total !== itemTotal || row.next_offset !== offset) throw new JobError('inventory_batch_order', '인벤토리 배치 순서가 맞지 않습니다 — 처음부터 다시 보내세요'); + const items = parseJson(row.items_json, []); items.push(...cleanSites); if (items.length > itemTotal || offset + cleanSites.length !== items.length) throw new JobError('bad_inventory_batch', '인벤토리 항목 수가 선언값과 다릅니다'); const itemsJson = toJson(items); if (Buffer.byteLength(itemsJson) > 16 * 1024 * 1024) throw new JobError('inventory_too_large', '인벤토리 전체 크기가 안전 한도를 넘습니다'); + const complete = items.length === itemTotal; d.store.run('UPDATE agent_inventory_staging SET next_offset=?, items_json=?, updated_at=? WHERE agent_id=? AND kind=?', items.length, itemsJson, nowIso(), agent.id, 'webmaker'); + if (!complete) return { ok: true, complete: false, nextOffset: items.length }; + const inventory = { ...parseJson(row.meta_json, {}), returned: items.length, sites: items }; const aFacts = parseJson(d.store.get('SELECT facts_json FROM agents WHERE id=?', agent.id)?.facts_json, {}) || {}; aFacts.webmakerInventory = inventory; + const sFacts = parseJson(d.store.get('SELECT facts_json FROM servers WHERE id=?', agent.server_id)?.facts_json, {}) || {}; sFacts.webmakerInventory = inventory; + d.store.run('UPDATE agents SET facts_json=? WHERE id=?', toJson(aFacts), agent.id); d.store.run('UPDATE servers SET facts_json=?, updated_at=? WHERE id=?', toJson(sFacts), nowIso(), agent.server_id); d.store.run('DELETE FROM agent_inventory_staging WHERE agent_id=? AND kind=?', agent.id, 'webmaker'); + return { ok: true, complete: true, nextOffset: items.length }; + }); +} /** heartbeat 처리: 지표 저장, 서버 last_seen·시계 차이·모듈 실제 버전·용량 갱신 */ export function heartbeat(d, agent, body) { const now = nowIso(); const m = body.metrics || {}; const skew = Number.isFinite(body.clockSkewMs) ? Math.round(body.clockSkewMs) : null; - d.store.run('UPDATE agents SET last_seen_at=?, version=?, facts_json=COALESCE(?, facts_json) WHERE id=?', now, body.version || agent.version, body.facts ? toJson(redact(body.facts)) : null, agent.id); + const mergeFacts = (currentText) => { if (!body.facts) return null; const current = parseJson(currentText, {}) || {}; const incoming = redact(body.facts); if (!Object.hasOwn(incoming, 'webmakerInventory') && current.webmakerInventory) incoming.webmakerInventory = current.webmakerInventory; return toJson(incoming); }; + const agentFacts = mergeFacts(agent.facts_json); d.store.run('UPDATE agents SET last_seen_at=?, version=?, facts_json=COALESCE(?, facts_json) WHERE id=?', now, body.version || agent.version, agentFacts, agent.id); if (!agent.server_id) return { ok: true }; const capacity = { diskFreeBytes: m.disk ? m.disk.total - m.disk.used : 0, diskTotalBytes: m.disk?.total || 0, memFreeBytes: m.mem ? m.mem.total - m.mem.used : 0, cpuLoad: m.load1 || 0, sitesHosted: body.sitesHosted || 0, reservedBytes: body.reservedBytes || 0, exhaustEta: body.exhaustEta || null }; - d.store.run('UPDATE servers SET last_seen_at=?, clock_skew_ms=?, agent_version=?, metrics_json=?, capacity_json=?, facts_json=COALESCE(?, facts_json), status=CASE WHEN status=\'offline\' THEN \'ready\' ELSE status END, updated_at=? WHERE id=?', now, skew, body.version || null, toJson(m), toJson(capacity), body.facts ? toJson(redact(body.facts)) : null, now, agent.server_id); + const serverRow = d.store.get('SELECT facts_json FROM servers WHERE id=?', agent.server_id); const serverFacts = mergeFacts(serverRow?.facts_json); d.store.run('UPDATE servers SET last_seen_at=?, clock_skew_ms=?, agent_version=?, metrics_json=?, capacity_json=?, facts_json=COALESCE(?, facts_json), status=CASE WHEN status=\'offline\' THEN \'ready\' ELSE status END, updated_at=? WHERE id=?', now, skew, body.version || null, toJson(m), toJson(capacity), serverFacts, now, agent.server_id); d.store.run('INSERT OR REPLACE INTO metrics_samples(server_id, at, cpu, mem_used, mem_total, disk_used, disk_total, inode_used, inode_total, io_read, io_write, net_in, net_out, clock_skew_ms, load1) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', agent.server_id, now, m.cpu ?? null, m.mem?.used ?? null, m.mem?.total ?? null, m.disk?.used ?? null, m.disk?.total ?? null, m.inode?.used ?? null, m.inode?.total ?? null, m.io?.read ?? null, m.io?.write ?? null, m.net?.in ?? null, m.net?.out ?? null, skew, m.load1 ?? null); for (const [name, mod] of Object.entries(body.modules || {})) { diff --git a/hal/schema.mjs b/hal/schema.mjs index 4d4aee1..eb8e08b 100644 --- a/hal/schema.mjs +++ b/hal/schema.mjs @@ -88,4 +88,20 @@ CREATE TABLE IF NOT EXISTS portal_events_outbox (id TEXT PRIMARY KEY, payload_js { id: 'hal_0002_live_host', sql: `ALTER TABLE bootstrap_state ADD COLUMN live_host TEXT; ALTER TABLE bootstrap_state ADD COLUMN live_pid TEXT;` }, // [legacy 메일 서비스 가져오기] HAL 관리형 메일 서비스 등록부·가져오기 단위(hal/mailsvc.mjs). 표 추가만(기존 표 변경 없음). ...MAIL_MIGRATIONS, + // Webmaker 전체 인벤토리는 heartbeat 2 MiB 한도를 넘을 수 있어 인증된 배치로 원자적으로 조립한다. + { id: 'hal_0004_webmaker_inventory_stage', sql: ` +CREATE TABLE IF NOT EXISTS agent_inventory_staging ( + agent_id TEXT NOT NULL REFERENCES agents(id) ON DELETE CASCADE, kind TEXT NOT NULL, scan_id TEXT NOT NULL, total INTEGER NOT NULL, next_offset INTEGER NOT NULL DEFAULT 0, + meta_json TEXT NOT NULL, items_json TEXT NOT NULL DEFAULT '[]', updated_at TEXT NOT NULL, + PRIMARY KEY(agent_id, kind) +); +` }, + // 사전 릴리스에서 FK 없이 같은 staging 표를 만든 경우에도 고아 row를 정리하고 이후 삭제를 보장한다. + { id: 'hal_0005_webmaker_inventory_cleanup', sql: ` +DELETE FROM agent_inventory_staging WHERE agent_id NOT IN (SELECT id FROM agents); +CREATE TRIGGER IF NOT EXISTS agent_inventory_staging_agent_cleanup +AFTER DELETE ON agents BEGIN + DELETE FROM agent_inventory_staging WHERE agent_id=OLD.id; +END; +` }, ]; diff --git a/hal/ui/render.mjs b/hal/ui/render.mjs index ef917f3..ed8260e 100644 --- a/hal/ui/render.mjs +++ b/hal/ui/render.mjs @@ -55,10 +55,21 @@ export function serversTable(servers) { ], servers, { empty: '등록된 서버가 없습니다' }); } export function serverDetail(s, { jobs = [], agentJobs = [] } = {}) { + const webmakerData = s.facts?.webmakerInventory || null; const webmakerSites = Array.isArray(webmakerData?.sites) ? webmakerData.sites.filter((x) => x && typeof x === 'object' && !Array.isArray(x)) : []; const inventoryErrors = Array.isArray(webmakerData?.errors) ? webmakerData.errors : []; + const webmakerInventory = webmakerData ? `
Node Agent가 사이트 디렉터리와 Webmaker 메타데이터·배포 이력을 대조한 운영 인벤토리입니다. 전체 ${Number(webmakerData.total || 0).toLocaleString('ko-KR')}개 · 수집 ${Number(webmakerData.returned || 0).toLocaleString('ko-KR')}개${webmakerData.truncated ? ' · 일부만 표시됨' : ''}. 페이지 본문·프롬프트·연락처·인증정보는 HAL로 보내지 않습니다.
${inventoryErrors.length || webmakerData.errorsTruncated ? statePanel('failed', `인벤토리 스캔 오류 ${inventoryErrors.length}건${webmakerData.errorsTruncated ? ' 이상(전송 한도에 따라 일부 오류만 표시)' : ''}: ${inventoryErrors.join(', ')}`) : ''}${table([ + { l: '사이트', f: (x) => `${esc(x.id)}