From 6b5a014041c447ed58a414a8ff588221a5c7514b Mon Sep 17 00:00:00 2001 From: zzabisagent-lab Date: Mon, 26 Oct 2026 11:58:15 +0000 Subject: [PATCH] =?UTF-8?q?HAL=20=EC=A0=9C=EC=96=B4=20=ED=8F=89=EB=A9=B4(?= =?UTF-8?q?=EB=A6=B4=EB=A6=AC=EC=8A=A4=C2=B7=EC=84=9C=EB=B2=84=C2=B7Agent?= =?UTF-8?q?=C2=B7=EC=9E=91=EC=97=85=C2=B7=EB=AA=A8=EB=93=88=20=EB=B0=B0?= =?UTF-8?q?=ED=8F=AC=C2=B7=EC=9A=B4=EC=98=81=EC=9E=90=20=EA=B3=84=EC=A0=95?= =?UTF-8?q?=C2=B7=EA=B8=B4=EA=B8=89=20=EB=B3=B5=EA=B5=AC=C2=B7=EC=9E=AC?= =?UTF-8?q?=ED=95=B4=EB=B3=B5=EA=B5=AC=C2=B7=EA=B3=B5=EA=B0=9C=20=EC=83=81?= =?UTF-8?q?=ED=83=9C),=20Node=20Agent,=20=EB=B0=B0=ED=8F=AC=20=EC=8A=A4?= =?UTF-8?q?=ED=81=AC=EB=A6=BD=ED=8A=B8=C2=B7=EC=9C=A0=EB=8B=9B=C2=B7nginx?= =?UTF-8?q?=20=ED=85=9C=ED=94=8C=EB=A6=BF,=20=EB=A6=B4=EB=A6=AC=EC=8A=A4?= =?UTF-8?q?=20=EB=8F=84=EA=B5=AC,=20=ED=8F=AC=ED=84=B8=20HAL=20=ED=81=B4?= =?UTF-8?q?=EB=9D=BC=EC=9D=B4=EC=96=B8=ED=8A=B8=20(=C2=A728=C2=B7=C2=A729?= =?UTF-8?q?=C2=B7=C2=A732~=C2=A734)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- agent/client.mjs | 40 ++++ agent/config.mjs | 32 +++ agent/index.mjs | 50 +++++ agent/jobs/backup.mjs | 50 +++++ agent/jobs/common.mjs | 66 +++++++ agent/jobs/domain.mjs | 27 +++ agent/jobs/index.mjs | 6 + agent/jobs/module.mjs | 67 +++++++ agent/jobs/site.mjs | 114 +++++++++++ agent/metrics.mjs | 25 +++ agent/runner.mjs | 57 ++++++ agent/sys.mjs | 36 ++++ deploy/agent-install.sh | 27 +++ deploy/bootstrap-hal.sh | 57 ++++++ deploy/hal-restore.sh | 16 ++ deploy/nginx/ariweb-hal.conf.tmpl | 31 +++ deploy/nginx/ariweb-portal.conf.tmpl | 29 +++ deploy/nginx/awstats.conf | 3 + deploy/nginx/redirect.tmpl | 7 + deploy/nginx/site-body-node.tmpl | 7 + deploy/nginx/site-body-static.tmpl | 11 ++ deploy/nginx/site-https.tmpl | 12 ++ deploy/nginx/site.conf.tmpl | 11 ++ deploy/sudoers/ariweb-agent | 9 + deploy/units/arimail.service.tmpl | 31 +++ deploy/units/ariweb-agent.service.tmpl | 24 +++ deploy/units/ariweb-hal.service.tmpl | 29 +++ deploy/units/ariweb-portal.service.tmpl | 27 +++ deploy/units/ariweb-site.service.tmpl | 25 +++ deploy/units/webmaker.service.tmpl | 25 +++ docs/HAL-AGENT.md | 69 +++++++ hal/app.mjs | 186 ++++++++++++++++++ hal/cidr.mjs | 21 ++ hal/config.mjs | 41 ++++ hal/core.mjs | 249 ++++++++++++++++++++++++ hal/deps.mjs | 24 +++ hal/dr.mjs | 90 +++++++++ hal/emergency.mjs | 72 +++++++ hal/index.mjs | 55 ++++++ hal/jobs.mjs | 167 ++++++++++++++++ hal/modules/index.mjs | 39 ++++ hal/ops.mjs | 69 +++++++ hal/schema.mjs | 85 ++++++++ hal/totp.mjs | 26 +++ lang/ko/hal.json | 13 ++ portal/hal/client.mjs | 46 +++++ scripts/release.mjs | 52 +++++ test/agent.test.mjs | 116 +++++++++++ test/hal.test.mjs | 213 ++++++++++++++++++++ 49 files changed, 2584 insertions(+) create mode 100644 agent/client.mjs create mode 100644 agent/config.mjs create mode 100644 agent/index.mjs create mode 100644 agent/jobs/backup.mjs create mode 100644 agent/jobs/common.mjs create mode 100644 agent/jobs/domain.mjs create mode 100644 agent/jobs/index.mjs create mode 100644 agent/jobs/module.mjs create mode 100644 agent/jobs/site.mjs create mode 100644 agent/metrics.mjs create mode 100644 agent/runner.mjs create mode 100644 agent/sys.mjs create mode 100755 deploy/agent-install.sh create mode 100755 deploy/bootstrap-hal.sh create mode 100755 deploy/hal-restore.sh create mode 100644 deploy/nginx/ariweb-hal.conf.tmpl create mode 100644 deploy/nginx/ariweb-portal.conf.tmpl create mode 100644 deploy/nginx/awstats.conf create mode 100644 deploy/nginx/redirect.tmpl create mode 100644 deploy/nginx/site-body-node.tmpl create mode 100644 deploy/nginx/site-body-static.tmpl create mode 100644 deploy/nginx/site-https.tmpl create mode 100644 deploy/nginx/site.conf.tmpl create mode 100644 deploy/sudoers/ariweb-agent create mode 100644 deploy/units/arimail.service.tmpl create mode 100644 deploy/units/ariweb-agent.service.tmpl create mode 100644 deploy/units/ariweb-hal.service.tmpl create mode 100644 deploy/units/ariweb-portal.service.tmpl create mode 100644 deploy/units/ariweb-site.service.tmpl create mode 100644 deploy/units/webmaker.service.tmpl create mode 100644 docs/HAL-AGENT.md create mode 100644 hal/app.mjs create mode 100644 hal/cidr.mjs create mode 100644 hal/config.mjs create mode 100644 hal/core.mjs create mode 100644 hal/deps.mjs create mode 100644 hal/dr.mjs create mode 100644 hal/emergency.mjs create mode 100644 hal/index.mjs create mode 100644 hal/jobs.mjs create mode 100644 hal/modules/index.mjs create mode 100644 hal/ops.mjs create mode 100644 hal/schema.mjs create mode 100644 hal/totp.mjs create mode 100644 lang/ko/hal.json create mode 100644 portal/hal/client.mjs create mode 100644 scripts/release.mjs create mode 100644 test/agent.test.mjs create mode 100644 test/hal.test.mjs diff --git a/agent/client.mjs b/agent/client.mjs new file mode 100644 index 0000000..12145a9 --- /dev/null +++ b/agent/client.mjs @@ -0,0 +1,40 @@ +/** + * agent/client.mjs — HAL 호출(서명·시계 차이·artifact 내려받기·전송 올리기/내려받기). + * 모든 요청은 lib/serviceauth 로 서명한다(keyId 'agent:'). 응답의 Date/X-HAL-Time 으로 시계 차이를 재어 heartbeat 에 싣는다. + * 큰 파일 업로드는 본문 해시를 서명에 넣지 않고(빈 본문 서명) X-Transfer-Sha256 헤더로 무결성을 전한다(HAL 이 받은 뒤 다시 계산해 비교). + */ +import { signHeaders } from '../lib/serviceauth.mjs'; +import { createHash } from 'node:crypto'; +import { createWriteStream, createReadStream, statSync } from 'node:fs'; +import { pipeline } from 'node:stream/promises'; +import { Readable } from 'node:stream'; + +export function createClient({ halUrl, agentId, secret, fetchImpl = fetch }) { + const base = String(halUrl).replace(/\/$/, ''); const keyId = `agent:${agentId}`; let clockSkewMs = 0; + function skew(r) { const t = Number(r.headers.get('x-hal-time') || 0) || Date.parse(r.headers.get('date') || ''); if (t) clockSkewMs = Date.now() - t; } + async function request(method, path, body, { timeoutMs = 30000 } = {}) { + const text = body == null ? undefined : JSON.stringify(body); + const r = await fetchImpl(base + path, { method, headers: signHeaders(keyId, secret, method, path, text), body: text, signal: AbortSignal.timeout(timeoutMs) }); skew(r); + let json = null; try { json = await r.json(); } catch (_) {} + if (!r.ok || (json && json.ok === false)) { const e = new Error(`HAL ${method} ${path} → ${r.status} ${json?.error || ''} ${json?.message || ''}`.trim()); e.status = r.status; e.code = json?.error || `http_${r.status}`; throw e; } + return json; + } + /** 파일 내려받기 + sha256 검증(expected 가 있으면) */ + async function download(path, dest, expected = null) { + const r = await fetchImpl(base + path, { headers: signHeaders(keyId, secret, 'GET', path, undefined), signal: AbortSignal.timeout(3600000) }); skew(r); + if (!r.ok) { const e = new Error(`HAL 내려받기 실패 ${path} → ${r.status}`); e.code = `http_${r.status}`; throw e; } + const hash = createHash('sha256'); const src = Readable.fromWeb(r.body); src.on('data', (c) => hash.update(c)); await pipeline(src, createWriteStream(dest, { mode: 0o600 })); + const sum = hash.digest('hex'); const want = String(expected || r.headers.get('x-sha256') || '').replace(/^sha256:/i, '').toLowerCase(); if (want && want !== sum) throw Object.assign(new Error(`내려받은 파일 해시가 다릅니다(기대 ${want.slice(0, 12)}… 실제 ${sum.slice(0, 12)}…)`), { code: 'checksum_mismatch' }); + return { sha256: sum, size: statSync(dest).size }; + } + /** 전송 올리기(스트리밍). 반환 { id, sha256, size } */ + async function upload(transferId, file, { purpose = '' } = {}) { + const hash = createHash('sha256'); for await (const c of createReadStream(file)) hash.update(c); const sum = hash.digest('hex'); const size = statSync(file).size; + const path = `/agent/v1/transfers/${transferId}${purpose ? `?purpose=${encodeURIComponent(purpose)}` : ''}`; + const r = await fetchImpl(base + path, { method: 'POST', headers: { ...signHeaders(keyId, secret, 'POST', path, ''), 'Content-Type': 'application/octet-stream', 'Content-Length': String(size), 'X-Transfer-Sha256': sum }, body: createReadStream(file), duplex: 'half', signal: AbortSignal.timeout(3600000) }); skew(r); + const json = await r.json().catch(() => null); if (!r.ok) throw Object.assign(new Error(`전송 올리기 실패 ${r.status} ${json?.error || ''}`), { code: json?.error || `http_${r.status}` }); + return { id: transferId, sha256: sum, size }; + } + const downloadTransfer = (id, dest, expected = null) => download(`/agent/v1/transfers/${id}`, dest, expected); + return { request, download, upload, downloadTransfer, get clockSkewMs() { return clockSkewMs; }, agentId, base }; +} diff --git a/agent/config.mjs b/agent/config.mjs new file mode 100644 index 0000000..c43b165 --- /dev/null +++ b/agent/config.mjs @@ -0,0 +1,32 @@ +/** + * agent/config.mjs — Node Agent 설정과 경로. + * 데이터 디렉터리(기본 /srv/ariweb/agent/data)의 agent.json 에 id·HAL 주소·역할·사이트(외부 모드)를 두고, 비밀은 같은 디렉터리의 보관소(lib/vault)에 'agent-secret' 로 둔다. + * 실제 시스템 경로는 환경 변수로 바꿀 수 있어 테스트는 임시 디렉터리에서 돈다. + */ +import { readFileSync, writeFileSync, existsSync, mkdirSync, renameSync } from 'node:fs'; +import { join } from 'node:path'; +import { openVault } from '../lib/vault.mjs'; + +export function loadAgentConfig(env = process.env) { + const dataDir = env.ARIWEB_AGENT_DATA || '/srv/ariweb/agent/data'; mkdirSync(dataDir, { recursive: true, mode: 0o700 }); + const file = join(dataDir, 'agent.json'); let saved = {}; try { saved = existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')) : {}; } catch (_) {} + const cfg = { + dataDir, file, ...saved, + roots: { + sites: env.ARIWEB_SITES_ROOT || '/var/www/ariweb-sites', // 사이트 파일: //releases/, current, data(uploads) + srv: env.ARIWEB_SRV_ROOT || '/srv/ariweb', // 모듈 설치: //releases/, current + nginx: env.ARIWEB_NGINX_DIR || '/etc/nginx', // sites-available/sites-enabled + backups: env.ARIWEB_BACKUP_ROOT || '/srv/ariweb/backups', // 백업 저장소 + logs: env.ARIWEB_LOG_DIR || '/var/log/nginx', // 사이트별 접속 로그 aw-.access.log + acme: env.ARIWEB_ACME_WEBROOT || '/var/www/letsencrypt', // certbot webroot + letsencrypt: env.ARIWEB_LE_DIR || '/etc/letsencrypt/live', + wildcardCert: env.ARIWEB_WILDCARD_CERT || '', // 플랫폼 와일드카드 인증서 디렉터리(예: /etc/letsencrypt/live/wh-wild-ariweb.net) + platformDomain: env.ARIWEB_PLATFORM_DOMAIN || 'ariweb.net', + maintPage: env.ARIWEB_MAINT_PAGE || '/var/www/ariweb-maint.html', + }, + heartbeatMs: Number(env.ARIWEB_AGENT_HEARTBEAT_MS || 30000), + save(patch) { Object.assign(cfg, patch); const out = {}; for (const k of ['id', 'halUrl', 'serverId', 'kind', 'siteId', 'groupId', 'registeredAt', 'label']) if (cfg[k] !== undefined) out[k] = cfg[k]; const tmp = file + '.tmp'; writeFileSync(tmp, JSON.stringify(out, null, 1), { mode: 0o600 }); renameSync(tmp, file); }, + vault: () => openVault(dataDir), + }; + return cfg; +} diff --git a/agent/index.mjs b/agent/index.mjs new file mode 100644 index 0000000..88a2c01 --- /dev/null +++ b/agent/index.mjs @@ -0,0 +1,50 @@ +/** + * agent/index.mjs — Node Agent 진입점·CLI(§5·§28.2·§10.2). + * + * node agent/index.mjs register --hal --token [--role web|mail|backup|control] [--external --site ] [--label <이름>] + * node agent/index.mjs run 서비스 루프: heartbeat(30초) + 작업 long-poll + 재시작 뒤 미완료 작업 재개 + * node agent/index.mjs status 등록 상태·HAL 연결 확인 + * 비밀(Agent secret)은 데이터 디렉터리의 보관소에만 둔다. 등록 응답의 비밀은 그 자리에서 저장하고 화면에 보이지 않는다. + */ +import { loadAgentConfig } from './config.mjs'; +import { createClient } from './client.mjs'; +import { createSys } from './sys.mjs'; +import { createRunner } from './runner.mjs'; +import { KINDS } from './jobs/index.mjs'; +import { sample, facts } from './metrics.mjs'; +import { readFileSync, existsSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { hostname } from 'node:os'; + +const VERSION = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version; +const log = (...a) => console.log(new Date().toISOString(), ...a); +const args = process.argv.slice(2); const cmd = args[0] || 'run'; const opt = (n, d = null) => { const i = args.indexOf('--' + n); return i >= 0 ? (args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : true) : d; }; +const cfg = loadAgentConfig(); const sys = createSys({ log }); + +if (cmd === 'register') { + const hal = opt('hal'); const token = opt('token'); if (!hal || !token) { console.error('--hal 과 --token 이 필요합니다'); process.exit(1); } + const f = facts(sys); f.role = opt('role', 'web'); f.location = opt('location', ''); + const r = await fetch(hal.replace(/\/$/, '') + '/agent/v1/register', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ token, facts: f, version: VERSION }) }); const j = await r.json().catch(() => ({})); + if (!r.ok || !j.ok) { console.error(`등록 실패: ${j.error || r.status} ${j.message || ''}`); process.exit(1); } + cfg.vault().set('agent-secret', j.secret, { kind: 'agent-secret' }); cfg.save({ id: j.agentId, halUrl: hal, serverId: j.serverId || null, kind: j.kind, siteId: j.siteId || null, groupId: j.groupId || null, registeredAt: new Date().toISOString(), label: opt('label', hostname()) }); + console.log(`등록 완료: agent ${j.agentId} (${j.kind}${j.serverId ? `, 서버 ${j.serverId}` : ''}). 비밀은 ${cfg.dataDir} 의 보관소에 저장했습니다. 운영자 승인 뒤 배치가 가능합니다`); +} else if (cmd === 'status') { + if (!cfg.id) { console.log('등록되지 않음'); process.exit(1); } const client = createClient({ halUrl: cfg.halUrl, agentId: cfg.id, secret: cfg.vault().get('agent-secret') }); + try { const r = await client.request('POST', '/agent/v1/heartbeat', { version: VERSION, metrics: sample(cfg.roots), modules: {}, clockSkewMs: 0 }); console.log(`연결 정상: agent ${cfg.id} (${cfg.kind}) → ${cfg.halUrl}, 대기 작업 ${r.pendingJobs}, 시계 차이 ${client.clockSkewMs}ms`); } catch (e) { console.error(`HAL 연결 실패: ${e.message}`); process.exit(1); } +} else if (cmd === 'run') { + if (!cfg.id) { console.error('등록되지 않았습니다 — register 먼저'); process.exit(1); } + const client = createClient({ halUrl: cfg.halUrl, agentId: cfg.id, secret: cfg.vault().get('agent-secret') }); + const runner = createRunner({ kinds: KINDS, roots: cfg.roots, sys, client, dataDir: cfg.dataDir, log }); + let stopping = false; let running = 0; let lastFacts = null; let restartUnit = null; + /** 모듈 상태(heartbeat): facts 의 유닛이 살아 있는지 + current/package.json 버전 */ + async function moduleStatus(f) { const out = {}; for (const [m, unit] of Object.entries(f.units || {})) { const appDir = (f.appDirs || {})[m] || join(cfg.roots.srv, m); const cur = join(appDir, 'current', 'package.json'); let version = null; try { version = JSON.parse(readFileSync(cur, 'utf8')).version; } catch (_) {} const active = await sys.isActive(unit); out[m] = { version, active, health: active ? 'ok' : 'down' }; } out.agent = { version: VERSION, active: true, health: 'ok' }; out.observability = { version: VERSION, active: true, health: 'ok' }; if (existsSync(cfg.roots.backups)) out.backup = { version: VERSION, active: true, health: 'ok' }; return out; } + async function heartbeat() { try { if (!lastFacts || Date.now() - lastFacts.at > 10 * 60000) lastFacts = { at: Date.now(), f: facts(sys) }; const sites = existsSync(cfg.roots.sites) ? readdirSync(cfg.roots.sites).filter((x) => existsSync(join(cfg.roots.sites, x, 'current'))).length : 0; await client.request('POST', '/agent/v1/heartbeat', { version: VERSION, metrics: sample(cfg.roots), modules: await moduleStatus(lastFacts.f), clockSkewMs: client.clockSkewMs, facts: lastFacts.f, sitesHosted: sites, jobs: { running } }); } catch (e) { log(`[agent] heartbeat 실패: ${e.message}`); } } + async function execute(job) { running++; try { const out = await runner.execute(job); if (out?.result?.restartAfterReport) restartUnit = out.result.restartAfterReport; } finally { running--; } if (restartUnit) { const u = restartUnit; restartUnit = null; log(`[agent] 자기 갱신 — ${u} 재시작`); setTimeout(() => sys.systemctl('restart', u, { optional: true }), 1500); } } + // 재시작 뒤: 보고 못 한 결과 전송, 미완료 작업 재개 + await runner.flushPending(); try { const u = await client.request('GET', '/agent/v1/jobs/unfinished'); for (const j of u.jobs || []) { log(`[agent] 미완료 작업 재개 ${j.id} ${j.kind}`); execute(j); } } catch (e) { log(`[agent] 미완료 작업 조회 실패: ${e.message}`); } + await heartbeat(); const hb = setInterval(heartbeat, cfg.heartbeatMs); + process.on('SIGTERM', () => { stopping = true; clearInterval(hb); }); + log(`[agent] Ariweb Node Agent v${VERSION} ${cfg.id} (${cfg.kind}) → ${cfg.halUrl}`); + while (!stopping) { try { const r = await client.request('GET', '/agent/v1/jobs/next?wait=25', undefined, { timeoutMs: 40000 }); if (r.job) execute(r.job); } catch (e) { log(`[agent] 작업 조회 실패: ${e.message}`); await new Promise((s) => setTimeout(s, 5000)); } } + process.exit(0); +} else { console.error(`알 수 없는 명령: ${cmd}`); process.exit(1); } diff --git a/agent/jobs/backup.mjs b/agent/jobs/backup.mjs new file mode 100644 index 0000000..626e04f --- /dev/null +++ b/agent/jobs/backup.mjs @@ -0,0 +1,50 @@ +/** + * agent/jobs/backup.mjs — 관리형 사이트 백업·복원(§19·§20). + * + * 백업 묶음(tar.gz) 구조: files/(현재 배포본) data/(uploads 등 영속 데이터) db/(SQLite 는 VACUUM INTO 일관 사본; 그 밖의 DB 는 payload.db.dumpCommand 가 있을 때만) config/(vhost·버전 정보·ariweb.json) meta.json + * 메일 원문·소스 저장소·Webmaker 이력은 포함하지 않는다(§19.1). 만든 뒤 목록을 읽어 검증하고 sha256 을 돌려준다. + * 저장소: //.tar.gz. 운영 서버와 다른 저장 서버로 보내야 하면 uploadToHal 로 HAL 전송에 올린다(backup.store 가 저장 서버에서 받는다). + * 복원: 보호 시점(현재 상태 백업) → 격리 디렉터리에서 풀어 확인 → 새 릴리스로 배치·데이터 교체(이전 데이터는 .pre-restore 로 보존) → current 전환 → 내부 확인. + */ +import { existsSync, mkdirSync, readdirSync, renameSync, copyFileSync, writeFileSync, statSync } from 'node:fs'; +import { join, basename, dirname } from 'node:path'; +import { siteDir, assertSiteId, readLink, switchLink, tarDir, untar, tarList, sha256File, rmrf, readJson, writeJson, internalCheck, fmtSum, normSum } from './common.mjs'; +import { configureVhost } from './site.mjs'; + +const stamp = () => new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14); +const backupDir = (ctx, siteId) => join(ctx.roots.backups, assertSiteId(siteId)); +/** SQLite 일관 사본 */ +async function copySqlite(src, dest) { const { DatabaseSync } = await import('node:sqlite'); const db = new DatabaseSync(src, { readOnly: true }); try { db.exec(`VACUUM INTO '${dest.replace(/'/g, "''")}'`); } finally { db.close(); } } +/** 현재 상태를 백업 파일로 묶는다 */ +export async function makeBackup(ctx, siteId, { backupId, scope = {}, outDir, db = null, tag = null }) { + const sd = siteDir(ctx.roots, siteId); const cur = readLink(join(sd, 'current')); const work = join(sd, 'work', `bk-${backupId}`); rmrf(work); mkdirSync(work, { recursive: true }); + const included = {}; + if (scope.files !== false && cur && existsSync(cur)) { mkdirSync(join(work, 'files'), { recursive: true }); await ctx.sys.tar(['-cf', join(work, 'files.tar'), '-C', cur, '.']); included.files = true; } + if (scope.uploads !== false && existsSync(join(sd, 'data'))) { await ctx.sys.tar(['-cf', join(work, 'data.tar'), '-C', join(sd, 'data'), '.']); included.data = true; } + if (scope.db !== false) { mkdirSync(join(work, 'db'), { recursive: true }); const dbs = []; const dbDir = join(sd, 'data', 'db'); if (existsSync(dbDir)) for (const f of readdirSync(dbDir).filter((x) => /\.(sqlite|db)$/.test(x))) { await copySqlite(join(dbDir, f), join(work, 'db', f)); dbs.push(f); } if (db?.sqlite && existsSync(db.sqlite)) { await copySqlite(db.sqlite, join(work, 'db', basename(db.sqlite))); dbs.push(basename(db.sqlite)); } included.db = dbs; } + if (scope.config !== false) { mkdirSync(join(work, 'config'), { recursive: true }); const vh = join(ctx.roots.nginx, 'sites-available', `aw-${siteId}.conf`); if (existsSync(vh)) copyFileSync(vh, join(work, 'config', 'vhost.conf')); if (cur) for (const f of ['.ariweb-version.json', 'ariweb.json']) if (existsSync(join(cur, f))) copyFileSync(join(cur, f), join(work, 'config', f)); included.config = true; } + writeJson(join(work, 'meta.json'), { backupId, siteId, createdAt: new Date().toISOString(), version: cur ? cur.split('/').pop() : null, included, tag, excluded: ['mail', 'source_repo', 'webmaker_history', 'external_deploy_data'] }); + mkdirSync(outDir, { recursive: true }); const out = join(outDir, `${backupId}.tar.gz`); const t = await tarDir(ctx, out, work); rmrf(work); + const list = await tarList(ctx, out); if (!list.some((x) => x.endsWith('meta.json'))) throw Object.assign(new Error('백업 파일을 읽을 수 없습니다'), { code: 'backup_unreadable' }); + return { backupId, path: out, sizeBytes: t.size, checksum: t.sha256, createdAt: new Date().toISOString(), scope: included }; +} +export const kinds = { + 'backup.create': { title: '백업 생성', async run(ctx) { const p = ctx.payload; const siteId = assertSiteId(p.siteId); const backupId = p.backupId || `bk-${stamp()}-${siteId.slice(0, 8)}`; const r = await ctx.step('create', () => makeBackup(ctx, siteId, { backupId, scope: p.scope || {}, outDir: backupDir(ctx, siteId), db: p.db || null, tag: p.retentionTag || null }), '백업 생성'); if (p.uploadToHal) { const up = await ctx.step('upload', () => ctx.client.upload(p.transferId || `tr_bk_${backupId}`, r.path, { purpose: 'backup' }), 'HAL 전송에 올리기'); r.transferId = up.id; r.uploaded = true; if (!p.keepLocalCopy) rmrf(r.path); } return r; } }, + 'backup.store': { title: '백업 저장소에 받기', async run(ctx) { const p = ctx.payload; const siteId = assertSiteId(p.siteId); const out = join(backupDir(ctx, siteId), `${p.backupId}.tar.gz`); mkdirSync(dirname(out), { recursive: true }); const dl = await ctx.step('download', () => ctx.client.downloadTransfer(p.transferId, out, p.sha256 || null), '받기'); const list = await tarList(ctx, out); if (!list.some((x) => x.endsWith('meta.json'))) throw Object.assign(new Error('받은 백업을 읽을 수 없습니다'), { code: 'backup_unreadable' }); return { path: out, sizeBytes: dl.size, checksum: fmtSum(dl.sha256), verified: true }; } }, + 'backup.verify': { title: '백업 검증', async run(ctx) { const p = ctx.payload; if (!p.location || !existsSync(p.location)) return { ok: false, reason: 'missing' }; const sum = sha256File(p.location); const list = await tarList(ctx, p.location).catch(() => []); const ok = (!p.checksum || normSum(p.checksum) === sum) && list.some((x) => x.endsWith('meta.json')); return { ok, checksum: fmtSum(sum), entries: list.length, reason: ok ? null : p.checksum && normSum(p.checksum) !== sum ? 'checksum_mismatch' : 'unreadable' }; } }, + 'backup.fetch': { title: '백업 저장소에서 보내기', async run(ctx) { const p = ctx.payload; if (!existsSync(p.location)) throw Object.assign(new Error('백업 파일이 없습니다'), { code: 'backup_missing' }); if (p.checksum && sha256File(p.location) !== normSum(p.checksum)) throw Object.assign(new Error('백업 파일 해시가 다릅니다'), { code: 'checksum_mismatch' }); const up = await ctx.step('upload', () => ctx.client.upload(p.transferId, p.location, { purpose: 'restore' }), '올리기'); return { transferId: up.id, checksum: fmtSum(up.sha256), sizeBytes: up.size }; } }, + 'backup.delete': { title: '백업 삭제', async run(ctx) { const p = ctx.payload; const ok = existsSync(p.location) && p.location.startsWith(ctx.roots.backups); if (ok) rmrf(p.location); return { deleted: ok, location: p.location }; } }, + 'backup.restore': { title: '백업 복원', async run(ctx) { + const p = ctx.payload; const siteId = assertSiteId(p.siteId); const sd = siteDir(ctx.roots, siteId); const work = join(sd, 'work', `rs-${stamp()}`); mkdirSync(work, { recursive: true }); + const src = await ctx.step('fetch', async () => { if (p.transferId) { const f = join(work, 'backup.tar.gz'); await ctx.client.downloadTransfer(p.transferId, f, p.checksum || null); return f; } if (!p.location || !existsSync(p.location)) throw Object.assign(new Error('복원할 백업이 없습니다'), { code: 'backup_missing' }); if (p.checksum && sha256File(p.location) !== normSum(p.checksum)) throw Object.assign(new Error('백업 해시가 다릅니다'), { code: 'checksum_mismatch' }); return p.location; }, '백업 가져오기'); + const protect = await ctx.step('protect', async () => { if (p.protectPoint === false) return null; const r = await makeBackup(ctx, siteId, { backupId: `protect-${stamp()}`, outDir: join(backupDir(ctx, siteId), 'protect'), tag: 'protect' }); return r.path; }, '현재 상태 보호 시점 만들기'); + const iso = await ctx.step('isolated', async () => { const dir = join(work, 'iso'); await untar(ctx, src, dir); const meta = readJson(join(dir, 'meta.json'), null); if (!meta || meta.siteId !== siteId) throw Object.assign(new Error('백업의 사이트 정보가 다릅니다'), { code: 'backup_site_mismatch' }); if (existsSync(join(dir, 'files.tar'))) { mkdirSync(join(dir, 'files'), { recursive: true }); await ctx.sys.tar(['-xf', join(dir, 'files.tar'), '-C', join(dir, 'files')]); } if (existsSync(join(dir, 'data.tar'))) { mkdirSync(join(dir, 'data'), { recursive: true }); await ctx.sys.tar(['-xf', join(dir, 'data.tar'), '-C', join(dir, 'data')]); } const files = existsSync(join(dir, 'files')) ? readdirSync(join(dir, 'files')).length : 0; if (!files) throw Object.assign(new Error('백업에 사이트 파일이 없습니다'), { code: 'backup_empty' }); return { dir, meta, files }; }, '격리된 환경에서 백업 확인'); + await ctx.progress(50); + const placed = await ctx.step('restore', async () => { const rel = join(sd, 'releases', `${stamp()}-restore`); mkdirSync(join(sd, 'releases'), { recursive: true }); renameSync(join(iso.dir, 'files'), rel); writeJson(join(rel, '.ariweb-version.json'), { version: rel.split('/').pop(), restoredFrom: iso.meta.backupId, restoredAt: new Date().toISOString(), originalVersion: iso.meta.version }); if (existsSync(join(iso.dir, 'data'))) { const old = join(sd, 'data'); if (existsSync(old)) renameSync(old, join(sd, `data.pre-restore-${stamp()}`)); renameSync(join(iso.dir, 'data'), old); } const dbSrc = join(iso.dir, 'db'); if (existsSync(dbSrc)) { mkdirSync(join(sd, 'data', 'db'), { recursive: true }); for (const f of readdirSync(dbSrc)) { const dest = p.db?.sqlite && basename(p.db.sqlite) === f ? p.db.sqlite : join(sd, 'data', 'db', f); if (existsSync(dest)) copyFileSync(dest, dest + `.pre-restore-${stamp()}`); copyFileSync(join(dbSrc, f), dest); } } const prev = readLink(join(sd, 'current')); switchLink(join(sd, 'current'), rel); writeJson(join(sd, '.previous.json'), { previous: prev, at: new Date().toISOString() }); return { rel, prev }; }, '데이터 복원'); + const meta = readJson(join(placed.rel, 'ariweb.json'), {}); if (meta.runtime === 'node') await ctx.sys.systemctl('restart', `ariweb-site-${siteId}`, { optional: true }); else await ctx.sys.nginxReload().catch(() => {}); + const ic = await ctx.step('check', () => internalCheck(ctx, siteId, (p.domains && p.domains[0]) || `${siteId}.preview.${ctx.roots.platformDomain}`), '사이트 기능 확인'); + rmrf(work); + if (!ic.ok) { if (placed.prev && existsSync(placed.prev)) switchLink(join(sd, 'current'), placed.prev); throw Object.assign(new Error('복원 뒤 내부 확인 실패 — 이전 상태로 되돌렸습니다'), { code: 'internal_check_failed', partial: { protectPointLocation: protect, internalCheck: ic } }); } + return { protectPointLocation: protect, restoredVersion: placed.rel.split('/').pop(), isolatedVerify: true, internalCheck: ic, restoredFrom: iso.meta.backupId }; + } }, +}; diff --git a/agent/jobs/common.mjs b/agent/jobs/common.mjs new file mode 100644 index 0000000..f7219fb --- /dev/null +++ b/agent/jobs/common.mjs @@ -0,0 +1,66 @@ +/** + * agent/jobs/common.mjs — 사이트 작업 공통: 경로·템플릿·vhost 생성·내부 확인·tar·체크섬. + * 사이트 레이아웃: //releases/<버전디렉터리>, current(심링크), data/(uploads·db 등 영속 데이터), .maintenance / .limited 표식 파일. + */ +import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, statSync, symlinkSync, unlinkSync, renameSync, rmSync, lstatSync, readlinkSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { createHash } from 'node:crypto'; +import http from 'node:http'; +import { fileURLToPath } from 'node:url'; + +export const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..'); +export const SITE_ID_RE = /^[a-zA-Z0-9][a-zA-Z0-9_-]{1,63}$/; +export function assertSiteId(id) { if (!SITE_ID_RE.test(String(id || ''))) throw Object.assign(new Error(`사이트 id 형식이 올바르지 않습니다: ${id}`), { code: 'bad_site_id' }); return id; } +export const siteDir = (roots, siteId) => join(roots.sites, assertSiteId(siteId)); +export const tmpl = (name) => readFileSync(join(ROOT, 'deploy', 'nginx', name), 'utf8'); +export const render = (t, vars) => t.replace(/\{\{(\w+)\}\}/g, (_, k) => (vars[k] === undefined || vars[k] === null ? '' : String(vars[k]))); +export function sha256File(file) { const h = createHash('sha256'); h.update(readFileSync(file)); return h.digest('hex'); } +/** + * 체크섬 표기: 결과·payload 에 넣는 해시는 'sha256:' 형식으로 쓴다. 긴 hex 그대로 두면 lib/redact 가 비밀값으로 보고 '[redacted]' 로 가려 + * 포털·HAL 이 검증에 쓸 수 없기 때문이다. 비교할 때는 normSum 으로 접두어·대소문자를 벗긴다(두 형식 모두 허용). + */ +export const fmtSum = (hex) => (hex ? `sha256:${String(hex).replace(/^sha256:/i, '').toLowerCase()}` : null); +export const normSum = (v) => (v ? String(v).replace(/^sha256:/i, '').toLowerCase() : null); +export function dirSize(dir) { let n = 0; if (!existsSync(dir)) return 0; for (const e of readdirSync(dir, { withFileTypes: true })) { const p = join(dir, e.name); if (e.isSymbolicLink()) continue; if (e.isDirectory()) n += dirSize(p); else n += statSync(p).size; } return n; } +/** 심링크 원자 교체 */ +export function switchLink(linkPath, target) { const tmp = `${linkPath}.tmp-${process.pid}`; try { unlinkSync(tmp); } catch (_) {} symlinkSync(target, tmp); renameSync(tmp, linkPath); } +export function readLink(linkPath) { try { return lstatSync(linkPath).isSymbolicLink() ? readlinkSync(linkPath) : null; } catch (_) { return null; } } +export const MAINT_HTML = '점검 중

사이트 점검 중입니다

잠시 후 다시 접속해 주세요.

supported by ariweb.net

'; +export const LIMIT_HTML = '이용 제한

월간 전송량 한도에 도달했습니다

다음 달(KST 1일) 또는 추가 용량 적용 뒤 다시 제공됩니다.

supported by ariweb.net

'; +const esc1 = (s) => s.replace(/'/g, '''); +/** 사이트 vhost 설정 문자열 만들기 */ +export function renderVhost({ roots, siteId, domains = [], primaryDomain = null, runtime = 'static', port = null, certDir = null, redirectOld = [] }) { + const sd = siteDir(roots, siteId); const names = domains.length ? domains : [`${siteId}.preview.${roots.platformDomain}`]; + const body = render(tmpl(runtime === 'node' ? 'site-body-node.tmpl' : 'site-body-static.tmpl'), { ROOT: join(sd, 'current'), SITE_DIR: sd, LOG: join(roots.logs, `aw-${siteId}.access.log`), MAINT_HTML: esc1(MAINT_HTML), LIMIT_HTML: esc1(LIMIT_HTML), PORT: port || 0 }); + const https = certDir ? render(tmpl('site-https.tmpl'), { SERVER_NAMES: names.join(' '), CERT_DIR: certDir, BODY: body }) : ''; + const httpBody = certDir ? ` location / { return 301 https://${primaryDomain || names[0]}$request_uri; }` : body; + let out = render(tmpl('site.conf.tmpl'), { SITE_ID: siteId, DOMAIN_LIST: names.join(', '), GENERATED_AT: new Date().toISOString(), SERVER_NAMES: names.join(' '), ACME_ROOT: roots.acme, HTTP_BODY: httpBody, HTTPS_SERVER: https }); + if (redirectOld.length && primaryDomain) out += '\n' + render(tmpl('redirect.tmpl'), { OLD_NAMES: redirectOld.join(' '), ACME_ROOT: roots.acme, PRIMARY: primaryDomain }); + return out; +} +/** 와일드카드 인증서 적용 가능 여부: 모든 도메인이 플랫폼 도메인의 1단계 하위이고 인증서 디렉터리가 있을 때 */ +export function wildcardCertFor(roots, domains) { if (!roots.wildcardCert || !domains.length) return null; const pd = roots.platformDomain.toLowerCase(); const ok = domains.every((d) => { const x = String(d).toLowerCase(); return x.endsWith('.' + pd) && !x.slice(0, -pd.length - 1).includes('.'); }); return ok && existsSync(join(roots.wildcardCert, 'fullchain.pem')) ? roots.wildcardCert : null; } +/** 사이트별 인증서 디렉터리(certbot --cert-name aw-) */ +export function ownCertFor(roots, siteId) { const p = join(roots.letsencrypt, `aw-${siteId}`); return existsSync(join(p, 'fullchain.pem')) ? p : null; } +/** 인증서 만료 읽기(openssl 없이: PEM 의 notAfter 는 파싱이 번거로워 파일 mtime+90일로 추정, 정확한 값은 domain.cert 결과가 준다) */ +export function certExpiry(certDir) { try { const st = statSync(join(certDir, 'fullchain.pem')); return new Date(st.mtimeMs + 90 * 86400000).toISOString(); } catch (_) { return null; } } +/** vhost 파일 쓰고 nginx 적용. stub 모드에서는 파일만 쓴다. */ +export async function applyVhost(ctx, siteId, content) { + const roots = ctx.roots; const avail = join(roots.nginx, 'sites-available', `aw-${siteId}.conf`); const enabled = join(roots.nginx, 'sites-enabled', `aw-${siteId}.conf`); + await ctx.sys.writeFile(avail, content); + try { mkdirSync(dirname(enabled), { recursive: true }); if (!existsSync(enabled)) symlinkSync(avail, enabled); } catch (e) { if (e.code === 'EACCES' || e.code === 'EPERM') { await ctx.sys.sudo(['ln', '-sf', avail, enabled]); } else if (e.code !== 'EEXIST') throw e; } + await ctx.sys.nginxReload(); return { file: avail }; +} +export async function removeVhost(ctx, siteId) { const roots = ctx.roots; const files = [join(roots.nginx, 'sites-enabled', `aw-${siteId}.conf`), join(roots.nginx, 'sites-available', `aw-${siteId}.conf`)]; let removed = 0; for (const f of files) { try { unlinkSync(f); removed++; } catch (e) { if (e.code === 'EACCES' || e.code === 'EPERM') { await ctx.sys.sudo(['rm', '-f', f]); removed++; } } } if (removed) await ctx.sys.nginxReload().catch(() => {}); return removed; } +/** 내부 접속 확인: 127.0.0.1:80 에 Host 헤더로. stub 모드면 current/index.html 존재로 대신한다. */ +export function internalCheck(ctx, siteId, host, { path = '/', expectMarker = null, port = 80 } = {}) { + if (ctx.sys.stub) { const sd = siteDir(ctx.roots, siteId); const idx = join(sd, 'current', 'index.html'); const ok = existsSync(idx) && (!expectMarker || readFileSync(idx, 'utf8').includes(expectMarker)); return Promise.resolve({ ok, status: ok ? 200 : 404, ms: 0, stubbed: true, host }); } + return new Promise((resolve) => { const t0 = Date.now(); const req = http.get({ host: '127.0.0.1', port, path, headers: { Host: host } }, (res) => { let body = ''; res.on('data', (c) => { if (body.length < 200000) body += c; }); res.on('end', () => resolve({ ok: res.statusCode < 400 && (!expectMarker || body.includes(expectMarker)), status: res.statusCode, ms: Date.now() - t0, host, markerFound: expectMarker ? body.includes(expectMarker) : null })); }); req.setTimeout(15000, () => { req.destroy(new Error('timeout')); }); req.on('error', (e) => resolve({ ok: false, error: e.message, ms: Date.now() - t0, host })); }); +} +/** 디렉터리를 tar.gz 로 */ +export async function tarDir(ctx, outFile, dir, items = ['.']) { mkdirSync(dirname(outFile), { recursive: true }); await ctx.sys.tar(['-czf', outFile, '-C', dir, ...items]); return { file: outFile, size: statSync(outFile).size, sha256: fmtSum(sha256File(outFile)) }; } +export async function untar(ctx, file, destDir) { mkdirSync(destDir, { recursive: true }); await ctx.sys.tar(['-xzf', file, '-C', destDir]); return destDir; } +export async function tarList(ctx, file) { const r = await ctx.sys.run('tar', ['-tzf', file], { allowStub: false }); if (r.code !== 0) throw new Error(`tar 목록 실패: ${r.stderr.slice(0, 200)}`); return r.stdout.split('\n').filter(Boolean); } +export const rmrf = (p) => rmSync(p, { recursive: true, force: true }); +export const readJson = (p, fb = null) => { try { return JSON.parse(readFileSync(p, 'utf8')); } catch (_) { return fb; } }; +export const writeJson = (p, v) => { mkdirSync(dirname(p), { recursive: true }); writeFileSync(p, JSON.stringify(v, null, 1)); }; diff --git a/agent/jobs/domain.mjs b/agent/jobs/domain.mjs new file mode 100644 index 0000000..a3a5f3b --- /dev/null +++ b/agent/jobs/domain.mjs @@ -0,0 +1,27 @@ +/** + * agent/jobs/domain.mjs — 도메인 작업: vhost 재구성, HTTPS 인증서 발급(certbot webroot), DNS 확인(공용 리졸버). + * 인증서: 플랫폼 서브 도메인은 와일드카드(*.ariweb.net)를 재사용하고, 고객 도메인은 certbot --webroot 로 사이트별(aw-) 인증서를 받는다(Webmaker deploy.js 와 같은 원칙). + * 유효하지 않은 인증서를 정상 HTTPS 로 표시하지 않는다(§12.3): certbot 결과가 성공이고 fullchain.pem 이 있을 때만 'issued'. + */ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { promises as dns } from 'node:dns'; +import { assertSiteId, ownCertFor, wildcardCertFor, certExpiry } from './common.mjs'; +import { configureVhost } from './site.mjs'; + +export const kinds = { + 'domain.vhost': { title: '도메인 연결 설정', async run(ctx) { const p = ctx.payload; const siteId = assertSiteId(p.siteId); const r = await ctx.step('vhost', () => configureVhost(ctx, siteId, { domains: p.domains || [], primaryDomain: p.primaryDomain || null, runtime: p.runtime || 'static', port: p.port || null, redirectOld: p.redirectOld ? (p.oldDomains || []) : [] }), 'nginx 설정'); if (p.maintenance !== undefined) { const { kinds: site } = await import('./site.mjs'); await site['site.maintenance'].run({ ...ctx, payload: { siteId, on: !!p.maintenance } }); } return { applied: r.serverNames, certs: r.certs, https: r.https }; } }, + 'domain.cert': { title: 'HTTPS 인증서 발급', async run(ctx) { + const p = ctx.payload; const siteId = assertSiteId(p.siteId); const domains = (p.domains || []).map((d) => String(d).toLowerCase()); if (!domains.length) throw Object.assign(new Error('도메인이 없습니다'), { code: 'domains_required' }); + const wild = wildcardCertFor(ctx.roots, domains); if (wild) { const r = await ctx.step('vhost', () => configureVhost(ctx, siteId, { domains, primaryDomain: p.primaryDomain || domains[0], runtime: p.runtime || 'static', port: p.port || null }), '와일드카드 인증서 적용'); return { issued: domains, failed: [], source: 'wildcard', expiresAt: certExpiry(wild), https: r.https }; } + // 1) HTTP vhost 가 먼저 있어야 webroot 인증이 된다 + await ctx.step('vhost-http', () => configureVhost(ctx, siteId, { domains, primaryDomain: p.primaryDomain || domains[0], runtime: p.runtime || 'static', port: p.port || null }), 'HTTP 연결 준비'); + const issued = []; const failed = []; + await ctx.step('certbot', async () => { const args = ['certonly', '--webroot', '-w', ctx.roots.acme, '--cert-name', `aw-${siteId}`, '--non-interactive', '--agree-tos', '--keep-until-expiring', '--expand', ...(p.email ? ['-m', p.email] : ['--register-unsafely-without-email'])]; for (const d of domains) args.push('-d', d); const r = await ctx.sys.certbot(args); if (r.code !== 0 && !ctx.sys.stub) { const msg = (r.stderr || r.stdout).trim(); for (const d of domains) failed.push({ domain: d, reason: msg.slice(0, 300) || 'certbot 실패' }); } else { issued.push(...domains); if (ctx.sys.stub && !ownCertFor(ctx.roots, siteId)) { const { mkdirSync, writeFileSync } = await import('node:fs'); const dir = join(ctx.roots.letsencrypt, `aw-${siteId}`); mkdirSync(dir, { recursive: true }); writeFileSync(join(dir, 'fullchain.pem'), 'stub'); writeFileSync(join(dir, 'privkey.pem'), 'stub'); } } return { code: r.code }; }, 'certbot 실행'); + if (!issued.length) { const e = new Error(`인증서 발급 실패: ${failed[0]?.reason || '원인 불명'}`); e.code = 'cert_failed'; e.partial = { failed }; throw e; } + const own = ownCertFor(ctx.roots, siteId); if (!own) throw Object.assign(new Error('certbot 은 성공했지만 인증서 파일이 없습니다'), { code: 'cert_missing' }); + const r = await ctx.step('vhost-https', () => configureVhost(ctx, siteId, { domains, primaryDomain: p.primaryDomain || domains[0], runtime: p.runtime || 'static', port: p.port || null }), 'HTTPS 연결'); + return { issued, failed, source: 'own', expiresAt: certExpiry(own), https: r.https }; + } }, + 'domain.probe': { title: 'DNS 확인', async run(ctx) { const p = ctx.payload; const name = String(p.name || '').toLowerCase(); if (!name) throw Object.assign(new Error('이름이 없습니다'), { code: 'name_required' }); if (ctx.sys.stub && p._stubAnswer) return { ...p._stubAnswer, resolvedAt: new Date().toISOString() }; const r = new dns.Resolver({ timeout: 5000, tries: 2 }); r.setServers(['1.1.1.1', '8.8.8.8']); const out = { name, a: [], txtOk: null, cname: [], mx: [], resolvedAt: new Date().toISOString(), resolver: 'public' }; try { out.a = await r.resolve4(name); } catch (e) { out.aError = e.code; } try { out.cname = await r.resolveCname(name); } catch (_) {} if (p.expectTxt) { try { const txts = (await r.resolveTxt(p.expectTxt.name || name)).map((x) => x.join('')); out.txtOk = txts.includes(p.expectTxt.value); out.txtFound = txts.length; } catch (e) { out.txtOk = false; out.txtError = e.code; } } if (p.mx) { try { out.mx = await r.resolveMx(name); } catch (_) {} } if (p.expectA) { const want = new Set(p.expectA); out.aOk = out.a.length > 0 && out.a.every((x) => want.has(x)); out.aMismatch = out.a.filter((x) => !want.has(x)); } return out; } }, +}; diff --git a/agent/jobs/index.mjs b/agent/jobs/index.mjs new file mode 100644 index 0000000..d4725f2 --- /dev/null +++ b/agent/jobs/index.mjs @@ -0,0 +1,6 @@ +/** agent/jobs/index.mjs — 작업 종류 등록부. HAL 이 보내는 kind 이름과 같다. External Pull Agent 는 HAL 이 배정을 제한하므로 여기서는 모두 등록한다. */ +import { kinds as site } from './site.mjs'; +import { kinds as domain } from './domain.mjs'; +import { kinds as backup } from './backup.mjs'; +import { kinds as mod } from './module.mjs'; +export const KINDS = { ...site, ...domain, ...backup, ...mod }; diff --git a/agent/jobs/module.mjs b/agent/jobs/module.mjs new file mode 100644 index 0000000..47e8299 --- /dev/null +++ b/agent/jobs/module.mjs @@ -0,0 +1,67 @@ +/** + * agent/jobs/module.mjs — 모듈 설치·갱신·되돌리기·건강 확인(§32·§29). + * + * 레이아웃: /releases// + current 심링크. 도입 모듈은 서버에 이미 있던 appDir(/srv/webmaker, /srv/arihakdang-mail …)과 유닛 이름을 facts 로 받아 그대로 쓴다. + * 절차(원본 remote-update.sh / deploy.sh 의 순서를 따른다): artifact 내려받기·sha256 확인 → releases/ 에 풀기 → preStop 절차(데이터 백업) → 유닛 갱신(템플릿이 있을 때) → current 전환 → postStart 절차(마이그레이션 등) → 재시작 → 건강 확인 → 실제 버전 읽기. + * 실패하면 current 를 이전 릴리스로 되돌리고 재시작한 뒤 오류를 보고한다(명령 성공만으로 완료하지 않는다). + * 자기 자신(agent)·HAL 갱신: 결과를 먼저 보고한 뒤 재시작하도록 restartAfterReport 를 돌려준다. + */ +import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, copyFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import http from 'node:http'; +import { readLink, switchLink, untar, render, readJson, writeJson, rmrf } from './common.mjs'; + +/** 내부 건강 확인 URL 호출 */ +function httpGet(url, timeoutMs = 8000) { return new Promise((resolve) => { const t0 = Date.now(); const req = http.get(url, (res) => { let body = ''; res.on('data', (c) => { if (body.length < 100000) body += c; }); res.on('end', () => resolve({ status: res.statusCode, body, ms: Date.now() - t0 })); }); req.setTimeout(timeoutMs, () => req.destroy(new Error('timeout'))); req.on('error', (e) => resolve({ status: 0, error: e.message, ms: Date.now() - t0 })); }); } +async function health(ctx, desc) { if (ctx.sys.stub) return { ok: true, stubbed: true }; if (!desc?.health?.url) return { ok: true, skipped: 'no_health_url' }; let last = null; for (let i = 0; i < 20; i++) { const r = await httpGet(desc.health.url); last = r; if (r.status && r.status < 500 && (desc.health.expect !== 'ok' || /"ok"\s*:\s*true|ok/.test(r.body))) return { ok: true, status: r.status, ms: r.ms }; await new Promise((s) => setTimeout(s, 1500)); } return { ok: false, status: last?.status, error: last?.error || `상태 ${last?.status}` }; } +async function detectVersion(ctx, desc, appDir) { const cur = readLink(join(appDir, 'current')); if (desc?.version?.http && !ctx.sys.stub) { const r = await httpGet(desc.version.http); try { const j = JSON.parse(r.body); if (j[desc.version.field]) return j[desc.version.field]; } catch (_) {} } const pkg = cur ? readJson(join(cur, 'package.json'), null) : null; return pkg?.version || (cur ? cur.split('/').pop() : null); } +/** 기존 유닛의 Environment 줄을 보존해 템플릿을 채운다 */ +async function writeUnit(ctx, desc, unit, appDir, version) { + if (!ctx.payload.unitTemplate) return { skipped: 'no_template' }; + const old = await ctx.sys.unitText(unit); const envLines = old.split('\n').filter((l) => /^Environment=/.test(l) && !/^Environment=(NODE_ENV|HOST|PORT|ARIWEB_[A-Z_]+_DATA|NODE_OPTIONS)=/.test(l)).join('\n'); + const user = (/^User=(.+)$/m.exec(old) || [])[1] || desc.user || process.env.USER || 'master'; const envFile = (/^EnvironmentFile=(.+)$/m.exec(old) || [])[1] || ''; const node = (/^ExecStart=(\S+)/m.exec(old) || [])[1] || process.execPath; + const txt = render(ctx.payload.unitTemplate, { APP_DIR: appDir, USER: user, NODE: /node$/.test(node) ? node : process.execPath, ENV_LINES: envLines, VERSION: version, UNIT: unit, ENV_FILE: envFile }); + await ctx.sys.writeFile(`/etc/systemd/system/${unit}.service`, txt); await ctx.sys.systemctl('daemon-reload', '', { optional: true }); return { written: true, preservedEnv: envLines.split('\n').filter(Boolean).length }; +} +/** preStop/postStart 절차 */ +async function hook(ctx, name, { appDir, cur, unit }) { + if (name === 'webmaker.backupData') { const dataRoot = (/^Environment=DATA_ROOT=(.+)$/m.exec(await ctx.sys.unitText(unit)) || [])[1] || join(appDir, 'data'); if (!existsSync(dataRoot)) return { skipped: 'no_data' }; const out = join(appDir, 'backups', `data-${new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14)}.tar.gz`); mkdirSync(join(appDir, 'backups'), { recursive: true }); await ctx.sys.tar(['-czf', out, '-C', dataRoot, '--exclude=./backups', '.']); const old = readdirSync(join(appDir, 'backups')).sort(); for (const f of old.slice(0, Math.max(0, old.length - 5))) rmSync(join(appDir, 'backups', f), { force: true }); return { backup: out }; } + if (name === 'arimail.backupDb') { const db = join(appDir, 'data', 'mail-gateway.sqlite'); if (!existsSync(db)) return { skipped: 'no_db' }; const { DatabaseSync } = await import('node:sqlite'); mkdirSync(join(appDir, 'data', 'backups'), { recursive: true }); const out = join(appDir, 'data', 'backups', `pre-${new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14)}.sqlite`); const d = new DatabaseSync(db, { readOnly: true }); try { d.exec(`VACUUM INTO '${out}'`); } finally { d.close(); } return { backup: out }; } + if (name === 'arimail.migrate') { const db = join(appDir, 'data', 'mail-gateway.sqlite'); const r = await ctx.sys.run(process.execPath, [join(cur, 'server', 'db-migrate.mjs')], { allowStub: false, timeoutMs: 120000 }).catch((e) => ({ code: 1, stderr: e.message })); if (!ctx.sys.stub && !existsSync(join(cur, 'server', 'db-migrate.mjs'))) return { skipped: 'no_migrate' }; if (r.code !== 0 && !ctx.sys.stub) throw Object.assign(new Error(`AriMail 마이그레이션 실패: ${(r.stderr || '').slice(0, 300)}`), { code: 'migrate_failed' }); return { migrated: true, db }; } + if (name === 'arimail.haraka') { const r = await ctx.sys.systemctl('restart', `${unit}-haraka`, { optional: true }); return { haraka: r.code === 0 ? 'restarted' : 'skipped' }; } + if (name === 'webmaker.unitCheck') { const r = await ctx.sys.run(process.execPath, [join(cur, 'scripts', 'unit-check.js')], { allowStub: true, timeoutMs: 60000 }).catch(() => ({ code: 1 })); return { unitCheck: r.code === 0 ? 'ok' : 'warn' }; } + return { skipped: `unknown hook ${name}` }; +} +async function installRelease(ctx, { module, version, appDir, desc, unit, action, toVersion }) { + const relDir = join(appDir, 'releases', action === 'rollback' ? (toVersion || '') : version); const prev = readLink(join(appDir, 'current')); + if (action === 'rollback') { const target = toVersion ? relDir : readJson(join(appDir, '.previous.json'), {}).previous; if (!target || !existsSync(target)) throw Object.assign(new Error('되돌릴 이전 릴리스가 없습니다'), { code: 'no_previous_release' }); return { relDir: target, prev }; } + await ctx.step('download', async () => { if (!ctx.payload.artifactUrl) throw Object.assign(new Error('artifact 주소가 없습니다'), { code: 'artifact_missing' }); mkdirSync(join(appDir, 'work'), { recursive: true }); const f = join(appDir, 'work', `${module}-${version}.tar.gz`); const r = await ctx.client.download(ctx.payload.artifactUrl, f, ctx.payload.sha256 || null); return { file: f, sha256: r.sha256, size: r.size }; }, 'artifact 내려받기·검증'); + await ctx.step('extract', async () => { rmrf(relDir); mkdirSync(relDir, { recursive: true }); await untar(ctx, join(appDir, 'work', `${module}-${version}.tar.gz`), relDir); if (desc.copyPaths && !existsSync(join(relDir, 'package.json'))) throw Object.assign(new Error('artifact 에 package.json 이 없습니다'), { code: 'bad_artifact' }); writeJson(join(relDir, 'REVISION.json'), { module, version, installedAt: new Date().toISOString() }); rmSync(join(appDir, 'work', `${module}-${version}.tar.gz`), { force: true }); return { dir: relDir }; }, '릴리스 풀기'); + return { relDir, prev }; +} +export const kinds = { + 'module.install': { title: '모듈 설치·갱신', async run(ctx) { return installOrRollback(ctx, ctx.payload.action || 'install'); } }, + 'module.rollback': { title: '모듈 되돌리기', async run(ctx) { return installOrRollback(ctx, 'rollback'); } }, + 'module.health': { title: '모듈 건강 확인', async run(ctx) { const p = ctx.payload; const desc = p.descriptor || {}; const appDir = p.appDir || join(ctx.roots.srv, p.module); const unit = p.unit || desc.unit; const active = unit ? await ctx.sys.isActive(unit) : true; const h = await health(ctx, desc); return { module: p.module, actualVersion: await detectVersion(ctx, desc, appDir), active, health: { ok: active && h.ok, ...h }, checkedAt: new Date().toISOString() }; } }, +}; +async function installOrRollback(ctx, action) { + const p = ctx.payload; const { module, version, descriptor: desc = {}, toVersion = null } = p; const appDir = p.appDir || join(ctx.roots.srv, module); const unit = (desc && desc.unit) || `ariweb-${module}`; + mkdirSync(join(appDir, 'releases'), { recursive: true }); + const { relDir, prev } = await ctx.step('place', () => installRelease(ctx, { module, version, appDir, desc, unit, action, toVersion }), action === 'rollback' ? '되돌릴 릴리스 확인' : '릴리스 배치'); + await ctx.progress(40); + for (const h of desc.preStop || []) await ctx.step(`pre:${h}`, () => hook(ctx, h, { appDir, cur: relDir, unit }), `사전 절차 ${h}`); + await ctx.step('unit', () => writeUnit(ctx, desc, unit, appDir, version), '실행 유닛 갱신'); + await ctx.step('switch', () => { switchLink(join(appDir, 'current'), relDir); writeJson(join(appDir, '.previous.json'), { previous: prev, at: new Date().toISOString() }); return { from: prev, to: relDir }; }, 'current 전환'); + for (const h of desc.postStart || []) await ctx.step(`post:${h}`, () => hook(ctx, h, { appDir, cur: relDir, unit }), `사후 절차 ${h}`); + const self = module === 'agent'; let healthRes = { ok: true }; + if (!self && desc.layout !== 'agent-feature') { + try { await ctx.step('restart', () => ctx.sys.systemctl('restart', unit), '서비스 재시작'); for (const u of desc.extraUnits || []) await ctx.sys.systemctl('restart', u, { optional: true }); } + catch (e) { if (prev && existsSync(prev)) { switchLink(join(appDir, 'current'), prev); await ctx.sys.systemctl('restart', unit, { optional: true }); } throw Object.assign(e, { code: e.code || 'restart_failed', partial: { rolledBackTo: prev } }); } + healthRes = await ctx.step('health', () => health(ctx, desc), '건강 확인'); + if (!healthRes.ok) { if (prev && existsSync(prev)) { switchLink(join(appDir, 'current'), prev); await ctx.sys.systemctl('restart', unit, { optional: true }); } throw Object.assign(new Error(`재시작 뒤 건강 확인 실패(${healthRes.error || healthRes.status}) — 이전 릴리스로 되돌렸습니다`), { code: 'health_failed', partial: { rolledBackTo: prev, health: healthRes } }); } + } + const actual = self ? readJson(join(relDir, 'package.json'), {}).version || version : await ctx.step('version', () => detectVersion(ctx, desc, appDir), '실제 버전 읽기'); + // 오래된 릴리스 정리(최근 3개 보존) + const rels = readdirSync(join(appDir, 'releases')).sort(); for (const r of rels.slice(0, Math.max(0, rels.length - 3))) { const d = join(appDir, 'releases', r); if (d !== relDir && d !== prev) rmrf(d); } + return { module, actualVersion: actual, previousVersion: prev ? prev.split('/').pop() : null, health: healthRes, active: true, restartAfterReport: self ? unit : null, releaseDir: relDir }; +} diff --git a/agent/jobs/site.mjs b/agent/jobs/site.mjs new file mode 100644 index 0000000..e595c63 --- /dev/null +++ b/agent/jobs/site.mjs @@ -0,0 +1,114 @@ +/** + * agent/jobs/site.mjs — 사이트 작업: 배포·되돌리기·제거·확인·사용량, 이전(relocate) 보조 작업. + * + * site.deploy(§11.2): 번들 가져오기(HAL 전송·서명 URL·로컬 경로) → releases/ 에 풀기 → 버전 기록 → current 교체 → (동적) 실행 유닛 → vhost → nginx → 내부 확인. + * 실패하면 current 를 바꾸지 않고(기존 버전 유지), 만들어 둔 디렉터리를 leftovers 로 보고해 포털이 "남은 파일" 과 정리를 보일 수 있게 한다. + * site.usage(§13.1): 사이트 디렉터리 크기(du) + nginx awstats 로그 증분 파싱 → 일별 집계(data/usage/.json, 400일) → 24h/7d/30d/누적. + * 이전 보조: site.prepare / site.export(전송 올리기) / site.import(받아 staging) / site.stage_check / site.activate / site.deactivate / site.freeze. + */ +import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync, statSync, renameSync, openSync, readSync, closeSync, fstatSync, copyFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { siteDir, assertSiteId, switchLink, readLink, renderVhost, applyVhost, removeVhost, internalCheck, wildcardCertFor, ownCertFor, certExpiry, dirSize, tarDir, untar, sha256File, rmrf, readJson, writeJson, render, ROOT, fmtSum, normSum } from './common.mjs'; + +const stamp = () => new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14); +/** 번들을 작업 디렉터리의 파일로 가져온다 */ +async function fetchBundle(ctx, siteId, dest) { + const p = ctx.payload; mkdirSync(join(dest), { recursive: true }); const file = join(dest, 'bundle.tar.gz'); + if (p.transferId) { const r = await ctx.client.downloadTransfer(p.transferId, file, p.checksum || null); return { file, sha256: fmtSum(r.sha256), source: 'transfer' }; } + if (p.bundleUrl) { const r = await ctx.client.download(p.bundleUrl.startsWith('http') ? p.bundleUrl : p.bundleUrl, file, p.checksum || null); return { file, sha256: fmtSum(r.sha256), source: 'url' }; } + if (p.bundlePath && existsSync(p.bundlePath)) { if (statSync(p.bundlePath).isDirectory()) { await tarDir(ctx, file, p.bundlePath); return { file, sha256: fmtSum(sha256File(file)), source: 'dir' }; } copyFileSync(p.bundlePath, file); return { file, sha256: fmtSum(sha256File(file)), source: 'file' }; } + if (p.bundleFromWebmaker?.path && existsSync(p.bundleFromWebmaker.path)) { await tarDir(ctx, file, p.bundleFromWebmaker.path); return { file, sha256: fmtSum(sha256File(file)), source: 'webmaker' }; } + throw Object.assign(new Error('배포할 저장본(transferId / bundleUrl / bundlePath / bundleFromWebmaker)이 없습니다'), { code: 'bundle_missing' }); +} +/** 동적(node) 사이트 포트: 지정 없으면 siteId 해시로 20000~29999 */ +function sitePort(siteId, p) { if (p.port) return Number(p.port); let h = 0; for (const c of siteId) h = (h * 31 + c.charCodeAt(0)) >>> 0; return 20000 + (h % 10000); } +async function nodeUnit(ctx, siteId, sd, start, port) { const unit = `ariweb-site-${siteId}`; const txt = render(readFileSync(join(ROOT, 'deploy', 'units', 'ariweb-site.service.tmpl'), 'utf8'), { SITE_ID: siteId, SITE_DIR: sd, PORT: port, USER: process.env.USER || 'master', NODE: process.execPath, START: start }); await ctx.sys.writeFile(`/etc/systemd/system/${unit}.service`, txt); await ctx.sys.systemctl('daemon-reload', '', { optional: true }); await ctx.sys.systemctl('enable', unit, { optional: true }); await ctx.sys.systemctl('restart', unit); return unit; } +/** 현재 운영 중인 도메인 설정으로 vhost 다시 쓰기 */ +export async function configureVhost(ctx, siteId, { domains = [], primaryDomain = null, runtime = 'static', port = null, redirectOld = [] }) { + const own = ownCertFor(ctx.roots, siteId); const wild = wildcardCertFor(ctx.roots, domains); const certDir = own || wild; + const conf = renderVhost({ roots: ctx.roots, siteId, domains, primaryDomain, runtime, port, certDir, redirectOld }); + const r = await applyVhost(ctx, siteId, conf); + const certs = Object.fromEntries(domains.map((dm) => [dm, certDir ? { state: 'active', source: own ? 'own' : 'wildcard', expiresAt: certExpiry(certDir) } : { state: 'none' }])); + return { ...r, https: !!certDir, certs, serverNames: domains.length ? domains : [`${siteId}.preview.${ctx.roots.platformDomain}`] }; +} + +export const kinds = { + 'site.deploy': { title: '사이트 배포', async run(ctx) { + const p = ctx.payload; const siteId = assertSiteId(p.siteId); const sd = siteDir(ctx.roots, siteId); const leftovers = []; + const rel = await ctx.step('prepare', () => { mkdirSync(join(sd, 'releases'), { recursive: true }); mkdirSync(join(sd, 'data', 'uploads'), { recursive: true }); const n = `${stamp()}-${String(p.versionRef || 'v').replace(/[^a-zA-Z0-9._-]/g, '').slice(0, 24)}`; return { dir: join(sd, 'releases', n), name: n }; }, '배포 위치 준비'); + leftovers.push(rel.dir); + try { + const b = await ctx.step('fetch', () => fetchBundle(ctx, siteId, join(sd, 'work', rel.name)), '저장본 준비'); leftovers.push(join(sd, 'work', rel.name)); + await ctx.step('extract', async () => { await untar(ctx, b.file, rel.dir); writeJson(join(rel.dir, '.ariweb-version.json'), { version: rel.name, versionRef: p.versionRef || null, deployedAt: new Date().toISOString(), sha256: b.sha256, source: b.source }); rmrf(join(sd, 'work', rel.name)); return { files: readdirSync(rel.dir).length }; }, '파일과 설정 적용'); + await ctx.step('data', () => { const meta = readJson(join(rel.dir, 'ariweb.json'), {}); if (meta.db?.sqlite) mkdirSync(join(sd, 'data', 'db'), { recursive: true }); return { runtime: p.runtime || meta.runtime || 'static', db: meta.db || null }; }, '데이터베이스와 업로드 데이터 확인'); + await ctx.progress(50); ctx.checkCancel(); + const prev = readLink(join(sd, 'current')); + const sw = await ctx.step('switch', async () => { switchLink(join(sd, 'current'), rel.dir); writeJson(join(sd, '.previous.json'), { previous: prev, at: new Date().toISOString() }); return { previous: prev }; }, '서비스 시작(버전 전환)'); + const meta = readJson(join(rel.dir, 'ariweb.json'), {}); const runtime = p.runtime || meta.runtime || 'static'; let port = null; + if (runtime === 'node') { port = sitePort(siteId, p); await ctx.step('unit', () => nodeUnit(ctx, siteId, sd, meta.start || 'server.js', port), '실행 유닛 준비'); } + const vh = await ctx.step('vhost', () => configureVhost(ctx, siteId, { domains: p.domains || [], primaryDomain: p.primaryDomain || null, runtime, port }), '도메인 연결(nginx)'); + await ctx.progress(80); + const ic = await ctx.step('internal', () => internalCheck(ctx, siteId, vh.serverNames[0], { expectMarker: p.expectMarker || null }), '내부 확인'); + if (!ic.ok) { if (prev && existsSync(prev)) { switchLink(join(sd, 'current'), prev); await ctx.sys.nginxReload().catch(() => {}); } const e = new Error(`내부 확인 실패(${ic.status || ic.error}) — 이전 버전을 유지합니다`); e.code = 'internal_check_failed'; e.partial = { leftovers, previousKept: !!prev, internalCheck: ic }; throw e; } + // 오래된 릴리스 정리(최근 5개 보존 — 되돌리기 대상) + await ctx.step('prune', () => { const rels = readdirSync(join(sd, 'releases')).sort(); const del = rels.slice(0, Math.max(0, rels.length - 5)).filter((r) => join(sd, 'releases', r) !== rel.dir); for (const r of del) rmrf(join(sd, 'releases', r)); return { pruned: del.length }; }, '이전 배포본 정리'); + return { version: rel.name, deployedAt: new Date().toISOString(), internalCheck: ic, siteUrl: p.primaryDomain ? `https://${p.primaryDomain}/` : `http://${vh.serverNames[0]}/`, serverNames: vh.serverNames, https: vh.https, certs: vh.certs, previousVersion: sw.previous ? sw.previous.split('/').pop() : null, runtime, port, leftovers: [] }; + } catch (e) { if (!e.partial) e.partial = { leftovers: leftovers.filter((x) => existsSync(x)), previousKept: true }; throw e; } + } }, + 'site.rollback': { title: '이전 버전으로 되돌리기', async run(ctx) { + const p = ctx.payload; const siteId = assertSiteId(p.siteId); const sd = siteDir(ctx.roots, siteId); + const target = await ctx.step('find', () => { const want = p.toVersion ? join(sd, 'releases', p.toVersion) : readJson(join(sd, '.previous.json'), {}).previous; if (!want || !existsSync(want)) throw Object.assign(new Error('되돌릴 이전 버전이 없습니다'), { code: 'no_previous_version' }); return want; }, '되돌릴 버전 확인'); + const cur = readLink(join(sd, 'current')); + await ctx.step('switch', () => { switchLink(join(sd, 'current'), target); writeJson(join(sd, '.previous.json'), { previous: cur, at: new Date().toISOString() }); return { from: cur, to: target }; }, '버전 전환'); + const unit = `ariweb-site-${siteId}`; if (readJson(join(target, 'ariweb.json'), {}).runtime === 'node') await ctx.step('restart', () => ctx.sys.systemctl('restart', unit), '실행 유닛 재시작'); else await ctx.sys.nginxReload().catch(() => {}); + const ic = await ctx.step('internal', () => internalCheck(ctx, siteId, (p.domains && p.domains[0]) || `${siteId}.preview.${ctx.roots.platformDomain}`), '내부 확인'); + if (!ic.ok) throw Object.assign(new Error('되돌린 버전의 내부 확인에 실패했습니다'), { code: 'internal_check_failed', partial: { internalCheck: ic } }); + return { version: target.split('/').pop(), internalCheck: ic }; + } }, + 'site.remove': { title: '배포 제거', async run(ctx) { + const p = ctx.payload; const siteId = assertSiteId(p.siteId); const sd = siteDir(ctx.roots, siteId); const left = []; + await ctx.step('vhost', () => removeVhost(ctx, siteId), '도메인 연결 해제'); + await ctx.step('unit', async () => { if (existsSync(`/etc/systemd/system/ariweb-site-${siteId}.service`) || ctx.sys.stub) { await ctx.sys.systemctl('stop', `ariweb-site-${siteId}`, { optional: true }); await ctx.sys.systemctl('disable', `ariweb-site-${siteId}`, { optional: true }); await ctx.sys.sudo(['rm', '-f', `/etc/systemd/system/ariweb-site-${siteId}.service`]); } return true; }, '실행 유닛 제거'); + await ctx.step('files', () => { if (p.keepData) { for (const x of ['releases', 'current', 'work']) rmrf(join(sd, x)); if (existsSync(join(sd, 'data'))) left.push(join(sd, 'data')); } else rmrf(sd); return { removed: true }; }, '파일 정리'); + return { removed: true, leftovers: left.filter((x) => existsSync(x)) }; + } }, + 'site.check': { title: '사이트 확인', async run(ctx) { const p = ctx.payload; const siteId = assertSiteId(p.siteId); const checks = await ctx.step('check', async () => { const out = []; for (const u of p.urls || []) { let host = u; try { host = new URL(u).hostname; } catch (_) {} out.push({ url: u, ...(await internalCheck(ctx, siteId, host, { expectMarker: p.expectMarker || null })) }); } return out; }, '내부 확인'); const sd = siteDir(ctx.roots, siteId); return { checks, currentVersion: (readLink(join(sd, 'current')) || '').split('/').pop() || null, maintenance: existsSync(join(sd, '.maintenance')), limited: existsSync(join(sd, '.limited')), tls: (() => { const c = ownCertFor(ctx.roots, siteId) || wildcardCertFor(ctx.roots, p.domains || []); return { valid: !!c, expiresAt: c ? certExpiry(c) : null }; })() }; } }, + 'site.usage': { title: '사용량 수집', async run(ctx) { const out = {}; for (const siteId of ctx.payload.siteIds || []) { try { out[siteId] = await ctx.step(`usage:${siteId}`, () => collectUsage(ctx, siteId), `사용량 ${siteId}`); } catch (e) { out[siteId] = { error: e.message }; } } return { sites: out, measuredAt: new Date().toISOString() }; } }, + 'site.limit': { title: '전송량 제한 표시', async run(ctx) { const sd = siteDir(ctx.roots, assertSiteId(ctx.payload.siteId)); mkdirSync(sd, { recursive: true }); const f = join(sd, '.limited'); if (ctx.payload.limited) writeFileSync(f, new Date().toISOString()); else rmrf(f); return { limited: !!ctx.payload.limited }; } }, + 'site.maintenance': { title: '유지보수 모드', async run(ctx) { const sd = siteDir(ctx.roots, assertSiteId(ctx.payload.siteId)); mkdirSync(sd, { recursive: true }); const f = join(sd, '.maintenance'); if (ctx.payload.on) writeFileSync(f, new Date().toISOString()); else rmrf(f); return { maintenance: !!ctx.payload.on }; } }, + // ---------------- 이전 보조(§21) + 'site.prepare': { title: '새 위치 준비', async run(ctx) { const sd = siteDir(ctx.roots, assertSiteId(ctx.payload.siteId)); mkdirSync(join(sd, 'releases'), { recursive: true }); mkdirSync(join(sd, 'staging'), { recursive: true }); mkdirSync(join(sd, 'data'), { recursive: true }); const free = (() => { try { const { statfsSync } = ctx.sys.stub ? { statfsSync: () => ({ bfree: 1e9, bsize: 1 }) } : {}; return null; } catch (_) { return null; } })(); return { prepared: true, dir: sd }; } }, + 'site.export': { title: '사이트 내보내기', async run(ctx) { const p = ctx.payload; const siteId = assertSiteId(p.siteId); const sd = siteDir(ctx.roots, siteId); const cur = readLink(join(sd, 'current')); if (!cur) throw Object.assign(new Error('운영 중인 배포본이 없습니다'), { code: 'no_current' }); const work = join(sd, 'work', `export-${stamp()}`); mkdirSync(work, { recursive: true }); const t = await ctx.step('tar', async () => { mkdirSync(join(work, 'pack'), { recursive: true }); await ctx.sys.tar(['-cf', join(work, 'pack', 'files.tar'), '-C', cur, '.']); if (p.includeData !== false && existsSync(join(sd, 'data'))) await ctx.sys.tar(['-cf', join(work, 'pack', 'data.tar'), '-C', join(sd, 'data'), '.']); writeJson(join(work, 'pack', 'meta.json'), { siteId, version: cur.split('/').pop(), exportedAt: new Date().toISOString(), ariweb: readJson(join(cur, 'ariweb.json'), {}) }); return tarDir(ctx, join(work, 'export.tar.gz'), join(work, 'pack')); }, '묶기'); const up = await ctx.step('upload', () => ctx.client.upload(p.transferId, t.file, { purpose: 'relocate' }), 'HAL 로 올리기'); rmrf(work); return { transferId: up.id, checksum: fmtSum(up.sha256), sizeBytes: up.size, version: cur.split('/').pop() }; } }, + 'site.import': { title: '사이트 받기', async run(ctx) { const p = ctx.payload; const siteId = assertSiteId(p.siteId); const sd = siteDir(ctx.roots, siteId); const stage = join(sd, 'staging', p.stage || 'pre'); rmrf(stage); mkdirSync(stage, { recursive: true }); const dl = await ctx.step('download', () => ctx.client.downloadTransfer(p.transferId, join(stage, 'export.tar.gz'), p.checksum || null), '받기'); await ctx.step('unpack', async () => { await untar(ctx, join(stage, 'export.tar.gz'), join(stage, 'pack')); mkdirSync(join(stage, 'files'), { recursive: true }); await ctx.sys.tar(['-xf', join(stage, 'pack', 'files.tar'), '-C', join(stage, 'files')]); if (existsSync(join(stage, 'pack', 'data.tar'))) { mkdirSync(join(stage, 'data'), { recursive: true }); await ctx.sys.tar(['-xf', join(stage, 'pack', 'data.tar'), '-C', join(stage, 'data')]); } return true; }, '풀기'); const meta = readJson(join(stage, 'pack', 'meta.json'), {}); writeJson(join(sd, 'staging', 'latest.json'), { stage: p.stage || 'pre', checksum: fmtSum(dl.sha256), meta, at: new Date().toISOString() }); return { ok: dl.sha256 === normSum(p.checksum || dl.sha256), checksum: fmtSum(dl.sha256), sizeBytes: dl.size, version: meta.version || null, files: readdirSync(join(stage, 'files')).length }; } }, + 'site.stage_check': { title: '새 위치 확인', async run(ctx) { const p = ctx.payload; const sd = siteDir(ctx.roots, assertSiteId(p.siteId)); const latest = readJson(join(sd, 'staging', 'latest.json'), null); const stage = latest ? join(sd, 'staging', latest.stage) : null; const ok = !!(latest && stage && existsSync(join(stage, 'files')) && (!p.expectedChecksum || normSum(latest.checksum) === normSum(p.expectedChecksum)) && readdirSync(join(stage, 'files')).length > 0); return { ok, checksum: latest?.checksum || null, expected: p.expectedChecksum || null, files: ok ? readdirSync(join(stage, 'files')).length : 0, reason: ok ? null : '스테이징 사본이 없거나 체크섬이 다릅니다' }; } }, + 'site.activate': { title: '새 위치에서 서비스 시작', async run(ctx) { const p = ctx.payload; const siteId = assertSiteId(p.siteId); const sd = siteDir(ctx.roots, siteId); const latest = readJson(join(sd, 'staging', 'latest.json'), null); const rel = await ctx.step('place', () => { if (p.reactivate && readLink(join(sd, 'current'))) return { dir: readLink(join(sd, 'current')), reused: true }; if (!latest) throw Object.assign(new Error('스테이징 사본이 없습니다'), { code: 'no_staging' }); const dir = join(sd, 'releases', `${stamp()}-reloc`); renameSync(join(sd, 'staging', latest.stage, 'files'), dir); if (existsSync(join(sd, 'staging', latest.stage, 'data'))) { const old = join(sd, 'data'); if (existsSync(old)) renameSync(old, join(sd, `data.pre-reloc-${stamp()}`)); renameSync(join(sd, 'staging', latest.stage, 'data'), old); } writeJson(join(dir, '.ariweb-version.json'), { version: latest.meta?.version || dir.split('/').pop(), relocatedAt: new Date().toISOString() }); return { dir }; }, '배포본 배치'); switchLink(join(sd, 'current'), rel.dir); rmrf(join(sd, '.frozen')); let port = null; const meta = readJson(join(rel.dir, 'ariweb.json'), {}); if ((p.runtime || meta.runtime) === 'node') { port = sitePort(siteId, p); await ctx.step('unit', () => nodeUnit(ctx, siteId, sd, meta.start || 'server.js', port), '실행 유닛'); } const vh = await ctx.step('vhost', () => configureVhost(ctx, siteId, { domains: p.domains || [], primaryDomain: p.primaryDomain || null, runtime: p.runtime || meta.runtime || 'static', port }), '도메인 연결'); const ic = await internalCheck(ctx, siteId, vh.serverNames[0]); if (!ic.ok) throw Object.assign(new Error('새 위치 내부 확인 실패'), { code: 'internal_check_failed', partial: { internalCheck: ic } }); return { version: rel.dir.split('/').pop(), internalCheck: ic, serverNames: vh.serverNames }; } }, + 'site.deactivate': { title: '이전 위치 서비스 중지(사본 보존)', async run(ctx) { const p = ctx.payload; const siteId = assertSiteId(p.siteId); const sd = siteDir(ctx.roots, siteId); await ctx.step('vhost', () => removeVhost(ctx, siteId), '도메인 연결 해제'); await ctx.step('unit', () => ctx.sys.systemctl('stop', `ariweb-site-${siteId}`, { optional: true }), '실행 유닛 중지'); const until = new Date(Date.now() + (p.retainDays || 7) * 86400000).toISOString(); writeJson(join(sd, '.retained.json'), { until, reason: 'relocated', at: new Date().toISOString() }); rmrf(join(sd, '.frozen')); return { retainedUntil: until }; } }, + 'site.freeze': { title: '쓰기 제한', async run(ctx) { const sd = siteDir(ctx.roots, assertSiteId(ctx.payload.siteId)); mkdirSync(sd, { recursive: true }); const f = join(sd, '.frozen'); if (ctx.payload.freeze) writeFileSync(f, new Date().toISOString()); else rmrf(f); return { frozen: !!ctx.payload.freeze, at: new Date().toISOString() }; } }, +}; + +// ---------------- 사용량(§13.1) +const DAY_KEEP = 400; +function usageFile(ctx, siteId) { return join(ctx.dataDir, 'usage', `${siteId}.json`); } +/** awstats 로그 증분 읽기(inode·offset 저장) → 일별 집계 */ +export function ingestLog(ctx, siteId, logPath, state) { + if (!existsSync(logPath)) return state; + const fd = openSync(logPath, 'r'); try { + const st = fstatSync(fd); const off = state.offsets?.[logPath] || { inode: null, offset: 0 }; let start = off.inode === st.ino && off.offset <= st.size ? off.offset : 0; + const buf = Buffer.alloc(Math.min(st.size - start, 64 * 1024 * 1024)); const n = readSync(fd, buf, 0, buf.length, start); const text = buf.toString('utf8', 0, n); const lastNl = text.lastIndexOf('\n'); const usable = lastNl >= 0 ? text.slice(0, lastNl + 1) : ''; + for (const line of usable.split('\n')) { if (!line) continue; const f = line.split('|'); if (f.length < 7) continue; const day = f[0].slice(0, 10); const d = state.days[day] || (state.days[day] = { req: 0, bytesIn: 0, bytesOut: 0, err: 0, hist: {} }); d.req++; d.bytesIn += Number(f[2]) || 0; d.bytesOut += Number(f[3]) || 0; if (Number(f[4]) >= 500) d.err++; const ms = Math.round((Number(f[5]) || 0) * 1000); const bucket = ms < 100 ? '0' : ms < 300 ? '100' : ms < 1000 ? '300' : ms < 3000 ? '1000' : '3000'; d.hist[bucket] = (d.hist[bucket] || 0) + 1; } + state.offsets = state.offsets || {}; state.offsets[logPath] = { inode: st.ino, offset: start + Buffer.byteLength(usable) }; + } finally { closeSync(fd); } + const days = Object.keys(state.days).sort(); for (const k of days.slice(0, Math.max(0, days.length - DAY_KEEP))) delete state.days[k]; + return state; +} +function p95(hist) { const order = ['0', '100', '300', '1000', '3000']; const total = order.reduce((a, k) => a + (hist[k] || 0), 0); if (!total) return null; let acc = 0; for (const k of order) { acc += hist[k] || 0; if (acc / total >= 0.95) return Number(k); } return 3000; } +export async function collectUsage(ctx, siteId) { + assertSiteId(siteId); const sd = siteDir(ctx.roots, siteId); const f = usageFile(ctx, siteId); const state = readJson(f, { days: {}, offsets: {} }); + ingestLog(ctx, siteId, join(ctx.roots.logs, `aw-${siteId}.access.log`), state); writeJson(f, state); + const today = new Date().toISOString().slice(0, 10); const kst = (d) => new Date(new Date(d).getTime() + 9 * 3600000).toISOString().slice(0, 10); const kToday = kst(Date.now()); const month = kToday.slice(0, 7); + const sum = (filter, key) => Object.entries(state.days).filter(([d]) => filter(d)).reduce((a, [, v]) => a + (v[key] || 0), 0); + const series = (n) => { const out = []; for (let i = n - 1; i >= 0; i--) { const d = new Date(Date.now() - i * 86400000).toISOString().slice(0, 10); const v = state.days[d] || {}; out.push({ date: d, req: v.req || 0, bytesOut: v.bytesOut || 0, bytesIn: v.bytesIn || 0, err: v.err || 0 }); } return out; }; + const cur = readLink(join(sd, 'current')); const dbDir = join(sd, 'data', 'db'); const last30 = (d) => Date.now() - Date.parse(d) < 30 * 86400000; + const errs = sum(last30, 'err'), reqs = sum(last30, 'req'); const histAll = {}; for (const [d, v] of Object.entries(state.days)) if (last30(d)) for (const [k, c] of Object.entries(v.hist || {})) histAll[k] = (histAll[k] || 0) + c; + return { diskBytes: cur ? dirSize(cur) : 0, dbBytes: dirSize(dbDir), uploadsBytes: dirSize(join(sd, 'data', 'uploads')), outBytesDay: sum((d) => d === today, 'bytesOut'), outBytesMonth: sum((d) => kst(d + 'T12:00:00Z').startsWith(month), 'bytesOut'), outBytesTotal: sum(() => true, 'bytesOut'), inBytesDay: sum((d) => d === today, 'bytesIn'), inBytesMonth: sum((d) => kst(d + 'T12:00:00Z').startsWith(month), 'bytesIn'), inBytesTotal: sum(() => true, 'bytesIn'), req24h: sum((d) => d === today, 'req'), pv24h: sum((d) => d === today, 'req'), series7d: series(7), series30d: series(30), errorRate: reqs ? Math.round(errs / reqs * 10000) / 100 : 0, p95ms: p95(histAll), measuredAt: new Date().toISOString(), kstMonth: month }; +} diff --git a/agent/metrics.mjs b/agent/metrics.mjs new file mode 100644 index 0000000..9e61a75 --- /dev/null +++ b/agent/metrics.mjs @@ -0,0 +1,25 @@ +/** + * agent/metrics.mjs — 서버 지표(§28.1): CPU·메모리·디스크·inode·I/O·네트워크·load. /proc 와 df 만 읽는다(외부 도구 없음). + * CPU 는 두 표본 사이의 사용률(%). I/O·네트워크는 누적 바이트(HAL 이 차분으로 속도를 구한다). 읽을 수 없는 항목은 null. + */ +import { readFileSync, statfsSync, existsSync } from 'node:fs'; +import { loadavg, hostname, release, totalmem, freemem } from 'node:os'; +import { execFileSync } from 'node:child_process'; + +let lastCpu = null; +function cpuPercent() { + try { const l = readFileSync('/proc/stat', 'utf8').split('\n')[0].trim().split(/\s+/).slice(1).map(Number); const idle = l[3] + (l[4] || 0); const total = l.reduce((a, b) => a + b, 0); const prev = lastCpu; lastCpu = { idle, total }; if (!prev) return null; const dt = total - prev.total; return dt > 0 ? Math.round((1 - (idle - prev.idle) / dt) * 1000) / 10 : null; } catch (_) { return null; } +} +function mem() { try { const m = Object.fromEntries(readFileSync('/proc/meminfo', 'utf8').split('\n').filter(Boolean).map((l) => { const [k, v] = l.split(':'); return [k, parseInt(v, 10) * 1024]; })); return { total: m.MemTotal, used: m.MemTotal - (m.MemAvailable ?? m.MemFree) }; } catch (_) { return { total: totalmem(), used: totalmem() - freemem() }; } } +function disk(path = '/') { try { const s = statfsSync(path); return { total: s.blocks * s.bsize, used: (s.blocks - s.bfree) * s.bsize, inode: { total: s.files, used: s.files - s.ffree } }; } catch (_) { return null; } } +function io() { try { let r = 0, w = 0; for (const l of readFileSync('/proc/diskstats', 'utf8').split('\n')) { const f = l.trim().split(/\s+/); if (f.length < 14 || !/^(sd|vd|nvme|xvd|md|dm)/.test(f[2]) || /\d+p?\d+$/.test(f[2]) && !/^(md|dm)/.test(f[2])) continue; r += Number(f[5]) * 512; w += Number(f[9]) * 512; } return { read: r, write: w }; } catch (_) { return null; } } +function net() { try { let i = 0, o = 0; for (const l of readFileSync('/proc/net/dev', 'utf8').split('\n').slice(2)) { const [name, rest] = l.split(':'); if (!rest || name.trim() === 'lo') continue; const f = rest.trim().split(/\s+/); i += Number(f[0]); o += Number(f[8]); } return { in: i, out: o }; } catch (_) { return null; } } +/** heartbeat 에 실을 지표 묶음 */ +export function sample(roots = {}) { const dk = disk('/'); return { cpu: cpuPercent(), mem: mem(), disk: dk ? { total: dk.total, used: dk.used } : null, inode: dk ? dk.inode : null, io: io(), net: net(), load1: loadavg()[0], at: new Date().toISOString() }; } +/** 서버 사실(facts): 등록·heartbeat 때 HAL 에 알린다. 비밀 없음. */ +export function facts(sys) { + const f = { hostname: hostname(), kernel: release(), node: process.version, nginx: existsSync('/usr/sbin/nginx') || existsSync('/usr/bin/nginx'), certbot: existsSync('/usr/bin/certbot') || existsSync('/snap/bin/certbot'), units: {}, appDirs: {} }; + try { const out = execFileSync('systemctl', ['list-units', '--type=service', '--all', '--no-pager', '--no-legend', '--plain'], { encoding: 'utf8', timeout: 5000 }); const names = out.split('\n').map((l) => l.trim().split(/\s+/)[0]).filter(Boolean).map((n) => n.replace(/\.service$/, '')); + for (const [mod, cands] of Object.entries({ webmaker: ['webmaker'], arimail: ['arimail', 'arihakdang-mail', 'heart-mail', 'mail-gateway'], hal: ['ariweb-hal'], portal: ['ariweb-portal'], agent: ['ariweb-agent'] })) { const u = cands.find((c) => names.includes(c)); if (u) { f.units[mod] = u; try { const txt = execFileSync('systemctl', ['cat', u], { encoding: 'utf8', timeout: 5000 }); const wd = /^WorkingDirectory=(.+)$/m.exec(txt); if (wd) f.appDirs[mod] = wd[1].replace(/\/current\/?$/, ''); const user = /^User=(.+)$/m.exec(txt); if (user) (f.users = f.users || {})[mod] = user[1]; const envf = /^EnvironmentFile=(.+)$/m.exec(txt); if (envf) (f.envFiles = f.envFiles || {})[mod] = envf[1]; const node = /^ExecStart=(\S+)/m.exec(txt); if (node) (f.nodeBins = f.nodeBins || {})[mod] = node[1]; } catch (_) {} } } } catch (_) {} + return f; +} diff --git a/agent/runner.mjs b/agent/runner.mjs new file mode 100644 index 0000000..f7204c3 --- /dev/null +++ b/agent/runner.mjs @@ -0,0 +1,57 @@ +/** + * agent/runner.mjs — Agent 작업 실행기(단계 저장·재개·진행 보고). + * + * HAL 의 lib/jobs 와 같은 원칙: 작업은 재진입 가능하고 끝난 단계는 건너뛴다. 단계 상태는 data/jobs/.json 에 저장되어 Agent 재시작 뒤 이어서 실행된다. + * ctx = { job, payload, roots, sys, client, dataDir, step(id, fn, title), log(line), progress(pct), needsUser(message, hint), checkCancel(), steps() } + * 결과 보고: 성공 { status:'done', result } / 실패 { status:'failed', error, errorCode } / 사용자·운영자 확인 { status:'needs_user', userMessage }. + */ +import { readFileSync, writeFileSync, mkdirSync, existsSync, unlinkSync, renameSync } from 'node:fs'; +import { join } from 'node:path'; +import { redact, safeError } from '../lib/redact.mjs'; +import { NeedsUser } from './sys.mjs'; + +class Signal extends Error { constructor(type, payload) { super(type); this.signal = type; this.payload = payload || {}; } } + +export function createRunner({ kinds, roots, sys, client, dataDir, log = () => {} }) { + const jobsDir = join(dataDir, 'jobs'); mkdirSync(jobsDir, { recursive: true }); + const stateFile = (id) => join(jobsDir, `${id}.json`); + const loadState = (id) => { try { return JSON.parse(readFileSync(stateFile(id), 'utf8')); } catch (_) { return { steps: [], log: [] }; } }; + const saveState = (id, st) => { const tmp = stateFile(id) + '.tmp'; writeFileSync(tmp, JSON.stringify(st)); renameSync(tmp, stateFile(id)); }; + /** 진행 보고(HAL). 실패해도 작업은 계속한다(다음 보고에서 따라잡음). */ + async function report(id, body) { if (!client) return null; try { return await client.request('POST', `/agent/v1/jobs/${id}/progress`, body); } catch (e) { log(`[agent] 진행 보고 실패 ${id}: ${safeError(e)}`); return null; } } + async function execute(job) { + const def = kinds[job.kind]; const st = loadState(job.id); let cancelled = false; + const ctx = { + job, payload: job.payload || {}, roots, sys, client, dataDir, kinds, + async step(id, fn, title) { + let s = st.steps.find((x) => x.id === id); if (!s) { s = { id, title: title || id, state: 'pending' }; st.steps.push(s); } + if (s.state === 'done') return s.result; + s.state = 'running'; s.startedAt = new Date().toISOString(); delete s.error; saveState(job.id, st); await report(job.id, { steps: st.steps }); + try { const r = await fn(); s.state = 'done'; s.endedAt = new Date().toISOString(); s.result = r === undefined ? null : redact(r); saveState(job.id, st); return r; } + catch (e) { s.state = e instanceof Signal ? 'needs_user' : 'failed'; s.error = safeError(e); s.endedAt = new Date().toISOString(); saveState(job.id, st); throw e; } + }, + log(line) { st.log.push({ at: new Date().toISOString(), line: String(line).slice(0, 500) }); st.log = st.log.slice(-200); saveState(job.id, st); log(`[${job.kind} ${job.id}] ${line}`); }, + async progress(pct) { st.progress = Math.max(0, Math.min(100, Math.round(pct))); saveState(job.id, st); const r = await report(job.id, { progress: st.progress, steps: st.steps, log: st.log.slice(-20) }); if (r && r.cancelRequested) cancelled = true; }, + needsUser(message, hint) { throw new Signal('needs_user', { message, hint }); }, + checkCancel() { if (cancelled) throw new Signal('cancel', {}); }, + steps: () => st.steps, state: st, + }; + let out; + if (!def) out = { status: 'failed', error: `알 수 없는 작업 종류: ${job.kind}`, errorCode: 'unknown_kind' }; + else { + try { const result = await def.run(ctx); out = { status: 'done', result: redact(result === undefined ? {} : result), steps: st.steps, log: st.log.slice(-50) }; } + catch (e) { + if (e instanceof Signal && e.signal === 'needs_user') out = { status: 'needs_user', userMessage: `${e.payload.message}${e.payload.hint ? ` (${e.payload.hint})` : ''}`, steps: st.steps, log: st.log.slice(-50) }; + else if (e instanceof Signal && e.signal === 'cancel') out = { status: 'failed', error: '취소 요청으로 중단', errorCode: 'cancelled', steps: st.steps, log: st.log.slice(-50) }; + else if (e instanceof NeedsUser || e.needsUser) out = { status: 'needs_user', userMessage: `${e.message}${e.hint ? ` — ${e.hint}` : ''}`, steps: st.steps, log: st.log.slice(-50) }; + else out = { status: 'failed', error: safeError(e), errorCode: e.code || 'failed', steps: st.steps, log: st.log.slice(-50), result: e.partial ? redact(e.partial) : undefined }; + } + } + try { unlinkSync(stateFile(job.id)); } catch (_) {} + if (client) { try { await client.request('POST', `/agent/v1/jobs/${job.id}/result`, out); } catch (e) { log(`[agent] 결과 보고 실패 ${job.id}: ${safeError(e)}`); saveState(job.id, { ...st, pendingResult: out }); } } + return out; + } + /** 재시작 뒤: 보고하지 못한 결과가 남아 있으면 다시 보고 */ + async function flushPending() { if (!client) return; const { readdirSync } = await import('node:fs'); for (const f of readdirSync(jobsDir).filter((x) => x.endsWith('.json'))) { const st = loadState(f.slice(0, -5)); if (st.pendingResult) { try { await client.request('POST', `/agent/v1/jobs/${f.slice(0, -5)}/result`, st.pendingResult); unlinkSync(join(jobsDir, f)); } catch (_) {} } } } + return { execute, loadState, flushPending, kinds }; +} diff --git a/agent/sys.mjs b/agent/sys.mjs new file mode 100644 index 0000000..9276929 --- /dev/null +++ b/agent/sys.mjs @@ -0,0 +1,36 @@ +/** + * agent/sys.mjs — 시스템 명령 실행 계층(sudo·systemctl·nginx·certbot·tar) 과 권한 있는 파일 쓰기. + * + * Agent 는 일반 계정(master)으로 돌고 systemctl/nginx/certbot/tee 만 sudo -n(비밀번호 없음)으로 쓴다(deploy/sudoers/ariweb-agent). + * sudo 가 막혀 있으면 조용히 실패하지 않고 NEEDS_USER 로 운영자에게 이유(힌트)를 보인다. + * stub 모드(ARIWEB_AGENT_STUB_CMDS=1, 테스트): 외부 명령을 실행하지 않고 호출만 기록하며 성공으로 돌려준다. tar 는 실제로 실행한다(파일 결과를 검증하기 위해). + */ +import { spawn } from 'node:child_process'; +import { writeFileSync, mkdirSync, existsSync, readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; + +export class NeedsUser extends Error { constructor(message, hint) { super(message); this.needsUser = true; this.hint = hint; } } +export function createSys({ stub = process.env.ARIWEB_AGENT_STUB_CMDS === '1', log = () => {} } = {}) { + const calls = []; + /** 명령 실행. 실패해도 예외 대신 { code, stdout, stderr } 를 돌려준다(호출자가 판단). */ + function run(cmd, args = [], { input = null, timeoutMs = 600000, allowStub = true } = {}) { + calls.push({ cmd, args }); + if (stub && allowStub && cmd !== 'tar' && cmd !== 'node') return Promise.resolve({ code: 0, stdout: '', stderr: '', stubbed: true }); + return new Promise((resolve) => { const p = spawn(cmd, args, { stdio: ['pipe', 'pipe', 'pipe'] }); let out = '', err = ''; const t = setTimeout(() => p.kill('SIGKILL'), timeoutMs); p.stdout.on('data', (c) => { out += c; }); p.stderr.on('data', (c) => { err += c; }); p.on('error', (e) => { clearTimeout(t); resolve({ code: 127, stdout: out, stderr: e.message }); }); p.on('close', (code) => { clearTimeout(t); resolve({ code, stdout: out, stderr: err }); }); if (input != null) p.stdin.end(input); else p.stdin.end(); }); + } + const sudo = (args, o) => run('sudo', ['-n', ...args], o); + async function hasSudo() { if (stub) return true; const r = await run('sudo', ['-n', 'true']); return r.code === 0; } + async function requireSudo(what) { if (!(await hasSudo())) throw new NeedsUser(`${what} 에 sudo 권한이 필요하지만 이 서버에서 비밀번호 없는 sudo 가 허용되지 않았습니다`, 'deploy/sudoers/ariweb-agent 를 /etc/sudoers.d/ 에 설치하세요'); } + /** 권한이 필요한 경로에 파일 쓰기: 직접 쓰기 → 안 되면 sudo tee */ + async function writeFile(path, content, mode = '644') { + try { mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, content); return { via: 'direct' }; } catch (e) { if (e.code !== 'EACCES' && e.code !== 'EPERM') throw e; } + await requireSudo(`${path} 쓰기`); await sudo(['mkdir', '-p', dirname(path)]); const r = await sudo(['tee', path], { input: content }); if (r.code !== 0) throw new Error(`쓰기 실패 ${path}: ${r.stderr}`); await sudo(['chmod', mode, path]); return { via: 'sudo' }; + } + async function systemctl(action, unit, { optional = false } = {}) { await requireSudo(`systemctl ${action} ${unit}`); const r = await sudo(['systemctl', action, unit]); if (r.code !== 0 && !optional) throw new Error(`systemctl ${action} ${unit} 실패: ${(r.stderr || r.stdout).trim().slice(0, 300)}`); return r; } + async function isActive(unit) { if (stub) return true; const r = await run('systemctl', ['is-active', unit]); return r.stdout.trim() === 'active'; } + async function unitText(unit) { if (stub) return ''; const r = await run('systemctl', ['cat', unit]); return r.code === 0 ? r.stdout : ''; } + async function nginxReload() { await requireSudo('nginx 설정 적용'); const t = await sudo(['nginx', '-t']); if (t.code !== 0) throw new Error(`nginx -t 실패: ${(t.stderr || t.stdout).trim().slice(0, 400)}`); const r = await sudo(['systemctl', 'reload', 'nginx']); if (r.code !== 0) throw new Error(`nginx reload 실패: ${r.stderr.slice(0, 300)}`); return true; } + async function certbot(args) { await requireSudo('certbot'); return sudo(['certbot', ...args], { timeoutMs: 300000 }); } + async function tar(args, o) { const r = await run('tar', args, { ...o, allowStub: false }); if (r.code !== 0) throw new Error(`tar 실패: ${r.stderr.slice(0, 300)}`); return r; } + return { stub, calls, run, sudo, hasSudo, requireSudo, writeFile, systemctl, isActive, unitText, nginxReload, certbot, tar, readIfExists: (p) => (existsSync(p) ? readFileSync(p, 'utf8') : null) }; +} diff --git a/deploy/agent-install.sh b/deploy/agent-install.sh new file mode 100755 index 0000000..ab8ec28 --- /dev/null +++ b/deploy/agent-install.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# deploy/agent-install.sh — 관리 서버 또는 고객 외부 서버에 Node Agent 설치·등록(§28.2·§10.2). HAL 이 /agent/v1/install.sh 로 제공한다. +# curl -fsSL https://hal.ariweb.net/agent/v1/install.sh | bash -s -- --hal https://hal.ariweb.net --token <일회용 토큰> [--role web|mail|backup|control] [--external --site ] [--app-dir /srv/ariweb/agent] +# 하는 일: node 22+ 확인(없으면 NodeSource 로 설치), agent artifact 를 HAL 에서 받아 /releases/ 에 풀고 current 연결, 등록(토큰 1회), systemd 유닛 ariweb-agent 설치·시작, sudoers 최소 규칙(관리형만). +# 멱등: 다시 실행하면 이미 등록된 Agent 는 재등록하지 않고 코드·유닛만 맞춘다. 토큰 외의 비밀은 출력하지 않는다. +set -euo pipefail +HAL=""; TOKEN=""; ROLE="web"; EXTERNAL=0; SITE=""; APP_DIR="/srv/ariweb/agent"; LABEL="$(hostname)" +while [ $# -gt 0 ]; do case "$1" in --hal) HAL="$2"; shift 2;; --token) TOKEN="$2"; shift 2;; --role) ROLE="$2"; shift 2;; --external) EXTERNAL=1; shift;; --site) SITE="$2"; shift 2;; --app-dir) APP_DIR="$2"; shift 2;; --label) LABEL="$2"; shift 2;; *) echo "알 수 없는 옵션 $1" >&2; exit 1;; esac; done +[ -n "$HAL" ] || { echo "--hal 이 필요합니다" >&2; exit 1; } +HAL="${HAL%/}"; USER_NAME="$(id -un)" +if ! command -v node >/dev/null 2>&1 || [ "$(node --version | sed -E 's/^v([0-9]+).*/\1/')" -lt 22 ]; then echo "== node 22 설치"; curl -fsSL https://deb.nodesource.com/setup_22.x | sudo -n bash - >/dev/null && sudo -n apt-get install -y -q nodejs >/dev/null; fi +NODE_BIN="$(command -v node)" +echo "== HAL 확인 $HAL"; HEALTH="$(curl -fsS "$HAL/api/health")"; VERSION="$(echo "$HEALTH" | $NODE_BIN -e "let s='';process.stdin.on('data',c=>s+=c).on('end',()=>console.log(JSON.parse(s).bootstrap.releaseVersion||JSON.parse(s).version))")" +sudo -n mkdir -p "$APP_DIR/releases/$VERSION" "$APP_DIR/data" /var/www/ariweb-sites /srv/ariweb/backups; sudo -n chown -R "$USER_NAME:$USER_NAME" "$APP_DIR" /var/www/ariweb-sites /srv/ariweb/backups; chmod 700 "$APP_DIR/data" +if [ ! -f "$APP_DIR/releases/$VERSION/agent/index.mjs" ]; then + echo "== agent artifact $VERSION 내려받기" + # 등록 전에는 서명을 만들 수 없으므로 부트스트랩 전용 공개 경로가 아니라 등록 토큰으로 1회 받는다: HAL 의 install.sh 는 토큰과 함께 artifact 를 함께 보낸다(아래 등록 뒤 재시도 경로). + if [ -n "$TOKEN" ]; then curl -fsS -H "X-Registration-Token: $TOKEN" "$HAL/agent/v1/bootstrap-artifact?module=agent&version=$VERSION" -o /tmp/ariweb-agent-$$.tgz; tar -xzf /tmp/ariweb-agent-$$.tgz -C "$APP_DIR/releases/$VERSION"; rm -f /tmp/ariweb-agent-$$.tgz; else echo "코드가 없고 토큰도 없습니다" >&2; exit 1; fi +fi +ln -sfn "$APP_DIR/releases/$VERSION" "$APP_DIR/current"; cd "$APP_DIR/current" +export ARIWEB_AGENT_DATA="$APP_DIR/data" +if [ ! -f "$APP_DIR/data/agent.json" ]; then [ -n "$TOKEN" ] || { echo "등록되지 않았고 --token 도 없습니다" >&2; exit 1; }; echo "== 등록"; if [ "$EXTERNAL" = "1" ]; then $NODE_BIN agent/index.mjs register --hal "$HAL" --token "$TOKEN" --external --site "$SITE" --label "$LABEL"; else $NODE_BIN agent/index.mjs register --hal "$HAL" --token "$TOKEN" --role "$ROLE" --label "$LABEL"; fi; else echo "== 이미 등록됨($(cat "$APP_DIR/data/agent.json" | $NODE_BIN -e "let s='';process.stdin.on('data',c=>s+=c).on('end',()=>console.log(JSON.parse(s).id))")) — 코드·유닛만 갱신"; fi +echo "== systemd 유닛" +sed -e "s#{{APP_DIR}}#$APP_DIR#g" -e "s#{{USER}}#$USER_NAME#g" -e "s#{{NODE}}#$NODE_BIN#g" -e "s#{{VERSION}}#$VERSION#g" -e "s#{{ENV_LINES}}#$([ "$EXTERNAL" = "1" ] && echo "Environment=ARIWEB_AGENT_EXTERNAL=1" || echo "")#" deploy/units/ariweb-agent.service.tmpl | sudo -n tee /etc/systemd/system/ariweb-agent.service >/dev/null +if [ "$EXTERNAL" = "0" ]; then sudo -n install -m 440 deploy/sudoers/ariweb-agent /etc/sudoers.d/ariweb-agent && sudo -n visudo -c >/dev/null; sudo -n cp deploy/nginx/awstats.conf /etc/nginx/conf.d/awstats.conf 2>/dev/null && (sudo -n nginx -t >/dev/null 2>&1 && sudo -n systemctl reload nginx || true); fi +sudo -n systemctl daemon-reload; sudo -n systemctl enable --now ariweb-agent >/dev/null; sudo -n systemctl restart ariweb-agent; sleep 2 +systemctl is-active --quiet ariweb-agent && echo "== 완료: ariweb-agent 실행 중. HAL 운영자 콘솔에서 서버를 승인하면 배치가 가능합니다" || { echo "ariweb-agent 기동 실패 — journalctl -u ariweb-agent" >&2; exit 1; } diff --git a/deploy/bootstrap-hal.sh b/deploy/bootstrap-hal.sh new file mode 100755 index 0000000..95192fb --- /dev/null +++ b/deploy/bootstrap-hal.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# deploy/bootstrap-hal.sh — HAL 최초 설치(부트스트랩, Dev-Order §32.2). 최초 설치에서만 허용되는 작업: +# 1) 승인된 main 릴리스의 고정 artifact 선택 2) HAL 저장 기능·비밀정보 보관소·인증서 기반·접근 경로 준비 3) HAL 설치 4) 첫 관리 서버 Node Agent 등록 토큰 발급 +# 그 뒤 모든 모듈(포털·Webmaker·AriMail·Agent 갱신 등)은 HAL 작업으로만 설치한다. 이 스크립트는 HAL 데이터에 부트스트랩이 완료된 흔적이 있으면 실행을 거부한다. +# +# 사용(대상 서버에서, sudo 가능한 일반 계정으로): +# bash deploy/bootstrap-hal.sh --dist 디렉터리(manifest.json·artifact)> --host hal.ariweb.net --cert /etc/letsencrypt/live/wh-wild-ariweb.net [--portal-host my.ariweb.net] [--app-dir /srv/ariweb/hal] +# 출력: 운영자 최초 설정 토큰(1회)과 첫 Agent 설치 명령. 비밀값은 그 밖에 출력하지 않는다. +set -euo pipefail +DIST=""; HOST="hal.ariweb.net"; CERT="/etc/letsencrypt/live/wh-wild-ariweb.net"; APP_DIR="/srv/ariweb/hal"; PORTAL_HOST="my.ariweb.net"; ACME="/var/www/letsencrypt"; USER_NAME="$(id -un)" +while [ $# -gt 0 ]; do case "$1" in --dist) DIST="$2"; shift 2;; --host) HOST="$2"; shift 2;; --cert) CERT="$2"; shift 2;; --app-dir) APP_DIR="$2"; shift 2;; --portal-host) PORTAL_HOST="$2"; shift 2;; --acme) ACME="$2"; shift 2;; *) echo "알 수 없는 옵션 $1" >&2; exit 1;; esac; done +[ -n "$DIST" ] && [ -f "$DIST/manifest.json" ] || { echo "--dist 가 필요합니다" >&2; exit 1; } +NODE_BIN="$(command -v node || true)"; [ -n "$NODE_BIN" ] || { echo "node 22.13+ 가 필요합니다" >&2; exit 1; } +MAJOR="$($NODE_BIN --version | sed -E 's/^v([0-9]+).*/\1/')"; [ "$MAJOR" -ge 22 ] || { echo "Node 22.13+ 필요: $($NODE_BIN --version)" >&2; exit 1; } +sudo -n true 2>/dev/null || { echo "비밀번호 없는 sudo 가 필요합니다" >&2; exit 1; } +VERSION="$($NODE_BIN -e "console.log(JSON.parse(require('fs').readFileSync('$DIST/manifest.json','utf8')).version)")" +COMMIT="$($NODE_BIN -e "console.log(JSON.parse(require('fs').readFileSync('$DIST/manifest.json','utf8')).mainCommit)")" +DIRTY="$($NODE_BIN -e "const m=JSON.parse(require('fs').readFileSync('$DIST/manifest.json','utf8'));console.log(m.checks&&m.checks.dirty?'1':'0')")" +[ "$DIRTY" = "0" ] || { echo "main 에 병합되지 않은(dirty) 빌드는 부트스트랩에 쓸 수 없습니다" >&2; exit 1; } +ART="$DIST/hal-$VERSION.tar.gz"; [ -f "$ART" ] || { echo "artifact 없음: $ART" >&2; exit 1; } +SUM="$(sha256sum "$ART" | cut -d' ' -f1)"; WANT="$($NODE_BIN -e "console.log(JSON.parse(require('fs').readFileSync('$DIST/manifest.json','utf8')).artifacts.hal.sha256)")" +[ "$SUM" = "$WANT" ] || { echo "artifact sha256 불일치" >&2; exit 1; } +echo "== 1/6 릴리스 $VERSION (main $COMMIT, hal artifact sha256 ${SUM:0:16}…)" +DATA="$APP_DIR/data" +if [ -f "$DATA/hal.sqlite" ] && $NODE_BIN -e "const {DatabaseSync}=require('node:sqlite');const d=new DatabaseSync('$DATA/hal.sqlite',{readOnly:true});const r=d.prepare('SELECT completed_at FROM bootstrap_state WHERE id=1').get();process.exit(r&&r.completed_at?0:1)" 2>/dev/null; then + echo "이미 부트스트랩이 완료된 HAL 데이터($DATA)가 있습니다 — 이후 설치·갱신은 HAL 작업(운영자 콘솔 › 배포 릴리스)으로만 합니다" >&2; exit 2 +fi +echo "== 2/6 디렉터리·보관소·접근 경로 준비" +sudo -n mkdir -p "$APP_DIR/releases/$VERSION" "$DATA" "$ACME" /srv/ariweb /var/www/ariweb-sites; sudo -n chown -R "$USER_NAME:$USER_NAME" "$APP_DIR" /srv/ariweb /var/www/ariweb-sites; chmod 700 "$DATA" +tar -xzf "$ART" -C "$APP_DIR/releases/$VERSION"; ln -sfn "$APP_DIR/releases/$VERSION" "$APP_DIR/current" +cd "$APP_DIR/current" +export ARIWEB_HAL_DATA="$DATA"; $NODE_BIN -e "import('./lib/vault.mjs').then(m=>{m.openVault('$DATA');console.log(' 보관소 준비(vault.key 0600)')})" +$NODE_BIN hal/index.mjs bootstrap-release --version "$VERSION" --commit "$COMMIT" --sha256 "$SUM" +$NODE_BIN scripts/release.mjs register --hal-data "$DATA" --dist "$DIST" --approve --by bootstrap +$NODE_BIN -e "const fs=require('fs');const f='$DATA/config.json';const v=fs.existsSync(f)?JSON.parse(fs.readFileSync(f,'utf8')).values:{};v.publicUrl='https://$HOST';v.portalUrl=v.portalUrl||'http://127.0.0.1:8910';fs.writeFileSync(f,JSON.stringify({version:1,updatedAt:new Date().toISOString(),updatedBy:'bootstrap',values:v},null,1),{mode:0o600})" +echo "== 3/6 systemd 유닛 ariweb-hal" +sed -e "s#{{APP_DIR}}#$APP_DIR#g" -e "s#{{USER}}#$USER_NAME#g" -e "s#{{NODE}}#$NODE_BIN#g" -e "s#{{VERSION}}#$VERSION#g" deploy/units/ariweb-hal.service.tmpl | sudo -n tee /etc/systemd/system/ariweb-hal.service >/dev/null +sudo -n install -m 440 deploy/sudoers/ariweb-agent /etc/sudoers.d/ariweb-agent; sudo -n visudo -c >/dev/null +sudo -n systemctl daemon-reload; sudo -n systemctl enable --now ariweb-hal >/dev/null; sleep 2; systemctl is-active --quiet ariweb-hal || { echo "ariweb-hal 기동 실패 — journalctl -u ariweb-hal" >&2; exit 1; } +echo "== 4/6 nginx $HOST (인증서 $CERT)" +[ -f "$CERT/fullchain.pem" ] || { echo "인증서 $CERT/fullchain.pem 이 없습니다" >&2; exit 1; } +sed -e "s#{{HAL_HOST}}#$HOST#g" -e "s#{{CERT_DIR}}#$CERT#g" -e "s#{{PORT}}#8900#g" -e "s#{{ACME_ROOT}}#$ACME#g" deploy/nginx/ariweb-hal.conf.tmpl | sudo -n tee /etc/nginx/sites-available/ariweb-hal.conf >/dev/null +sudo -n ln -sf /etc/nginx/sites-available/ariweb-hal.conf /etc/nginx/sites-enabled/ariweb-hal.conf; sudo -n cp deploy/nginx/awstats.conf /etc/nginx/conf.d/awstats.conf; sudo -n nginx -t && sudo -n systemctl reload nginx +echo "== 5/6 HAL 자체 건강 확인" +curl -fsS http://127.0.0.1:8900/api/health | $NODE_BIN -e "let s='';process.stdin.on('data',c=>s+=c).on('end',()=>{const h=JSON.parse(s);console.log(' HAL',h.version,'db',h.db,'jobs',h.jobs,'bootstrap',JSON.stringify(h.bootstrap))})" +curl -fsS --resolve "$HOST:443:127.0.0.1" "https://$HOST/api/health" >/dev/null && echo " https://$HOST/api/health OK (로컬 resolve)" || echo " 경고: https://$HOST 확인 실패 — DNS A 레코드($HOST → 이 서버)와 인증서를 확인하세요" +echo "== 6/6 최초 운영자 설정 토큰과 첫 Agent" +$NODE_BIN hal/index.mjs setup-token +cat <)이 있어야 열립니다. + 2) 운영자 콘솔 › 서버에서 이 서버(역할 control/web)를 만들고 등록 토큰을 발급해 첫 Node Agent 를 설치합니다: + curl -fsSL https://$HOST/agent/v1/install.sh | bash -s -- --hal https://$HOST --token <토큰> --role control + 첫 Agent 가 승인되면 부트스트랩이 닫히고 수동 설치가 잠깁니다. 이후 포털·Webmaker·AriMail 은 HAL 작업으로 배포합니다. + 3) 포털 연동 키: node hal/index.mjs link-portal (출력 값을 포털 보관소에 넣습니다) +MSG diff --git a/deploy/hal-restore.sh b/deploy/hal-restore.sh new file mode 100755 index 0000000..7154e22 --- /dev/null +++ b/deploy/hal-restore.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# deploy/hal-restore.sh — 플랫폼 재해복구(§34): HAL 복구 사본(dr snapshot tar.gz)으로 HAL 데이터를 복원한다. 운영자 콘솔 › 플랫폼 복구에서 "복원 시작" 승인과 기존 제어 위치 중지 확인 뒤에만 실행한다. +# sudo bash deploy/hal-restore.sh [--data /srv/ariweb/hal/data] [--vault-key ] +# vault.key 는 사본에 들어 있지 않으므로 운영자가 보관한 키를 --vault-key 또는 ARIWEB_VAULT_KEY 로 준다(없으면 기존 데이터 디렉터리의 키를 유지). +set -euo pipefail +SNAP="${1:?snapshot}"; SUM="${2:?sha256}"; shift 2; DATA="/srv/ariweb/hal/data"; VKEY="${ARIWEB_VAULT_KEY:-}" +while [ $# -gt 0 ]; do case "$1" in --data) DATA="$2"; shift 2;; --vault-key) VKEY="$2"; shift 2;; *) shift;; esac; done +[ "$(sha256sum "$SNAP" | cut -d' ' -f1)" = "$SUM" ] || { echo "복구 사본 sha256 불일치" >&2; exit 1; } +systemctl is-active --quiet ariweb-hal && { echo "ariweb-hal 이 실행 중입니다 — 기존 제어 위치를 먼저 중지하세요(systemctl stop ariweb-hal)" >&2; exit 2; } +STAMP="$(date +%Y%m%d-%H%M%S)"; mkdir -p "$DATA"; [ -f "$DATA/hal.sqlite" ] && cp -a "$DATA" "$DATA.pre-restore-$STAMP" && echo "기존 데이터를 $DATA.pre-restore-$STAMP 로 보존" +WORK="$(mktemp -d)"; tar -xzf "$SNAP" -C "$WORK"; [ -f "$WORK/hal.sqlite" ] || { echo "사본에 hal.sqlite 가 없습니다" >&2; exit 1; } +install -m 600 "$WORK/hal.sqlite" "$DATA/hal.sqlite"; rm -f "$DATA/hal.sqlite-wal" "$DATA/hal.sqlite-shm"; [ -f "$WORK/vault.json" ] && install -m 600 "$WORK/vault.json" "$DATA/vault.json"; [ -f "$WORK/config.json" ] && install -m 600 "$WORK/config.json" "$DATA/config.json" +[ -n "$VKEY" ] && printf '%s' "$VKEY" > "$DATA/vault.key" && chmod 600 "$DATA/vault.key"; [ -f "$DATA/vault.key" ] || echo "경고: vault.key 가 없습니다 — 보관소 값을 읽을 수 없으니 --vault-key 로 넣으세요" >&2 +OWNER="$(stat -c %U "$(dirname "$DATA")")"; chown -R "$OWNER:$OWNER" "$DATA"; rm -rf "$WORK" +echo "복원 완료($SNAP). 새 HAL 기동: systemctl start ariweb-hal (다른 인스턴스의 임대가 남아 있으면 기동이 거부됩니다 → 그 HAL 이 정말 중지되었는지 확인한 뒤 node hal/index.mjs --takeover 로 1회 기동)" +echo "기동 뒤 운영자 콘솔 › 플랫폼 복구 › 복원 완료 확인(운영자 로그인·HAL 제어·HTTPS·고객 서비스 외부 확인)을 기록하세요" diff --git a/deploy/nginx/ariweb-hal.conf.tmpl b/deploy/nginx/ariweb-hal.conf.tmpl new file mode 100644 index 0000000..a89e393 --- /dev/null +++ b/deploy/nginx/ariweb-hal.conf.tmpl @@ -0,0 +1,31 @@ +# hal.ariweb.net — HAL(제어 평면). 운영자 콘솔 /ops 는 HAL 이 네트워크 관문(허용 CIDR 또는 접근권 쿠키)을 적용한다. +# Agent 는 /agent/v1 로, 포털은 /portal/v1 로 서명된 요청을 보낸다. 큰 전송(/agent/v1/transfers)은 본문 제한을 넉넉히 둔다. +server { + listen 80; + listen [::]:80; + server_name {{HAL_HOST}}; + location ^~ /.well-known/acme-challenge/ { root {{ACME_ROOT}}; default_type text/plain; } + location / { return 301 https://$host$request_uri; } +} +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {{HAL_HOST}}; + server_tokens off; + client_max_body_size 0; + ssl_certificate {{CERT_DIR}}/fullchain.pem; + ssl_certificate_key {{CERT_DIR}}/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + add_header Strict-Transport-Security "max-age=31536000" always; + add_header X-Content-Type-Options "nosniff" always; + location / { + proxy_pass http://127.0.0.1:{{PORT}}; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto https; + proxy_request_buffering off; + proxy_buffering off; + proxy_read_timeout 120s; + } +} diff --git a/deploy/nginx/ariweb-portal.conf.tmpl b/deploy/nginx/ariweb-portal.conf.tmpl new file mode 100644 index 0000000..54676ec --- /dev/null +++ b/deploy/nginx/ariweb-portal.conf.tmpl @@ -0,0 +1,29 @@ +# my.ariweb.net — 고객 포털 +server { + listen 80; + listen [::]:80; + server_name {{PORTAL_HOST}}; + location ^~ /.well-known/acme-challenge/ { root {{ACME_ROOT}}; default_type text/plain; } + location / { return 301 https://$host$request_uri; } +} +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {{PORTAL_HOST}}; + server_tokens off; + client_max_body_size 200m; + ssl_certificate {{CERT_DIR}}/fullchain.pem; + ssl_certificate_key {{CERT_DIR}}/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + add_header Strict-Transport-Security "max-age=31536000" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + location / { + proxy_pass http://127.0.0.1:{{PORT}}; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto https; + proxy_read_timeout 300s; + } +} diff --git a/deploy/nginx/awstats.conf b/deploy/nginx/awstats.conf new file mode 100644 index 0000000..c7bebc6 --- /dev/null +++ b/deploy/nginx/awstats.conf @@ -0,0 +1,3 @@ +# deploy/nginx/awstats.conf — /etc/nginx/conf.d/awstats.conf : Ariweb 관리형 사이트 접속 통계 로그 형식(§13.1). +# Agent(site.usage)가 사이트별 로그(aw-.access.log)를 읽어 요청 수·수신/송신 바이트·오류율·응답 시간을 집계한다. IP 는 기록하지 않는다(개인정보 최소화). +log_format awstats '$time_iso8601|$host|$request_length|$bytes_sent|$status|$request_time|$uri'; diff --git a/deploy/nginx/redirect.tmpl b/deploy/nginx/redirect.tmpl new file mode 100644 index 0000000..9c30080 --- /dev/null +++ b/deploy/nginx/redirect.tmpl @@ -0,0 +1,7 @@ +server { + listen 80; + listen [::]:80; + server_name {{OLD_NAMES}}; + location ^~ /.well-known/acme-challenge/ { root {{ACME_ROOT}}; default_type text/plain; } + return 301 https://{{PRIMARY}}$request_uri; +} diff --git a/deploy/nginx/site-body-node.tmpl b/deploy/nginx/site-body-node.tmpl new file mode 100644 index 0000000..15985b8 --- /dev/null +++ b/deploy/nginx/site-body-node.tmpl @@ -0,0 +1,7 @@ + access_log {{LOG}} awstats; + error_page 503 /__aw_maint; + location = /__aw_maint { internal; default_type text/html; add_header Cache-Control "no-store" always; return 503 '{{MAINT_HTML}}'; } + if (-f {{SITE_DIR}}/.maintenance) { return 503; } + if (-f {{SITE_DIR}}/.limited) { rewrite ^ /__aw_limited last; } + location = /__aw_limited { internal; default_type text/html; add_header Cache-Control "no-store" always; return 503 '{{LIMIT_HTML}}'; } + location / { proxy_pass http://127.0.0.1:{{PORT}}; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 300s; } diff --git a/deploy/nginx/site-body-static.tmpl b/deploy/nginx/site-body-static.tmpl new file mode 100644 index 0000000..fa6dd98 --- /dev/null +++ b/deploy/nginx/site-body-static.tmpl @@ -0,0 +1,11 @@ + root {{ROOT}}; + index index.html; + access_log {{LOG}} awstats; + error_page 503 /__aw_maint; + location = /__aw_maint { internal; default_type text/html; add_header Cache-Control "no-store" always; return 503 '{{MAINT_HTML}}'; } + if (-f {{SITE_DIR}}/.maintenance) { return 503; } + if (-f {{SITE_DIR}}/.limited) { rewrite ^ /__aw_limited last; } + location = /__aw_limited { internal; default_type text/html; add_header Cache-Control "no-store" always; return 503 '{{LIMIT_HTML}}'; } + location ~ /\. { return 404; } + location / { try_files $uri $uri/ /index.html =404; add_header Cache-Control "no-cache" always; } + location /uploads/ { alias {{SITE_DIR}}/data/uploads/; } diff --git a/deploy/nginx/site-https.tmpl b/deploy/nginx/site-https.tmpl new file mode 100644 index 0000000..a991ae4 --- /dev/null +++ b/deploy/nginx/site-https.tmpl @@ -0,0 +1,12 @@ +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {{SERVER_NAMES}}; + server_tokens off; + client_max_body_size 64m; + ssl_certificate {{CERT_DIR}}/fullchain.pem; + ssl_certificate_key {{CERT_DIR}}/privkey.pem; + add_header Strict-Transport-Security "max-age=31536000" always; + add_header X-Content-Type-Options "nosniff" always; +{{BODY}} +} diff --git a/deploy/nginx/site.conf.tmpl b/deploy/nginx/site.conf.tmpl new file mode 100644 index 0000000..4b10147 --- /dev/null +++ b/deploy/nginx/site.conf.tmpl @@ -0,0 +1,11 @@ +# Ariweb 관리형 사이트 vhost — Agent 가 만든다(site.deploy/domain.vhost). 손으로 고치지 마세요(다음 배포 때 덮어씁니다). +# 사이트 {{SITE_ID}} · 도메인 {{DOMAIN_LIST}} · 생성 {{GENERATED_AT}} +server { + listen 80; + listen [::]:80; + server_name {{SERVER_NAMES}}; + server_tokens off; + location ^~ /.well-known/acme-challenge/ { root {{ACME_ROOT}}; default_type text/plain; } +{{HTTP_BODY}} +} +{{HTTPS_SERVER}} diff --git a/deploy/sudoers/ariweb-agent b/deploy/sudoers/ariweb-agent new file mode 100644 index 0000000..e9b97c9 --- /dev/null +++ b/deploy/sudoers/ariweb-agent @@ -0,0 +1,9 @@ +# /etc/sudoers.d/ariweb-agent — Node Agent(계정 master)가 비밀번호 없이 쓸 수 있는 최소 명령(§28·§32). 그 밖의 sudo 는 허용하지 않는다. +# 설치: sudo install -m 440 deploy/sudoers/ariweb-agent /etc/sudoers.d/ariweb-agent && sudo visudo -c +Defaults:master !requiretty +master ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload, /usr/bin/systemctl start *, /usr/bin/systemctl stop *, /usr/bin/systemctl restart *, /usr/bin/systemctl reload *, /usr/bin/systemctl enable *, /usr/bin/systemctl disable *, /usr/bin/systemctl is-active * +master ALL=(root) NOPASSWD: /usr/sbin/nginx -t, /usr/bin/certbot *, /snap/bin/certbot * +master ALL=(root) NOPASSWD: /usr/bin/tee /etc/nginx/sites-available/aw-*, /usr/bin/tee /etc/nginx/sites-enabled/aw-*, /usr/bin/tee /etc/nginx/conf.d/awstats.conf, /usr/bin/tee /etc/nginx/sites-available/ariweb-*, /usr/bin/tee /etc/systemd/system/ariweb-*, /usr/bin/tee /etc/systemd/system/webmaker.service, /usr/bin/tee /etc/systemd/system/webmaker-setup.service +master ALL=(root) NOPASSWD: /usr/bin/ln -sf /etc/nginx/sites-available/aw-* /etc/nginx/sites-enabled/*, /usr/bin/ln -sf /etc/nginx/sites-available/ariweb-* /etc/nginx/sites-enabled/*, /usr/bin/rm -f /etc/nginx/sites-enabled/aw-*, /usr/bin/rm -f /etc/nginx/sites-available/aw-*, /usr/bin/rm -f /etc/systemd/system/ariweb-site-* +master ALL=(root) NOPASSWD: /usr/bin/mkdir -p /etc/nginx/*, /usr/bin/mkdir -p /var/www/*, /usr/bin/chown master\:master /var/www/*, /usr/bin/chmod * /etc/nginx/sites-available/aw-*, /usr/bin/chmod * /etc/systemd/system/ariweb-*, /usr/bin/chmod * /etc/nginx/conf.d/awstats.conf +master ALL=(root) NOPASSWD: /usr/bin/true diff --git a/deploy/units/arimail.service.tmpl b/deploy/units/arimail.service.tmpl new file mode 100644 index 0000000..c178e2f --- /dev/null +++ b/deploy/units/arimail.service.tmpl @@ -0,0 +1,31 @@ +# AriMail(도입 모듈) — 원본 deploy/systemd/mail-gateway.service.in 과 같은 내용. 서버에 이미 있는 유닛(arihakdang-mail/arimail/heart-mail)은 그대로 두고 current 심링크만 바꾸므로 보통 쓰이지 않는다. +[Unit] +Description={{UNIT}} — AriMail gateway ({{VERSION}}) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{USER}} +Group={{USER}} +WorkingDirectory={{APP_DIR}}/current +EnvironmentFile={{ENV_FILE}} +Environment=NODE_ENV=production +Environment=HOST=127.0.0.1 +Environment=PORT=8790 +ExecStart={{NODE}} server/index.mjs +Restart=always +RestartSec=3 +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ProtectHome=yes +ReadWritePaths={{APP_DIR}}/data +StandardOutput=journal +StandardError=journal +SyslogIdentifier={{UNIT}} + +[Install] +WantedBy=multi-user.target diff --git a/deploy/units/ariweb-agent.service.tmpl b/deploy/units/ariweb-agent.service.tmpl new file mode 100644 index 0000000..e98868a --- /dev/null +++ b/deploy/units/ariweb-agent.service.tmpl @@ -0,0 +1,24 @@ +[Unit] +Description=Ariweb Node Agent ({{VERSION}}) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{USER}} +Group={{USER}} +WorkingDirectory={{APP_DIR}}/current +Environment=NODE_ENV=production +Environment=ARIWEB_AGENT_DATA={{APP_DIR}}/data +Environment=NODE_OPTIONS=--no-warnings +Environment=PATH=/usr/local/bin:/usr/bin:/bin:/snap/bin +{{ENV_LINES}} +ExecStart={{NODE}} agent/index.mjs run +Restart=always +RestartSec=5 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=ariweb-agent + +[Install] +WantedBy=multi-user.target diff --git a/deploy/units/ariweb-hal.service.tmpl b/deploy/units/ariweb-hal.service.tmpl new file mode 100644 index 0000000..fe7d6f1 --- /dev/null +++ b/deploy/units/ariweb-hal.service.tmpl @@ -0,0 +1,29 @@ +[Unit] +Description=Ariweb HAL — 제어 평면 ({{VERSION}}) +Documentation=https://source.ibit.kr/OZ-Group/AriWeb-Suite +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{USER}} +Group={{USER}} +WorkingDirectory={{APP_DIR}}/current +Environment=NODE_ENV=production +Environment=HOST=127.0.0.1 +Environment=PORT=8900 +Environment=ARIWEB_HAL_DATA={{APP_DIR}}/data +Environment=NODE_OPTIONS=--no-warnings +ExecStart={{NODE}} hal/index.mjs +Restart=always +RestartSec=3 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ReadWritePaths={{APP_DIR}} +StandardOutput=journal +StandardError=journal +SyslogIdentifier=ariweb-hal + +[Install] +WantedBy=multi-user.target diff --git a/deploy/units/ariweb-portal.service.tmpl b/deploy/units/ariweb-portal.service.tmpl new file mode 100644 index 0000000..8d24db1 --- /dev/null +++ b/deploy/units/ariweb-portal.service.tmpl @@ -0,0 +1,27 @@ +[Unit] +Description=Ariweb Portal — 고객 포털 ({{VERSION}}) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{USER}} +Group={{USER}} +WorkingDirectory={{APP_DIR}}/current +Environment=NODE_ENV=production +Environment=HOST=127.0.0.1 +Environment=PORT=8910 +Environment=ARIWEB_PORTAL_DATA={{APP_DIR}}/data +Environment=NODE_OPTIONS=--no-warnings +{{ENV_LINES}} +ExecStart={{NODE}} portal/index.mjs +Restart=always +RestartSec=3 +NoNewPrivileges=yes +PrivateTmp=yes +StandardOutput=journal +StandardError=journal +SyslogIdentifier=ariweb-portal + +[Install] +WantedBy=multi-user.target diff --git a/deploy/units/ariweb-site.service.tmpl b/deploy/units/ariweb-site.service.tmpl new file mode 100644 index 0000000..27771db --- /dev/null +++ b/deploy/units/ariweb-site.service.tmpl @@ -0,0 +1,25 @@ +# 관리형 동적(Node) 사이트 실행 유닛 — Agent 가 site.deploy(runtime=node) 때 만든다. 사이트 번들의 ariweb.json { "start": "node server.js", "port": N } 을 따른다. +[Unit] +Description=Ariweb site {{SITE_ID}} +After=network-online.target + +[Service] +Type=simple +User={{USER}} +WorkingDirectory={{SITE_DIR}}/current +Environment=NODE_ENV=production +Environment=PORT={{PORT}} +Environment=ARIWEB_SITE_DATA={{SITE_DIR}}/data +ExecStart={{NODE}} {{START}} +Restart=always +RestartSec=3 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ReadWritePaths={{SITE_DIR}}/data +StandardOutput=journal +StandardError=journal +SyslogIdentifier=ariweb-site-{{SITE_ID}} + +[Install] +WantedBy=multi-user.target diff --git a/deploy/units/webmaker.service.tmpl b/deploy/units/webmaker.service.tmpl new file mode 100644 index 0000000..4234bb2 --- /dev/null +++ b/deploy/units/webmaker.service.tmpl @@ -0,0 +1,25 @@ +# Webmaker(도입 모듈 WebHosting_AI_agent) — 원본 deploy/webhosting-agent.service 를 바탕으로 HAL 배치 레이아웃(releases/current)에 맞춘 유닛. +# 데이터·포트·경로 환경 변수(DATA_ROOT·DATA_DIR·SITES_ROOT·STATS_LOG·PUBLIC_BASE_URL 등)는 서버에 있던 기존 유닛의 Environment 줄({{ENV_LINES}})을 그대로 보존한다. +[Unit] +Description=AI Webmaker ({{VERSION}}, Ariweb Suite managed) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{USER}} +Group={{USER}} +WorkingDirectory={{APP_DIR}}/current +Environment=NODE_ENV=production +Environment=HOST=127.0.0.1 +Environment=PORT=3080 +{{ENV_LINES}} +ExecStart={{NODE}} server.js +Restart=always +RestartSec=3 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=webmaker + +[Install] +WantedBy=multi-user.target diff --git a/docs/HAL-AGENT.md b/docs/HAL-AGENT.md new file mode 100644 index 0000000..f916730 --- /dev/null +++ b/docs/HAL-AGENT.md @@ -0,0 +1,69 @@ +# HAL ↔ Node Agent 프로토콜·설치·운영 (v0.1.1) + +Dev-Order §5(아키텍처)·§28·§29·§32·§33·§34 의 백엔드 구현. 코드: `hal/`, `agent/`, `deploy/`, `scripts/release.mjs`, `portal/hal/client.mjs`. + +## 1. 구성 + +| 구성요소 | 위치 | 역할 | +|---|---|---| +| HAL | `/srv/ariweb/hal/current`, 127.0.0.1:8900, nginx `hal.ariweb.net` | 서버·Agent 등록부, 릴리스 등록부, 내구성 작업 엔진(lib/jobs), 모듈 배포, 포털 API, 운영자 API, 긴급 복구, 재해복구, 공개 상태 | +| Node Agent | `/srv/ariweb/agent/current`, 유닛 `ariweb-agent` | heartbeat(30초, 지표·모듈 버전·시계 차이), 작업 long-poll·실행(단계 저장·재개), 모듈 설치, 사이트 배포·백업·이전 | +| External Pull Agent | 고객 외부 서버, 같은 코드 `--external --site ` | `site.deploy / site.remove / site.check / site.rollback` 만, 사이트 하나에 귀속 | + +## 2. 인증 + +- Agent·포털·HAL 간 모든 요청은 `lib/serviceauth` HMAC: `X-Service-Key-Id`(`agent:` / `portal` / `hal`), `X-Service-Ts`(±5분), `X-Service-Sign`. +- Agent 비밀은 등록 응답에서 한 번만 전달되어 Agent 보관소(`data/vault.json`, 키 `agent-secret`)에, HAL 쪽은 `service:agent:` 로 보관. 표에는 해시만. +- 큰 전송(`POST /agent/v1/transfers/:id`)은 빈 본문 서명 + `X-Transfer-Sha256` 헤더, HAL 이 받은 뒤 해시를 다시 계산해 비교. + +## 3. Agent 프로토콜(`/agent/v1`) + +| 경로 | 설명 | +|---|---| +| `GET install.sh` | 설치 스크립트(deploy/agent-install.sh) | +| `GET bootstrap-artifact?module=agent&version=` + `X-Registration-Token` | 등록 전 Agent 코드 내려받기(유효 토큰 확인, 소비하지 않음) | +| `POST register {token, facts, version}` | 일회용 토큰(10분) → `{agentId, secret(1회), serverId, kind, siteId}` | +| `POST heartbeat {version, metrics, modules, clockSkewMs, facts, sitesHosted}` | 지표 저장(90일+), 서버 last_seen·용량·모듈 실제 버전 갱신. 90초 무응답 → offline | +| `GET jobs/next?wait=25` | long-poll. `{job:{id, kind, payload, halJobId}}` 또는 null | +| `GET jobs/unfinished` | 재시작 뒤 이어서 할 running 작업 목록 | +| `POST jobs/:id/progress {progress, steps, log}` → `{cancelRequested}` / `POST jobs/:id/result {status: done\|failed\|needs_user, result, error, errorCode, userMessage}` | | +| `GET releases/:version/artifact?module=` | tar.gz 스트리밍, `X-Sha256` | +| `POST/GET transfers/:id` | HAL 경유 파일 전송(이전·원격 백업), 24시간 보관 | + +## 4. 포털 API(`/portal/v1`, 키 `portal`) — docs/HAL-JOBS.md + +부트스트랩이 끝나기 전에는 `health` 외 503 `bootstrap_incomplete`. 작업 상태 변화는 outbox 를 통해 포털 `POST /internal/hal-events` 로 밀린다(실패 시 지수 백오프, 7일 보관). + +## 5. 작업 모델 + +- HAL 작업(lib/jobs) 한 개가 Agent 작업(agent_jobs) 하나 이상을 배정하고 결과를 기다린다(`core.awaitAgentJob`: 진행률 반영, 취소 전달, Agent 10분 무응답 → `사용자 확인 필요`). +- 복합 작업: `backup.create`(생성 → 저장 서버로 복사 → 검증), `backup.restore`(가져오기 → 보호 시점 → 격리 확인 → 복원 → 외부 확인, 되돌리기 지원), `site.relocate`(precheck → prepare → precopy → verify_copy → final_sync → verify_new → switch → external_check → cleanup; `verify_new` 실패 시 전환하지 않음, 되돌리기 지원). +- 운영자 작업: `module.deploy`/`module.rollback`/`module.health`/`module.reconcile` 은 서버마다 자식 작업(`module.deploy.target`)을 만들고 `childrenSummary` 로 대상별 성공·실패·대기를 보인다. 일부 실패는 `partial_failure` 로 끝난다. +- 배포 순서(§32.2): 릴리스 승인 → HAL 자체 건강 → 부트스트랩 완료 → `hal` 모듈이 그 릴리스로 실행 중일 때만 다른 모듈 배포(`hal_first`). + +## 6. 설치(부트스트랩)와 첫 Agent + +``` +# 작업 PC: main 에 병합된 커밋에서 +node scripts/release.mjs build # dist/0.1.1/*.tar.gz + manifest.json (테스트 통과 필요) +scp -r dist/0.1.1 hostinger:/tmp/ariweb-dist +# 대상 서버(ariweb.net 호스트, master, sudo): +bash /tmp/ariweb-dist/… (artifact 안의 deploy/bootstrap-hal.sh) --dist /tmp/ariweb-dist --host hal.ariweb.net --cert /etc/letsencrypt/live/wh-wild-ariweb.net +``` +부트스트랩은 HAL 설치·보관소·유닛·nginx·릴리스 등록(승인)·운영자 설정 토큰까지만 한다. 이어서 운영자 계정 생성 → 서버 생성·등록 토큰 → `agent-install.sh` → 서버 승인. 첫 Agent 승인 시 `bootstrap_state.completed_at` 이 기록되고 수동 설치가 잠긴다. 그 뒤 `portal`·`webmaker`·`arimail` 은 운영자 API `POST /ops/api/modules//deploy {version, serverIds}` 로 배포한다. + +도입 모듈 배치: Agent facts 가 서버의 기존 유닛(`webmaker`, `arihakdang-mail`/`arimail`/`heart-mail`)과 WorkingDirectory 를 읽어 그 appDir(`/srv/webmaker`, `/srv/arihakdang-mail` …)의 `releases/` 에 풀고 `current` 를 바꾼다. Webmaker 는 원본 `remote-update.sh` 순서(데이터 백업 → 교체 → 재시작 → unit-check), AriMail 은 원본 `deploy.sh` 순서(DB 백업 → 교체 → db-migrate → 재시작 → health → haraka 재시작)를 Agent 가 수행한다. + +## 7. 운영자 접근(§25) + +`/ops/*` 는 허용 CIDR(`config.opsAllowCidrs`) 안이거나 접근권 쿠키(`aw_ops_grant`, 사용자·기기 귀속, 최대 72시간)가 있어야 하며 아니면 로그인 화면도 403. 접근권: `node hal/index.mjs grant --login --hours 8` 또는 허용 네트워크 안의 운영자가 `POST /ops/api/access-grants`. 고위험 작업은 재인증(10분)과 3단계 확인(`confirmImpact`, `confirmIdentifiers`, `confirmExecute`). + +## 8. 긴급 복구(§33)·재해복구(§34) + +- `POST /ops/api/emergency` 선언(HAL 정상이고 열린 critical 사건이 없으면 거부) → `approve` 1·2·3단계(서로 다른 입력) → `credential`(최대 60분, 1회 표시) → `perform`(허용 범위·허용 행동만; 일반 배포 거부) → `verify`(자격 폐기) → `reconcile`(목표·실제 비교) → `close`. +- DR: `PUT /ops/api/dr/config`(RTO·RPO·주기·보존·외부 보관), 사본 `POST /ops/api/dr/snapshots`(VACUUM INTO + vault.json + config.json, vault.key 제외), 복원 요청→승인→시작(기존 제어 중지 확인 문구)→`deploy/hal-restore.sh`→완료 확인(운영자 로그인·HAL 제어·HTTPS·고객 서비스), 훈련 기록과 측정 RTO/RPO. `readiness.ready` 가 true 여야 정식 운영 준비 완료. +- 하나의 HAL: 기동 때 `bootstrap_state.live_instance_id/live_lease_at` 임대(20초 갱신, 60초 유효). 다른 인스턴스의 임대가 살아 있으면 기동 거부(`--takeover` 는 복원 절차에서만). + +## 9. 테스트 + +`node --no-warnings test/hal.test.mjs`, `node --no-warnings test/agent.test.mjs`. Agent 테스트는 `ARIWEB_AGENT_STUB_CMDS=1` 과 임시 루트로 실제 시스템을 건드리지 않는다. diff --git a/hal/app.mjs b/hal/app.mjs new file mode 100644 index 0000000..a1b5aa3 --- /dev/null +++ b/hal/app.mjs @@ -0,0 +1,186 @@ +/** + * hal/app.mjs — HAL HTTP 라우트. + * + * /api/health 공개 자체 건강(버전·DB·작업 엔진·부트스트랩 상태). 비밀 없음. + * /status, /api/status 공개 상태 페이지 데이터(§31.3): 구성요소 상태·계획 유지보수·공개 사건. 내부 서버 이름·개인정보 없음. + * /agent/v1/* Node Agent 프로토콜(§5). install.sh·register 외에는 Agent HMAC 인증. + * /portal/v1/* 포털 전용 API(docs/HAL-JOBS.md). 서비스 키 'portal'. 부트스트랩 미완료면 503. + * /ops/api/* 운영자 API(§25): 네트워크 관문 → 운영자 세션. 서버·릴리스·모듈 배포·작업·사건·긴급 복구·재해복구·계정. 화면(UI)은 별도 영역(hal/ui, 후속)에서 같은 API 를 쓴다. + */ +import { createApp, HttpError, cookie, esc } from '../lib/http.mjs'; +import { verifyCtx } from '../lib/serviceauth.mjs'; +import * as session from '../lib/session.mjs'; +import { describe, JobError } from '../lib/jobs.mjs'; +import { listAudit } from '../lib/audit.mjs'; +import { listNotifications } from '../lib/notify.mjs'; +import { redact } from '../lib/redact.mjs'; +import { nowIso, toJson, parseJson } from '../lib/db.mjs'; +import { newId, sha256 } from '../lib/crypto.mjs'; +import { createHash } from 'node:crypto'; +import { createReadStream, createWriteStream, statSync, readFileSync, existsSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as core from './core.mjs'; +import * as ops from './ops.mjs'; +import * as emergency from './emergency.mjs'; +import * as dr from './dr.mjs'; +import { MODULES, DEPLOY_ORDER } from './modules/index.mjs'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const bad = (e) => { if (e instanceof HttpError) throw e; if (e instanceof JobError || (e && e.code)) throw new HttpError(e.status || (e.code === 'quota_insufficient' ? 409 : 400), e.code, e.message, e.detail); throw e; }; +const wrap = (fn) => async (ctx) => { try { return await fn(ctx); } catch (e) { bad(e); } }; + +export function buildApp(d) { + const app = createApp({ log: d.log }); + const uiDir = join(ROOT, 'hal', 'ui', 'public'); if (existsSync(uiDir)) app.static('/ops/ui', uiDir); + + // ---------------- 공개 + app.get('/api/health', () => core.selfHealth(d)); + app.get('/api/status', () => publicStatus(d)); + app.get('/status', (ctx) => ctx.html(statusPage(d))); + app.get('/agent/v1/install.sh', (ctx) => { const f = join(ROOT, 'deploy', 'agent-install.sh'); return ctx.text(existsSync(f) ? readFileSync(f, 'utf8') : 'echo "install script missing"; exit 1'); }); + + // ---------------- Agent 프로토콜 + app.post('/agent/v1/register', wrap(async (ctx) => { const b = await ctx.body(); const r = core.registerAgent(d, { token: b.token, facts: b.facts || {}, version: b.version || '' }); return { ok: true, ...r, halUrl: d.config.publicUrl, serverTime: nowIso() }; })); + /** Agent 인증 미들웨어: /agent/v1/* (register·install.sh 제외) */ + const agentAuth = async (ctx) => { + const streaming = ctx.method === 'POST' && /^\/agent\/v1\/transfers\//.test(ctx.path); + const r = streaming ? verifyStreaming(ctx) : await verifyCtx(ctx, (keyId) => keyId.startsWith('agent:') ? d.vault.get(`service:${keyId}`) : null); + if (!r.ok) throw new HttpError(401, 'agent_auth_failed', `Agent 인증 실패(${r.reason})`); + const agent = core.getAgent(d, r.keyId.slice(6)); if (!agent || agent.status !== 'active') throw new HttpError(401, 'agent_revoked', 'Agent 가 해지되었습니다'); + ctx.agent = agent; ctx.outHeaders['X-HAL-Time'] = String(Date.now()); + }; + function verifyStreaming(ctx) { const { verifyRequest } = d.serviceauth; return verifyRequest({ method: ctx.method, path: ctx.path + (ctx.url.search || ''), headers: ctx.headers, rawBody: '' }, (keyId) => keyId.startsWith('agent:') ? d.vault.get(`service:${keyId}`) : null); } + /** 등록 전 Agent 코드 내려받기: 아직 서명을 만들 수 없으므로 유효한(미사용·미만료) 등록 토큰을 헤더로 확인한다. 토큰은 소비하지 않는다(등록 때 1회 소비). */ + app.get('/agent/v1/bootstrap-artifact', wrap((ctx) => { const tok = String(ctx.headers['x-registration-token'] || ''); const t = d.store.get('SELECT * FROM registration_tokens WHERE token_hash=?', sha256(tok)); if (!t || t.used_at || Date.parse(t.expires_at) < Date.now()) throw new HttpError(401, 'bad_token', '등록 토큰이 올바르지 않거나 만료되었습니다'); const version = ctx.query.version || core.bootstrapState(d).release_version; const a = core.artifactPath(d, version, ctx.query.module || 'agent'); if (!a) throw new HttpError(404, 'artifact_not_found', 'artifact 가 없습니다'); ctx.ended = true; ctx.res.writeHead(200, { 'Content-Type': 'application/gzip', 'Content-Length': a.size, 'X-Sha256': a.sha256, 'Cache-Control': 'no-store' }); createReadStream(a.path).pipe(ctx.res); })); + app.use(async (ctx) => { if (ctx.path.startsWith('/agent/v1/') && !['/agent/v1/register', '/agent/v1/install.sh', '/agent/v1/bootstrap-artifact'].includes(ctx.path)) await agentAuth(ctx); }); + app.post('/agent/v1/heartbeat', wrap(async (ctx) => core.heartbeat(d, ctx.agent, await ctx.body()))); + app.get('/agent/v1/jobs/next', wrap(async (ctx) => { const wait = Math.min(25, Number(ctx.query.wait || 0)); const until = Date.now() + wait * 1000; while (true) { const j = core.nextAgentJob(d, ctx.agent.id); if (j || Date.now() >= until || d.stopping) return { ok: true, job: j || null, serverTime: nowIso() }; await new Promise((r) => setTimeout(r, 500)); } })); + app.get('/agent/v1/jobs/unfinished', wrap((ctx) => ({ ok: true, jobs: core.unfinishedAgentJobs(d, ctx.agent.id) }))); + app.post('/agent/v1/jobs/:id/progress', wrap(async (ctx) => { const b = await ctx.body(); const ok = core.agentJobProgress(d, ctx.params.id, ctx.agent.id, b); return { ok, cancelRequested: core.isAgentJobCancelled(d, ctx.params.id) }; })); + app.post('/agent/v1/jobs/:id/result', wrap(async (ctx) => { const b = await ctx.body(); const ok = core.agentJobResult(d, ctx.params.id, ctx.agent.id, b); if (!ok) throw new HttpError(409, 'job_not_running', '실행 중인 작업이 아닙니다'); return { ok }; })); + app.get('/agent/v1/releases/:version/artifact', wrap((ctx) => { const a = core.artifactPath(d, ctx.params.version, ctx.query.module); if (!a) throw new HttpError(404, 'artifact_not_found', 'artifact 가 없습니다'); ctx.ended = true; ctx.res.writeHead(200, { 'Content-Type': 'application/gzip', 'Content-Length': a.size, 'X-Sha256': a.sha256, 'Cache-Control': 'no-store' }); createReadStream(a.path).pipe(ctx.res); })); + app.post('/agent/v1/transfers/:id', wrap((ctx) => new Promise((resolve, reject) => { + const id = ctx.params.id; if (!/^tr_[A-Za-z0-9_.-]{2,80}$/.test(id)) return reject(new HttpError(400, 'bad_transfer_id', '전송 id 형식이 올바르지 않습니다')); + const expected = String(ctx.headers['x-transfer-sha256'] || ''); const path = join(core.transferDir(d), id + '.bin'); const hash = createHash('sha256'); let size = 0; const limit = 20 * 1024 ** 3; + const out = createWriteStream(path, { mode: 0o600 }); ctx.req.on('data', (c) => { size += c.length; hash.update(c); if (size > limit) ctx.req.destroy(new Error('too large')); }); ctx.req.on('error', (e) => { out.destroy(); reject(new HttpError(413, 'transfer_too_large', e.message)); }); + ctx.req.pipe(out); out.on('finish', () => { const sum = hash.digest('hex'); if (expected && expected !== sum) return reject(new HttpError(400, 'checksum_mismatch', '전송 파일 해시가 다릅니다')); core.createTransfer(d, { id, agentId: ctx.agent.id, sha256: sum, size, purpose: String(ctx.query.purpose || '') }); resolve({ ok: true, id, sha256: sum, size }); }); + }))); + app.get('/agent/v1/transfers/:id', wrap((ctx) => { const t = core.getTransfer(d, ctx.params.id); if (!t) throw new HttpError(404, 'transfer_not_found', '전송 파일이 없거나 만료되었습니다'); ctx.ended = true; ctx.res.writeHead(200, { 'Content-Type': 'application/octet-stream', 'Content-Length': statSync(t.path).size, 'X-Sha256': t.sha256, 'Cache-Control': 'no-store' }); createReadStream(t.path).pipe(ctx.res); })); + + // ---------------- 포털 API + app.use(async (ctx) => { if (!ctx.path.startsWith('/portal/v1/')) return; const r = await verifyCtx(ctx, (k) => k === 'portal' ? d.vault.get('service:portal') : null); if (!r.ok) throw new HttpError(401, 'service_auth_failed', `서비스 인증 실패(${r.reason})`); if (ctx.path !== '/portal/v1/health' && !core.bootstrapComplete(d)) throw new HttpError(503, 'bootstrap_incomplete', 'HAL 부트스트랩(HAL 설치·첫 Agent 등록)이 끝나지 않았습니다'); }); + app.get('/portal/v1/health', () => ({ ok: true, ...core.selfHealth(d) })); + app.post('/portal/v1/jobs', wrap(async (ctx) => { const b = await ctx.body(); if (!d.jobs.kinds().includes(b.kind) || b.kind.startsWith('module.')) throw new HttpError(400, 'bad_kind', `포털이 요청할 수 없는 작업 종류: ${b.kind}`); const { job, created } = d.jobs.enqueue({ kind: b.kind, subject: { target: b.target, payload: b.payload || {}, _portal: true }, idempotencyKey: b.idempotencyKey || null, createdBy: 'portal', actorType: 'system', groupId: b.groupId || null, subjectType: b.subjectType || null, subjectId: b.subjectId || null, label: b.label || null }); return { ok: true, halJobId: job.id, status: job.status, created }; })); + const jobView = (j) => { if (!j) throw new HttpError(404, 'job_not_found', '작업이 없습니다'); const dsc = describe(j); return { ...dsc, result: j.result, children: j.kind.startsWith('module.') ? d.jobs.childrenSummary(j.id) : undefined }; }; + app.get('/portal/v1/jobs/:id', wrap((ctx) => ({ ok: true, ...jobView(d.jobs.get(ctx.params.id)) }))); + for (const act of ['cancel', 'retry', 'resume', 'revert']) app.post(`/portal/v1/jobs/:id/${act}`, wrap(async (ctx) => { const j = await d.jobs[act](ctx.params.id, 'portal'); return { ok: true, ...jobView(j) }; })); + app.get('/portal/v1/servers', wrap((ctx) => ({ ok: true, servers: core.listServers(d, { role: ctx.query.role || null }).map(stripFacts) }))); + app.post('/portal/v1/servers/pick', wrap(async (ctx) => { const b = await ctx.body(); return { ok: true, pick: core.pickServer(d, { role: b.role || 'web', needBytes: Number(b.needBytes || 0) }) }; })); + app.post('/portal/v1/external-agents/token', wrap(async (ctx) => { const b = await ctx.body(); if (!b.siteId || !b.groupId) throw new HttpError(400, 'site_required', 'siteId·groupId 가 필요합니다'); const t = core.issueRegistrationToken(d, { kind: 'external', groupId: b.groupId, siteId: b.siteId, label: b.label || '', by: null }); return { ok: true, agentId: null, token: t.token, expiresAt: t.expiresAt, installCommand: t.installCommand }; })); + app.get('/portal/v1/external-agents/by-site/:siteId', wrap((ctx) => { const a = d.store.get('SELECT id FROM agents WHERE kind=\'external\' AND site_id=? AND status=\'active\' ORDER BY registered_at DESC LIMIT 1', ctx.params.siteId); return { ok: true, agent: a ? core.externalAgentStatus(d, a.id) : null }; })); + app.get('/portal/v1/external-agents/:id', wrap((ctx) => { const s = core.externalAgentStatus(d, ctx.params.id); if (!s) throw new HttpError(404, 'agent_not_found', '외부 Agent 가 없습니다'); return { ok: true, agent: s }; })); + app.post('/portal/v1/external-agents/:id/revoke', wrap((ctx) => { core.revokeAgent(d, ctx.params.id, 'portal'); return { ok: true }; })); + + // ---------------- 운영자 API(§25) + app.use(async (ctx) => { + if (!ctx.path.startsWith('/ops')) return; + const gate = ops.networkGate(d, ctx); + if (ctx.path === '/ops/access') { const code = ctx.query.code; try { const g = ops.bindGrant(d, code, { ua: ctx.headers['user-agent'], ip: ctx.ip }); ctx.setCookie(ops.grantCookie(g.cookie, ctx.secure)); return ctx.redirect('/ops/'); } catch (e) { throw new HttpError(403, 'access_denied', '접근할 수 없습니다'); } } + if (!gate.ok) throw new HttpError(403, 'access_denied', '접근할 수 없습니다'); // 로그인 화면도 주지 않는다(§25) + ctx.opsVia = gate.via; + const tok = ctx.cookies[session.cookieName('operator')]; ctx.session = session.resolveSession(d.store, tok, 'operator'); ctx.operator = ctx.session ? ops.publicAccount(d.store.get('SELECT * FROM ops_accounts WHERE id=?', ctx.session.user_id)) : null; + if (ctx.path.startsWith('/ops/api/') && !['/ops/api/login', '/ops/api/setup', '/ops/api/me'].includes(ctx.path)) { if (!ctx.operator) throw new HttpError(401, 'unauthenticated', '운영자 로그인이 필요합니다'); if (!['GET', 'HEAD'].includes(ctx.method) && !session.csrfOk(ctx.session, ctx.headers['x-csrf'])) throw new HttpError(403, 'csrf', '보안 토큰이 맞지 않습니다'); } + }); + const requireRecent = (ctx) => { if (!session.recentlyAuthed(ctx.session)) throw new HttpError(403, 'reauth_required', '고위험 작업은 최근 10분 안의 재인증이 필요합니다', { action: 'reauth' }); }; + /** 고위험 작업 3단계 확인(§25): 영향 범위 확인 → 대상 식별정보 재입력 → 최종 실행 */ + const threeStep = (b, expectIds) => { if (b.confirmImpact !== true) throw new HttpError(400, 'confirm_impact_required', '1단계: 영향 범위를 확인해야 합니다'); const ids = [].concat(b.confirmIdentifiers || []).sort(); if (JSON.stringify(ids) !== JSON.stringify([].concat(expectIds).sort())) throw new HttpError(400, 'identifiers_mismatch', '2단계: 대상 식별정보가 다릅니다'); if (b.confirmExecute !== true) throw new HttpError(400, 'confirm_execute_required', '3단계: 최종 실행을 확인해야 합니다'); }; + app.post('/ops/api/setup', wrap(async (ctx) => { const b = await ctx.body(); const st = core.bootstrapState(d); const tokFile = join(d.config.dataDir, 'setup.token'); const expected = existsSync(tokFile) ? readFileSync(tokFile, 'utf8').trim() : ''; if (core.ensureOpsAccountExists(d) || !expected || b.setupToken !== expected) throw new HttpError(403, 'setup_closed', '최초 설정은 설정 토큰으로 한 번만 할 수 있습니다'); const a = ops.createAccount(d, { login: b.login, password: b.password, displayName: b.displayName, by: null }); try { (await import('node:fs')).unlinkSync(tokFile); } catch (_) {} return { ok: true, account: a, bootstrap: st }; })); + app.post('/ops/api/login', wrap(async (ctx) => { const b = await ctx.body(); const r = ops.login(d, { login: b.login, password: b.password, totp: b.totp, ip: ctx.ip, ua: ctx.headers['user-agent'] }); if (r.mfaRequired) return { ok: true, mfaRequired: true, message: r.message }; ctx.setCookie(cookie(session.cookieName('operator'), r.session.token, { maxAge: 86400, path: '/ops', secure: ctx.secure })); return { ok: true, account: r.account, csrf: r.session.csrf, via: ctx.opsVia }; })); + app.get('/ops/api/me', (ctx) => ({ ok: true, operator: ctx.operator, csrf: ctx.session?.csrf || null, via: ctx.opsVia, recentlyAuthed: session.recentlyAuthed(ctx.session), bootstrap: core.selfHealth(d).bootstrap, setupNeeded: !core.ensureOpsAccountExists(d) })); + app.post('/ops/api/logout', (ctx) => { session.revokeSession(d.store, ctx.session.id, ctx.session.user_id); ctx.setCookie(cookie(session.cookieName('operator'), '', { path: '/ops' })); return { ok: true }; }); + app.post('/ops/api/reauth', wrap(async (ctx) => { const b = await ctx.body(); ops.reauth(d, ctx.operator.id, b.password, ctx.session.id); return { ok: true }; })); + app.post('/ops/api/password', wrap(async (ctx) => { const b = await ctx.body(); ops.changePassword(d, ctx.operator.id, b.current, b.next, ctx.session.id); return { ok: true }; })); + app.post('/ops/api/mfa/begin', wrap((ctx) => ({ ok: true, ...ops.mfaBegin(d, ctx.operator.id) }))); + app.post('/ops/api/mfa/confirm', wrap(async (ctx) => ({ ok: true, ...ops.mfaConfirm(d, ctx.operator.id, (await ctx.body()).code) }))); + app.post('/ops/api/mfa/disable', wrap(async (ctx) => { requireRecent(ctx); ops.mfaDisable(d, ctx.operator.id, (await ctx.body()).code); return { ok: true }; })); + app.get('/ops/api/sessions', (ctx) => ({ ok: true, sessions: session.listSessions(d.store, ctx.operator.id, 'operator').map((s) => ({ ...s, current: s.id === ctx.session.id })) })); + app.post('/ops/api/sessions/revoke', wrap(async (ctx) => { const b = await ctx.body(); const n = b.all ? session.revokeOtherSessions(d.store, ctx.operator.id, ctx.session.id, 'operator') : session.revokeSession(d.store, b.id, ctx.operator.id); return { ok: true, revoked: n }; })); + app.get('/ops/api/accounts', (ctx) => ({ ok: true, accounts: d.store.all('SELECT * FROM ops_accounts ORDER BY created_at').map(ops.publicAccount) })); + app.post('/ops/api/accounts', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); return { ok: true, account: ops.createAccount(d, { ...b, by: ctx.operator.id, mustChange: true }) }; })); + app.post('/ops/api/access-grants', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); return { ok: true, grant: ops.issueGrant(d, { accountId: b.accountId || ctx.operator.id, hours: b.hours || 8, by: ctx.operator.id, reason: b.reason || '' }) }; })); + app.get('/ops/api/access-grants', (ctx) => ({ ok: true, grants: d.store.all('SELECT id, account_id, created_by, created_at, expires_at, bound_at, revoked_at, reason, ip FROM ops_access_grants ORDER BY created_at DESC LIMIT 200') })); + app.post('/ops/api/access-grants/:id/revoke', wrap((ctx) => { ops.revokeGrant(d, ctx.params.id, ctx.operator.id); return { ok: true }; })); + // 운영 현황(§26.1) + app.get('/ops/api/overview', (ctx) => overview(d)); + app.get('/ops/api/config', () => ({ ok: true, config: { publicUrl: d.config.publicUrl, portalUrl: d.config.portalUrl, opsAllowCidrs: d.config.opsAllowCidrs, metricsRetentionDays: d.config.metricsRetentionDays, offlineAfterMs: d.config.offlineAfterMs, emergencyTtlMinutes: d.config.emergencyTtlMinutes } })); + app.put('/ops/api/config', wrap(async (ctx) => { requireRecent(ctx); d.config.set(await ctx.body(), ctx.operator.id); return { ok: true }; })); + // 서버(§28) + app.get('/ops/api/servers', () => ({ ok: true, servers: core.listServers(d) })); + app.post('/ops/api/servers', wrap(async (ctx) => { const b = await ctx.body(); const s = core.createServer(d, { ...b, by: ctx.operator.id }); return { ok: true, server: core.publicServer(d, s) }; })); + app.get('/ops/api/servers/:id', wrap((ctx) => { const s = core.getServer(d, ctx.params.id); if (!s) throw new HttpError(404, 'server_not_found', '서버가 없습니다'); return { ok: true, server: core.publicServer(d, s), jobs: d.jobs.list({ subjectType: 'server', subjectId: s.id, limit: 50 }).map(describe), agentJobs: d.store.all('SELECT id, kind, status, progress, created_at, ended_at, error FROM agent_jobs WHERE agent_id=? ORDER BY created_at DESC LIMIT 50', s.agent_id || '') }; })); + app.get('/ops/api/servers/:id/metrics', (ctx) => ({ ok: true, samples: core.metrics(d, ctx.params.id, { since: ctx.query.since, until: ctx.query.until, limit: Number(ctx.query.limit || 5000) }) })); + app.patch('/ops/api/servers/:id', wrap(async (ctx) => { const b = await ctx.body(); if (b.status === 'isolated' || b.role) requireRecent(ctx); return { ok: true, server: core.publicServer(d, core.updateServer(d, ctx.params.id, b, ctx.operator.id)) }; })); + app.post('/ops/api/servers/:id/token', wrap((ctx) => ({ ok: true, ...core.issueRegistrationToken(d, { kind: 'managed', serverId: ctx.params.id, by: ctx.operator.id }) }))); + app.post('/ops/api/servers/:id/approve', wrap((ctx) => ({ ok: true, server: core.publicServer(d, core.approveServer(d, ctx.params.id, ctx.operator.id)) }))); + app.post('/ops/api/servers/:id/maintenance', wrap(async (ctx) => { const b = await ctx.body(); const s = core.updateServer(d, ctx.params.id, b.end ? { status: 'ready', accepts_new: b.acceptsNew === false ? 0 : 1, maintenance_note: null } : { status: 'maintenance', accepts_new: 0, maintenance_note: b.note || '' }, ctx.operator.id); return { ok: true, server: core.publicServer(d, s) }; })); + app.post('/ops/api/servers/:id/delete', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); const s = core.getServer(d, ctx.params.id); if (!s) throw new HttpError(404, 'server_not_found', '서버가 없습니다'); threeStep(b, [s.id]); const hosted = (s.capacity && s.capacity.sitesHosted) || 0; if (hosted > 0 && !b.force) throw new HttpError(409, 'server_in_use', `이 서버에 ${hosted}개 서비스가 배치되어 있습니다 — 먼저 이전하세요`); if (s.agent_id) core.revokeAgent(d, s.agent_id, ctx.operator.id); d.store.run('DELETE FROM servers WHERE id=?', s.id); d.store.run('DELETE FROM module_targets WHERE server_id=?', s.id); listAudit; return { ok: true }; })); + // 릴리스·모듈 배포(§32) + app.get('/ops/api/releases', () => ({ ok: true, releases: core.listReleases(d), running: core.selfHealth(d).bootstrap, modules: Object.keys(MODULES), deployOrder: DEPLOY_ORDER, targets: d.store.all('SELECT * FROM module_targets ORDER BY module, server_id') })); + app.post('/ops/api/releases/:version/approve', wrap((ctx) => { requireRecent(ctx); return { ok: true, release: core.approveRelease(d, ctx.params.version, ctx.operator.id) }; })); + app.post('/ops/api/modules/:name/deploy', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); const r = d.jobs.enqueue({ kind: 'module.deploy', subject: { module: ctx.params.name, version: b.version, serverIds: b.serverIds || [] }, idempotencyKey: b.idempotencyKey || `module.deploy:${ctx.params.name}:${b.version}:${(b.serverIds || []).slice().sort().join(',')}:${b.nonce || ''}`, createdBy: ctx.operator.id, actorType: 'operator', subjectType: 'module', subjectId: ctx.params.name, label: `${ctx.params.name} ${b.version} 배포` }); return { ok: true, job: describe(r.job), created: r.created }; })); + app.post('/ops/api/modules/:name/rollback', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); threeStep(b, b.serverIds || []); const r = d.jobs.enqueue({ kind: 'module.rollback', subject: { module: ctx.params.name, toVersion: b.toVersion || null, serverIds: b.serverIds || [] }, createdBy: ctx.operator.id, actorType: 'operator', subjectType: 'module', subjectId: ctx.params.name, label: `${ctx.params.name} 되돌리기` }); return { ok: true, job: describe(r.job) }; })); + app.post('/ops/api/modules/:name/health', wrap(async (ctx) => { const b = await ctx.body(); const r = d.jobs.enqueue({ kind: 'module.health', subject: { module: ctx.params.name, serverIds: b.serverIds || null }, createdBy: ctx.operator.id, actorType: 'operator', subjectType: 'module', subjectId: ctx.params.name, label: `${ctx.params.name} 건강 확인` }); return { ok: true, job: describe(r.job) }; })); + app.post('/ops/api/modules/reconcile', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); const r = d.jobs.enqueue({ kind: 'module.reconcile', subject: { module: b.module || null }, createdBy: ctx.operator.id, actorType: 'operator', label: '목표·실제 상태 맞추기' }); return { ok: true, job: describe(r.job) }; })); + // 작업(§29) + app.get('/ops/api/jobs', (ctx) => ({ ok: true, jobs: d.jobs.list({ status: ctx.query.status ? ctx.query.status.split(',') : null, kind: ctx.query.kind || null, limit: Number(ctx.query.limit || 200) }).map((j) => ({ ...describe(j), subject: redact(j.subject), createdBy: j.created_by, actorType: j.actor_type, groupId: j.group_id })) })); + app.get('/ops/api/jobs/:id', wrap((ctx) => { const j = d.jobs.get(ctx.params.id); if (!j) throw new HttpError(404, 'job_not_found', '작업이 없습니다'); return { ok: true, job: { ...describe(j), subject: redact(j.subject), result: j.result, log: j.log, createdBy: j.created_by, actorType: j.actor_type, groupId: j.group_id }, children: d.jobs.childrenSummary(j.id), agentJobs: d.store.all('SELECT id, agent_id, kind, status, progress, error, user_message, created_at, ended_at FROM agent_jobs WHERE hal_job_id=? ORDER BY created_at', j.id) }; })); + for (const act of ['cancel', 'retry', 'resume', 'revert', 'pause']) app.post(`/ops/api/jobs/:id/${act}`, wrap(async (ctx) => { if (act === 'revert') requireRecent(ctx); const j = await d.jobs[act](ctx.params.id, ctx.operator.id); return { ok: true, job: describe(j) }; })); + // 사건·알림(§31) + app.get('/ops/api/incidents', () => ({ ok: true, incidents: d.store.all('SELECT * FROM incidents ORDER BY started_at DESC LIMIT 200'), notifications: listNotifications(d.store, { groupId: null, limit: 200 }) })); + app.post('/ops/api/incidents', wrap(async (ctx) => { const b = await ctx.body(); const open = d.store.get('SELECT id FROM incidents WHERE key=? AND resolved_at IS NULL', b.key); if (open) { d.store.run('UPDATE incidents SET severity=?, title=?, customer_impact=?, updated_at=? WHERE id=?', b.severity || 'warning', b.title, b.customerImpact || null, nowIso(), open.id); return { ok: true, id: open.id, merged: true }; } const id = newId('inc_'); d.store.run('INSERT INTO incidents(id, key, severity, title, component, customer_impact, started_at, assignee, public, public_note, notes_json, updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,\'[]\',?)', id, b.key, b.severity || 'warning', b.title, b.component || null, b.customerImpact || null, nowIso(), ctx.operator.id, b.public ? 1 : 0, b.publicNote || null, nowIso()); return { ok: true, id }; })); + app.patch('/ops/api/incidents/:id', wrap(async (ctx) => { const b = await ctx.body(); const inc = d.store.get('SELECT * FROM incidents WHERE id=?', ctx.params.id); if (!inc) throw new HttpError(404, 'not_found', '사건이 없습니다'); if (b.resolve && !b.verifiedUserPath) throw new HttpError(400, 'verify_required', '실제 사용자 경로 확인 뒤에 종료할 수 있습니다(§31.2)'); const notes = parseJson(inc.notes_json, []); if (b.note) notes.push({ at: nowIso(), by: ctx.operator.id, note: b.note }); d.store.run('UPDATE incidents SET severity=COALESCE(?, severity), assignee=COALESCE(?, assignee), public=COALESCE(?, public), public_note=COALESCE(?, public_note), notes_json=?, resolved_at=CASE WHEN ? THEN ? ELSE resolved_at END, verified_at=CASE WHEN ? THEN ? ELSE verified_at END, updated_at=? WHERE id=?', b.severity || null, b.assignee || null, b.public == null ? null : (b.public ? 1 : 0), b.publicNote || null, toJson(notes), b.resolve ? 1 : 0, nowIso(), b.resolve ? 1 : 0, nowIso(), nowIso(), inc.id); return { ok: true }; })); + app.get('/ops/api/audit', (ctx) => ({ ok: true, entries: listAudit(d.store, { action: ctx.query.action, subjectType: ctx.query.subjectType, subjectId: ctx.query.subjectId, since: ctx.query.since, limit: Number(ctx.query.limit || 200) }) })); + // 긴급 복구(§33) + app.get('/ops/api/emergency', () => ({ ok: true, sessions: emergency.listSessions(d), scopes: emergency.SCOPES, actions: emergency.ACTIONS })); + app.post('/ops/api/emergency', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); return { ok: true, session: emergency.declare(d, { ...b, by: ctx.operator.id }) }; })); + app.post('/ops/api/emergency/:id/approve', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); return { ok: true, session: emergency.approve(d, ctx.params.id, { ...b, by: ctx.operator.id }) }; })); + app.post('/ops/api/emergency/:id/credential', wrap((ctx) => { requireRecent(ctx); return { ok: true, ...emergency.issueCredential(d, ctx.params.id, ctx.operator.id) }; })); + app.post('/ops/api/emergency/:id/perform', wrap(async (ctx) => { const b = await ctx.body(); return { ok: true, result: await emergency.perform(d, ctx.params.id, { ...b, by: ctx.operator.id }) }; })); + app.post('/ops/api/emergency/:id/verify', wrap(async (ctx) => ({ ok: true, session: emergency.verifyAndRevoke(d, ctx.params.id, { by: ctx.operator.id, verify: (await ctx.body()).verify }) }))); + app.post('/ops/api/emergency/:id/reconcile', wrap((ctx) => ({ ok: true, result: emergency.reconcile(d, ctx.params.id, ctx.operator.id) }))); + app.post('/ops/api/emergency/:id/close', wrap(async (ctx) => ({ ok: true, session: emergency.close(d, ctx.params.id, ctx.operator.id, (await ctx.body()).note) }))); + // 재해복구(§34) + app.get('/ops/api/dr', () => ({ ok: true, readiness: dr.readiness(d), snapshots: dr.listSnapshots(d), restores: d.store.all('SELECT * FROM dr_restores ORDER BY rowid DESC LIMIT 50'), drills: d.store.all('SELECT * FROM dr_drills ORDER BY started_at DESC LIMIT 50') })); + app.put('/ops/api/dr/config', wrap(async (ctx) => { requireRecent(ctx); return { ok: true, config: dr.setConfig(d, await ctx.body(), ctx.operator.id) }; })); + app.post('/ops/api/dr/snapshots', wrap((ctx) => ({ ok: true, snapshot: dr.snapshot(d, { kind: 'manual', by: ctx.operator.id }) }))); + app.post('/ops/api/dr/restores', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); return { ok: true, restore: dr.requestRestore(d, { snapshotId: b.snapshotId, by: ctx.operator.id, note: b.note }) }; })); + app.post('/ops/api/dr/restores/:id/approve', wrap((ctx) => { requireRecent(ctx); return { ok: true, restore: dr.approveRestore(d, ctx.params.id, ctx.operator.id) }; })); + app.post('/ops/api/dr/restores/:id/start', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); return { ok: true, ...dr.startRestore(d, ctx.params.id, { by: ctx.operator.id, oldControlStopped: b.oldControlStopped, confirmText: b.confirmText }) }; })); + app.post('/ops/api/dr/restores/:id/finish', wrap(async (ctx) => ({ ok: true, restore: dr.finishRestore(d, ctx.params.id, { by: ctx.operator.id, checks: (await ctx.body()).checks }) }))); + app.post('/ops/api/dr/drills', wrap(async (ctx) => ({ ok: true, drill: dr.startDrill(d, { by: ctx.operator.id, snapshotId: (await ctx.body()).snapshotId }) }))); + app.post('/ops/api/dr/drills/:id/finish', wrap(async (ctx) => ({ ok: true, drill: dr.finishDrill(d, ctx.params.id, { by: ctx.operator.id, ...(await ctx.body()) }) }))); + // 포털 운영 화면 중계(§25·§26.2): 포털의 /internal/ops/* 를 운영자 세션으로 대신 부른다 + app.route('*', '/ops/api/portal/*', wrap(async (ctx) => { const secret = d.vault.get('service:hal'); if (!secret) throw new HttpError(503, 'portal_not_linked', '포털 연동 키가 없습니다'); if (!['GET', 'HEAD'].includes(ctx.method)) requireRecent(ctx); const rest = ctx.path.replace(/^\/ops\/api\/portal/, '') + (ctx.url.search || ''); const body = ['GET', 'HEAD'].includes(ctx.method) ? undefined : { ...(await ctx.body()), _operator: { id: ctx.operator.id, login: ctx.operator.login } }; const { serviceFetch } = d.serviceauth; const r = await serviceFetch(d.config.portalUrl, 'hal', secret, ctx.method, '/internal/ops' + rest, body, { timeoutMs: 20000, fetchImpl: d.fetch || fetch }).catch((e) => { throw new HttpError(503, 'portal_unreachable', '포털에 연결할 수 없습니다 — 정보 확인 지연'); }); ctx.status = r.status; return r.json || { ok: false, error: 'portal_bad_response' }; })); + app.get('/ops', (ctx) => ctx.redirect('/ops/')); app.get('/ops/', (ctx) => { const f = join(uiDir, 'index.html'); return ctx.html(existsSync(f) ? readFileSync(f, 'utf8') : `Ariweb HAL

Ariweb HAL 운영자 콘솔

화면(UI)은 아직 설치되지 않았습니다. API: /ops/api/me

`); }); + return app; +} +function stripFacts(s) { const { facts, ...rest } = s; return rest; } +/** 운영 현황(§26.1) */ +function overview(d) { + const servers = core.listServers(d); const health = core.selfHealth(d); + const jobs = d.jobs.list({ status: ['queued', 'running', 'awaiting_external', 'needs_user', 'reverting'], limit: 500 }); + const comp = (name, ok, extra = {}) => ({ name, status: ok === null ? 'unknown' : ok ? 'ok' : 'down', ...extra }); + const modState = (m) => { const t = d.store.all('SELECT health, actual_version, last_check_at FROM module_targets WHERE module=?', m); if (!t.length) return null; const stale = t.some((x) => !x.last_check_at || Date.now() - Date.parse(x.last_check_at) > 10 * 60 * 1000); return { ok: t.every((x) => x.health === 'ok'), stale, versions: [...new Set(t.map((x) => x.actual_version))] }; }; + const components = ['hal', 'portal', 'webmaker', 'arimail', 'backup', 'observability'].map((m) => { if (m === 'hal') return comp('HAL', health.ok, { version: health.version }); const s = modState(m); return s ? comp(m, s.stale ? null : s.ok, { versions: s.versions, stale: s.stale }) : comp(m, null, { note: '배치 없음' }); }); + return { ok: true, hal: health, components, servers: { total: servers.length, offline: servers.filter((s) => s.status === 'offline').length, pending: servers.filter((s) => s.status === 'pending').length, maintenance: servers.filter((s) => s.rawStatus === 'maintenance').length, diskWarning: servers.filter((s) => s.capacity.diskTotalBytes && (s.capacity.diskTotalBytes - s.capacity.diskFreeBytes) / s.capacity.diskTotalBytes >= 0.9).length }, + jobs: { running: jobs.filter((j) => j.status === 'running').length, waiting: jobs.filter((j) => ['awaiting_external', 'queued'].includes(j.status)).length, needsUser: jobs.filter((j) => j.status === 'needs_user').length, byKind: jobs.reduce((a, j) => { a[j.kind] = (a[j.kind] || 0) + 1; return a; }, {}) }, + incidents: d.store.all('SELECT id, key, severity, title, started_at, customer_impact FROM incidents WHERE resolved_at IS NULL ORDER BY started_at DESC'), drift: d.store.all('SELECT module, server_id, target_version, actual_version FROM module_targets WHERE target_version IS NOT NULL AND (actual_version IS NULL OR actual_version<>target_version)'), dr: dr.readiness(d), emergencyOpen: d.store.all('SELECT id, status, incident_key FROM emergency_sessions WHERE status<>\'closed\''), generatedAt: nowIso() }; +} +/** 공개 상태(§31.3): 내부 이름·고객 정보 없음. 고객 영향이 없는 내부 경고는 올리지 않는다. */ +function publicStatus(d) { + const o = overview(d); + const label = { HAL: '관리 플랫폼', portal: '고객 포털', webmaker: 'Webmaker', arimail: '메일', backup: '백업', observability: '모니터링' }; + return { ok: true, generatedAt: o.generatedAt, components: o.components.map((c) => ({ name: label[c.name] || c.name, status: c.status === 'ok' ? 'ok' : c.status === 'down' ? 'down' : 'unknown', label: c.status === 'ok' ? '정상' : c.status === 'down' ? '장애' : '확인 중' })), incidents: o.incidents.filter((i) => d.store.get('SELECT public FROM incidents WHERE id=?', i.id)?.public).map((i) => ({ severity: i.severity, title: i.title, startedAt: i.started_at, note: d.store.get('SELECT public_note FROM incidents WHERE id=?', i.id)?.public_note || '' })), maintenance: d.store.all('SELECT name, maintenance_note FROM servers WHERE status=\'maintenance\'').map((s) => ({ note: s.maintenance_note || '계획된 유지보수' })) }; +} +function statusPage(d) { const s = publicStatus(d); return `Ariweb 서비스 상태

Ariweb 서비스 상태

갱신 ${esc(s.generatedAt)}

    ${s.components.map((c) => `
  • ${c.status === 'ok' ? '●' : c.status === 'down' ? '✘' : '◔'} ${esc(c.label)} — ${esc(c.name)}
    • `).join('')}

진행 중인 사건

${s.incidents.length ? `
    ${s.incidents.map((i) => `
  • ${esc(i.title)} (${esc(i.startedAt)})
    ${esc(i.note)}
    • `).join('')}
` : '

진행 중인 사건이 없습니다.

'}

계획된 유지보수

${s.maintenance.length ? `
    ${s.maintenance.map((m) => `
  • ${esc(m.note)}
    • `).join('')}
` : '

예정된 유지보수가 없습니다.

'}`; } diff --git a/hal/cidr.mjs b/hal/cidr.mjs new file mode 100644 index 0000000..d1d2496 --- /dev/null +++ b/hal/cidr.mjs @@ -0,0 +1,21 @@ +/** hal/cidr.mjs — IP 허용 목록(CIDR) 검사(§25 운영자 콘솔 네트워크 제한). IPv4·IPv6·::ffff: 로 감싼 IPv4 를 다룬다. */ +import { isIP } from 'node:net'; + +function toBytes(ip) { + let s = String(ip || '').trim(); if (s.startsWith('::ffff:') && isIP(s.slice(7)) === 4) s = s.slice(7); + if (isIP(s) === 4) return { v: 4, b: s.split('.').map(Number) }; + if (isIP(s) !== 6) return null; + const [head, tail] = s.split('::'); const h = head ? head.split(':') : []; const t = tail ? tail.split(':') : []; const parts = [...h, ...Array(8 - h.length - t.length).fill('0'), ...t]; + const b = []; for (const p of parts) { const n = parseInt(p || '0', 16); b.push(n >> 8, n & 255); } return { v: 6, b }; +} +/** ip 가 cidr 목록 중 하나에 속하면 true. 목록이 비면 false(허용 네트워크 없음). */ +export function ipAllowed(ip, cidrs) { + const a = toBytes(ip); if (!a) return false; + for (const c of cidrs || []) { + const [net, lenS] = String(c).split('/'); const n = toBytes(net); if (!n || n.v !== a.v) continue; + const len = lenS === undefined ? (n.v === 4 ? 32 : 128) : Number(lenS); if (!(len >= 0 && len <= (n.v === 4 ? 32 : 128))) continue; + let ok = true; for (let i = 0; i < len; i++) { const byte = i >> 3, bit = 7 - (i & 7); if (((a.b[byte] >> bit) & 1) !== ((n.b[byte] >> bit) & 1)) { ok = false; break; } } + if (ok) return true; + } + return false; +} diff --git a/hal/config.mjs b/hal/config.mjs new file mode 100644 index 0000000..ccd64a9 --- /dev/null +++ b/hal/config.mjs @@ -0,0 +1,41 @@ +/** + * hal/config.mjs — HAL 실행 설정. + * + * 배포 때 정해지는 값(포트·데이터 디렉터리)은 환경 변수로, 운영 중 바꾸는 값(공개 주소·포털 주소·허용 네트워크·메트릭 보존 일수)은 + * data/config.json 에 둔다. 비밀값(서비스 키·Agent 비밀)은 설정이 아니라 보관소(lib/vault)에만 둔다. + */ +import { readFileSync, existsSync, writeFileSync, mkdirSync, renameSync } from 'node:fs'; +import { join } from 'node:path'; + +export function loadConfig() { + const dataDir = process.env.ARIWEB_HAL_DATA || process.env.DATA_DIR || join(process.cwd(), 'data'); + mkdirSync(dataDir, { recursive: true, mode: 0o700 }); + const file = join(dataDir, 'config.json'); + let values = {}; try { values = existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')).values || {} : {}; } catch (_) { values = {}; } + const cfg = { + dataDir, file, + port: Number(process.env.PORT || 8900), host: process.env.HOST || '127.0.0.1', + /** HAL 공개 주소(Agent 가 접속하는 주소) */ + publicUrl: values.publicUrl || process.env.HAL_PUBLIC_URL || 'https://hal.ariweb.net', + /** 포털 내부 주소(작업 이벤트를 미는 곳) */ + portalUrl: values.portalUrl || process.env.PORTAL_URL || 'http://127.0.0.1:8910', + /** 운영자 콘솔 접근 허용 네트워크(CIDR 목록, §25). 비어 있으면 접근권(grant) 쿠키만 허용 */ + opsAllowCidrs: values.opsAllowCidrs || (process.env.OPS_ALLOW_CIDRS ? process.env.OPS_ALLOW_CIDRS.split(',').map((s) => s.trim()).filter(Boolean) : ['127.0.0.1/32', '::1/128', '10.0.0.0/8', '192.168.0.0/16']), + /** 메트릭 보존 일수(최소 90, §28.1) */ + metricsRetentionDays: Math.max(90, Number(values.metricsRetentionDays || 90)), + /** heartbeat 가 이만큼 없으면 서버를 offline 으로 본다 */ + offlineAfterMs: Number(values.offlineAfterMs || 90 * 1000), + /** 전송(transfer) 파일 보관 시간 */ + transferTtlMs: Number(values.transferTtlMs || 24 * 3600 * 1000), + /** 긴급 자격 최대 유효 시간(분) */ + emergencyTtlMinutes: Math.min(60, Number(values.emergencyTtlMinutes || 30)), + values, + /** 설정 저장(운영자 화면·CLI). 비밀값은 거부한다. */ + set(patch, by = 'ops') { + for (const k of Object.keys(patch)) if (/key|secret|token|pass/i.test(k)) throw new Error(`비밀값(${k})은 설정 파일에 둘 수 없습니다 — 보관소를 쓰세요`); + Object.assign(values, patch); const tmp = file + '.tmp'; + writeFileSync(tmp, JSON.stringify({ version: 1, updatedAt: new Date().toISOString(), updatedBy: by, values }, null, 1), { mode: 0o600 }); renameSync(tmp, file); Object.assign(cfg, patch); + }, + }; + return cfg; +} diff --git a/hal/core.mjs b/hal/core.mjs new file mode 100644 index 0000000..a23a575 --- /dev/null +++ b/hal/core.mjs @@ -0,0 +1,249 @@ +/** + * hal/core.mjs — HAL 핵심 서비스: 서버·Agent 등록부, Agent 작업 배정·대기, 릴리스 등록부, 부트스트랩 상태, 지표, 포털 이벤트 outbox. + * app.mjs(라우트)와 jobs.mjs(작업 종류)가 함께 쓴다. 모든 함수는 deps(store·vault·config·log·jobs) 를 받는다. + */ +import { newId, randomToken, sha256, hashPassword } from '../lib/crypto.mjs'; +import { nowIso, toJson, parseJson } from '../lib/db.mjs'; +import { audit } from '../lib/audit.mjs'; +import { notify } from '../lib/notify.mjs'; +import { redact, safeError } from '../lib/redact.mjs'; +import { serviceFetch } from '../lib/serviceauth.mjs'; +import { JobError } from '../lib/jobs.mjs'; +import { existsSync, statSync, mkdirSync } from 'node:fs'; +import { join } from 'node:path'; + +export const EXTERNAL_KINDS = new Set(['site.deploy', 'site.remove', 'site.check', 'site.rollback']); // 외부 Agent 가 받을 수 있는 작업(§10.2) +export const SERVER_ROLES = ['web', 'mail', 'backup', 'control']; + +// ------------------------------------------------------------------ 부트스트랩 상태(§32.2) +export function bootstrapState(d) { let r = d.store.get('SELECT * FROM bootstrap_state WHERE id=1'); if (!r) { d.store.run('INSERT INTO bootstrap_state(id, updated_at) VALUES(1, ?)', nowIso()); r = d.store.get('SELECT * FROM bootstrap_state WHERE id=1'); } return r; } +export function setBootstrap(d, patch) { const sets = Object.keys(patch).map((k) => `${k}=?`); d.store.run(`UPDATE bootstrap_state SET ${sets.join(', ')}, updated_at=? WHERE id=1`, ...Object.values(patch), nowIso()); return bootstrapState(d); } +/** 부트스트랩 완료 여부: HAL 실행 버전 기록 + 첫 Agent 등록·승인 */ +export function bootstrapComplete(d) { const b = bootstrapState(d); return !!(b.hal_running_version && b.first_agent_id && b.completed_at); } +/** HAL 기동 때: 실행 버전·인스턴스 임대 기록. 다른 인스턴스의 최근 임대가 있으면 거부(§34 "하나의 HAL 만 운영"). */ +export function claimLive(d, { instanceId, version, takeover = false, now = Date.now() }) { + const b = bootstrapState(d); + if (b.live_instance_id && b.live_instance_id !== instanceId && b.live_lease_at && now - Date.parse(b.live_lease_at) < 60000 && !takeover) { + throw Object.assign(new Error(`다른 HAL 인스턴스(${b.live_instance_id})가 ${b.live_lease_at} 에 운영 중입니다. 기존 제어 위치를 먼저 중지하세요`), { code: 'another_hal_live' }); + } + return setBootstrap(d, { live_instance_id: instanceId, live_lease_at: new Date(now).toISOString(), hal_running_version: version, hal_started_at: nowIso() }); +} +export function renewLease(d, instanceId) { d.store.run('UPDATE bootstrap_state SET live_lease_at=? WHERE id=1 AND live_instance_id=?', nowIso(), instanceId); } +/** 첫 Agent 가 승인되면 부트스트랩을 닫고 수동 설치를 잠근다 */ +export function maybeCompleteBootstrap(d, agentId) { + const b = bootstrapState(d); if (b.completed_at) return b; + if (!b.first_agent_id) setBootstrap(d, { first_agent_id: agentId, first_agent_registered_at: nowIso() }); + if (bootstrapState(d).hal_running_version) { const r = setBootstrap(d, { completed_at: nowIso(), manual_install_locked: 1, first_install: 0 }); audit(d.store, { actorType: 'system', action: 'bootstrap.complete', subjectType: 'agent', subjectId: agentId, immutable: true }); return r; } + return bootstrapState(d); +} +/** HAL 자체 건강(§32.2 4단계) */ +export function selfHealth(d) { + const b = bootstrapState(d); let dbOk = true; try { d.store.get('SELECT 1 AS x'); } catch (_) { dbOk = false; } + return { ok: dbOk && !!d.jobsRunning, version: d.version, uptimeSec: Math.round(process.uptime()), db: dbOk, jobs: !!d.jobsRunning, bootstrap: { complete: bootstrapComplete(d), firstInstall: !!b.first_install, releaseVersion: b.release_version, mainCommit: b.main_commit, artifactSha256: b.artifact_sha256, halRunningVersion: b.hal_running_version, firstAgent: b.first_agent_id, manualInstallLocked: !!b.manual_install_locked }, instanceId: d.instanceId }; +} + +// ------------------------------------------------------------------ 서버·Agent +export function createServer(d, { name, role, location = '', by = null }) { + if (!SERVER_ROLES.includes(role)) throw new JobError('bad_role', `서버 역할은 ${SERVER_ROLES.join('/')} 중 하나입니다`); + const id = newId('srv_'); const now = nowIso(); + d.store.run('INSERT INTO servers(id, name, role, location, status, created_at, updated_at) VALUES(?,?,?,?,\'pending\',?,?)', id, String(name).slice(0, 80), role, String(location).slice(0, 120), now, now); + audit(d.store, { actorType: 'operator', actorId: by, action: 'server.create', subjectType: 'server', subjectId: id, detail: { name, role, location } }); + return getServer(d, id); +} +export function getServer(d, id) { const s = d.store.get('SELECT * FROM servers WHERE id=?', id); return s ? { ...s, metrics: parseJson(s.metrics_json, null), capacity: parseJson(s.capacity_json, null), facts: parseJson(s.facts_json, {}) } : null; } +export function listServers(d, { role = null } = {}) { return d.store.all(`SELECT * FROM servers ${role ? 'WHERE role=?' : ''} ORDER BY created_at`, ...(role ? [role] : [])).map((s) => publicServer(d, s)); } +/** 화면·포털용 서버 요약(비밀 없음). offline 판정은 last_seen 기준(§28.1 heartbeat). */ +export function publicServer(d, s) { + const seen = s.last_seen_at ? Date.now() - Date.parse(s.last_seen_at) : Infinity; const offline = seen > d.config.offlineAfterMs; + const status = s.status === 'pending' ? 'pending' : offline ? 'offline' : s.status; + const mods = d.store.all('SELECT module, target_version, actual_version, health, active, last_check_at, last_error FROM module_targets WHERE server_id=?', s.id); + const capacity = parseJson(s.capacity_json, {}) || {}; const metrics = parseJson(s.metrics_json, null); + return { id: s.id, name: s.name, role: s.role, location: s.location, status, rawStatus: s.status, acceptsNew: !!s.accepts_new && status === 'ready', approvedAt: s.approved_at, agentId: s.agent_id, agentVersion: s.agent_version, lastSeenAt: s.last_seen_at, stale: offline, clockSkewMs: s.clock_skew_ms, + health: status === 'offline' ? 'down' : mods.some((m) => m.health === 'down') ? 'partial_outage' : mods.some((m) => m.health === 'degraded') ? 'degraded' : status === 'ready' ? 'ok' : 'unknown', + metrics, capacity: { diskFreeBytes: capacity.diskFreeBytes || 0, diskTotalBytes: capacity.diskTotalBytes || 0, memFreeBytes: capacity.memFreeBytes || 0, cpuLoad: capacity.cpuLoad || 0, sitesHosted: capacity.sitesHosted || 0, reservedBytes: capacity.reservedBytes || 0, safetyMarginBytes: capacity.safetyMarginBytes || Math.round((capacity.diskTotalBytes || 0) * 0.1), exhaustEta: capacity.exhaustEta || null }, + modules: Object.fromEntries(mods.map((m) => [m.module, { target: m.target_version, actual: m.actual_version, drift: !!(m.target_version && m.actual_version && m.target_version !== m.actual_version), health: m.health, active: !!m.active, lastCheckAt: m.last_check_at, lastError: m.last_error }])), + maintenanceNote: s.maintenance_note, isolatedReason: s.isolated_reason, facts: parseJson(s.facts_json, {}) }; +} +export function updateServer(d, id, patch, by = null) { + const allowed = ['name', 'location', 'status', 'accepts_new', 'maintenance_note', 'isolated_reason', 'role']; + const keys = Object.keys(patch).filter((k) => allowed.includes(k)); if (!keys.length) return getServer(d, id); + d.store.run(`UPDATE servers SET ${keys.map((k) => `${k}=?`).join(', ')}, updated_at=? WHERE id=?`, ...keys.map((k) => patch[k]), nowIso(), id); + audit(d.store, { actorType: 'operator', actorId: by, action: 'server.update', subjectType: 'server', subjectId: id, detail: patch }); + return getServer(d, id); +} +/** 운영자 승인(§28.2 6단계): pending → ready, 신규 배치 허용 */ +export function approveServer(d, id, by) { const s = getServer(d, id); if (!s) throw new JobError('server_not_found', '서버가 없습니다'); if (!s.agent_id) throw new JobError('agent_missing', 'Agent 가 아직 연결되지 않았습니다'); updateServer(d, id, { status: 'ready', accepts_new: 1 }, by); d.store.run('UPDATE servers SET approved_at=?, approved_by=? WHERE id=?', nowIso(), by, id); maybeCompleteBootstrap(d, s.agent_id); return getServer(d, id); } +/** 신규 배치 가능 서버 고르기(§28.3): ready·accepts_new·여유 용량(안전 여유 제외) */ +export function pickServer(d, { role = 'web', needBytes = 0 } = {}) { + const cands = listServers(d, { role }).filter((s) => s.acceptsNew && s.status === 'ready' && (s.capacity.diskFreeBytes - s.capacity.reservedBytes - s.capacity.safetyMarginBytes) >= needBytes); + cands.sort((a, b) => (b.capacity.diskFreeBytes - b.capacity.reservedBytes) - (a.capacity.diskFreeBytes - a.capacity.reservedBytes)); + return cands[0] ? { serverId: cands[0].id, name: cands[0].name } : null; +} +/** 일회용 등록 토큰(10분). 토큰 원문은 이 반환값에서 한 번만 보인다. */ +export function issueRegistrationToken(d, { kind = 'managed', serverId = null, groupId = null, siteId = null, label = '', by = null, ttlMs = 10 * 60 * 1000 }) { + if (kind === 'managed' && serverId && !getServer(d, serverId)) throw new JobError('server_not_found', '서버가 없습니다'); + const token = randomToken(32); const id = newId('rt_'); const expiresAt = new Date(Date.now() + ttlMs).toISOString(); + d.store.run('INSERT INTO registration_tokens(id, token_hash, kind, server_id, group_id, site_id, label, created_by, created_at, expires_at) VALUES(?,?,?,?,?,?,?,?,?,?)', id, sha256(token), kind, serverId, groupId, siteId, label, by, nowIso(), expiresAt); + audit(d.store, { actorType: by ? 'operator' : 'system', actorId: by, action: 'agent.token_issue', subjectType: kind === 'external' ? 'site' : 'server', subjectId: siteId || serverId, detail: { kind, label } }); + return { id, token, expiresAt, installCommand: `curl -fsSL ${d.config.publicUrl}/agent/v1/install.sh | bash -s -- --hal ${d.config.publicUrl} --token ${token}${kind === 'external' ? ` --external --site ${siteId}` : ''}` }; +} +/** + * Agent 등록(토큰 1회 사용). 비밀은 보관소에 두고 응답에서 한 번만 돌려준다. + * @returns {{agentId, secret, serverId, kind}} + */ +export function registerAgent(d, { token, facts = {}, version = '' }) { + const t = d.store.get('SELECT * FROM registration_tokens WHERE token_hash=?', sha256(String(token || ''))); + if (!t) throw new JobError('bad_token', '등록 토큰이 올바르지 않습니다'); + if (t.used_at) throw new JobError('token_used', '이미 사용한 등록 토큰입니다 — 새 토큰을 발급받으세요'); + if (Date.parse(t.expires_at) < Date.now()) throw new JobError('token_expired', '등록 토큰이 만료되었습니다 — 새 토큰을 발급받으세요'); + const agentId = newId('ag_'); const secret = randomToken(32); const now = nowIso(); + return d.store.tx(() => { + const used = d.store.run('UPDATE registration_tokens SET used_at=?, used_by_agent=? WHERE id=? AND used_at IS NULL', now, agentId, t.id).changes; + if (!used) throw new JobError('token_used', '이미 사용한 등록 토큰입니다'); + let serverId = t.server_id; + if (t.kind === 'managed' && !serverId) { serverId = createServer(d, { name: facts.hostname || agentId, role: facts.role || 'web', location: facts.location || '' }).id; } + d.store.run('INSERT INTO agents(id, server_id, kind, secret_hash, status, label, group_id, site_id, version, registered_at, last_seen_at, facts_json) VALUES(?,?,?,?,\'active\',?,?,?,?,?,?,?)', agentId, serverId, t.kind, sha256(secret), t.label || facts.hostname || '', t.group_id, t.site_id, version, now, now, toJson(redact(facts))); + d.vault.set(`service:agent:${agentId}`, secret, { kind: 'agent-secret' }); + if (serverId) d.store.run('UPDATE servers SET agent_id=?, last_seen_at=?, agent_version=?, facts_json=?, updated_at=? WHERE id=?', agentId, now, version, toJson(redact(facts)), now, serverId); + audit(d.store, { actorType: 'agent', actorId: agentId, action: 'agent.register', subjectType: t.kind === 'external' ? 'site' : 'server', subjectId: t.site_id || serverId, detail: { kind: t.kind, version, hostname: facts.hostname } }); + return { agentId, secret, serverId, kind: t.kind, siteId: t.site_id, groupId: t.group_id }; + }); +} +export function getAgent(d, id) { return d.store.get('SELECT * FROM agents WHERE id=?', id) || null; } +export function revokeAgent(d, id, by = null) { d.store.run('UPDATE agents SET status=\'revoked\', revoked_at=? WHERE id=?', nowIso(), id); d.vault.remove(`service:agent:${id}`); d.store.run('UPDATE agent_jobs SET status=\'cancelled\', ended_at=? WHERE agent_id=? AND status IN (\'queued\',\'running\')', nowIso(), id); audit(d.store, { actorType: by ? 'operator' : 'system', actorId: by, action: 'agent.revoke', subjectType: 'agent', subjectId: id, immutable: true }); } +/** 외부 Agent 상태(포털 화면용, §8.1) */ +export function externalAgentStatus(d, id) { + const a = getAgent(d, id); if (!a || a.kind !== 'external') return null; + const seen = a.last_seen_at ? Date.now() - Date.parse(a.last_seen_at) : Infinity; + const last = d.store.get('SELECT * FROM agent_jobs WHERE agent_id=? AND kind=\'site.deploy\' ORDER BY created_at DESC LIMIT 1', id); + const lastCheck = d.store.get('SELECT * FROM agent_jobs WHERE agent_id=? AND kind=\'site.check\' AND status=\'done\' ORDER BY created_at DESC LIMIT 1', id); + return { id: a.id, status: a.status === 'revoked' ? 'revoked' : !a.last_seen_at ? 'pending' : seen > d.config.offlineAfterMs ? 'offline' : 'connected', lastSeenAt: a.last_seen_at, version: a.version, siteId: a.site_id, facts: parseJson(a.facts_json, {}), + lastDeploy: last ? { id: last.id, status: last.status, at: last.ended_at || last.started_at || last.created_at, result: parseJson(last.result_json, null), error: last.error } : null, + lastCheck: lastCheck ? { at: lastCheck.ended_at, result: parseJson(lastCheck.result_json, null) } : null }; +} +/** heartbeat 처리: 지표 저장, 서버 last_seen·시계 차이·모듈 실제 버전·용량 갱신 */ +export function heartbeat(d, agent, body) { + const now = nowIso(); const m = body.metrics || {}; const skew = Number.isFinite(body.clockSkewMs) ? Math.round(body.clockSkewMs) : null; + d.store.run('UPDATE agents SET last_seen_at=?, version=?, facts_json=COALESCE(?, facts_json) WHERE id=?', now, body.version || agent.version, body.facts ? toJson(redact(body.facts)) : null, agent.id); + if (!agent.server_id) return { ok: true }; + const capacity = { diskFreeBytes: m.disk ? m.disk.total - m.disk.used : 0, diskTotalBytes: m.disk?.total || 0, memFreeBytes: m.mem ? m.mem.total - m.mem.used : 0, cpuLoad: m.load1 || 0, sitesHosted: body.sitesHosted || 0, reservedBytes: body.reservedBytes || 0, exhaustEta: body.exhaustEta || null }; + d.store.run('UPDATE servers SET last_seen_at=?, clock_skew_ms=?, agent_version=?, metrics_json=?, capacity_json=?, facts_json=COALESCE(?, facts_json), status=CASE WHEN status=\'offline\' THEN \'ready\' ELSE status END, updated_at=? WHERE id=?', now, skew, body.version || null, toJson(m), toJson(capacity), body.facts ? toJson(redact(body.facts)) : null, now, agent.server_id); + d.store.run('INSERT OR REPLACE INTO metrics_samples(server_id, at, cpu, mem_used, mem_total, disk_used, disk_total, inode_used, inode_total, io_read, io_write, net_in, net_out, clock_skew_ms, load1) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)', + agent.server_id, now, m.cpu ?? null, m.mem?.used ?? null, m.mem?.total ?? null, m.disk?.used ?? null, m.disk?.total ?? null, m.inode?.used ?? null, m.inode?.total ?? null, m.io?.read ?? null, m.io?.write ?? null, m.net?.in ?? null, m.net?.out ?? null, skew, m.load1 ?? null); + for (const [name, mod] of Object.entries(body.modules || {})) { + d.store.run('INSERT INTO module_targets(module, server_id, actual_version, health, active, last_check_at, updated_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(module, server_id) DO UPDATE SET actual_version=excluded.actual_version, health=excluded.health, active=excluded.active, last_check_at=excluded.last_check_at, updated_at=excluded.updated_at', + name, agent.server_id, mod.version || null, mod.health || 'unknown', mod.active ? 1 : 0, now, now); + } + // 시계 차이·디스크 고갈 경고(§28.1) + if (skew != null && Math.abs(skew) > 60000) notify(d.store, { category: 'ops', severity: 'warning', incidentKey: `server:${agent.server_id}:clock`, title: `서버 시계 차이 ${Math.round(skew / 1000)}초`, link: `/ops/servers/${agent.server_id}` }); + else { try { const { resolveIncident } = d.notifyLib; resolveIncident(d.store, `server:${agent.server_id}:clock`); } catch (_) {} } + if (m.disk && m.disk.total && m.disk.used / m.disk.total >= 0.9) notify(d.store, { category: 'ops', severity: m.disk.used / m.disk.total >= 0.95 ? 'critical' : 'warning', incidentKey: `server:${agent.server_id}:disk`, title: `서버 디스크 ${Math.round(m.disk.used / m.disk.total * 100)}% 사용`, link: `/ops/servers/${agent.server_id}` }); + return { ok: true, serverTime: now, pendingJobs: d.store.get('SELECT COUNT(*) AS n FROM agent_jobs WHERE agent_id=? AND status=\'queued\'', agent.id).n }; +} +/** 오래 heartbeat 가 없는 서버를 offline 으로 표시(주기 작업). 변경된 서버 목록 반환 */ +export function markOffline(d) { + const cut = new Date(Date.now() - d.config.offlineAfterMs).toISOString(); + const rows = d.store.all('SELECT id, name FROM servers WHERE status IN (\'ready\',\'maintenance\') AND agent_id IS NOT NULL AND (last_seen_at IS NULL OR last_seen_at=? AND at<=? ORDER BY at DESC LIMIT ?', serverId, since || new Date(Date.now() - 90 * 86400000).toISOString(), until || nowIso(), Math.min(limit, 50000)); } + +// ------------------------------------------------------------------ Agent 작업 +/** + * 재진입 가능한 배정: 'dispatch' 단계에 저장된 Agent 작업이 실패·취소로 끝났으면(재시도) 새 Agent 작업을 만든다. + * 호출자는 ctx.step('dispatch', …) 대신 이 함수를 쓴다. + */ +export async function ensureAgentJob(d, ctx, job, { stepId = 'dispatch', agentId, kind, payload, title = '대상 Agent 에 작업 배정' }) { + const prev = ctx.steps().find((x) => x.id === stepId); + if (prev && prev.state === 'done' && prev.result?.agentJobId) { const aj = getAgentJob(d, prev.result.agentJobId); if (aj && ['failed', 'cancelled'].includes(aj.status)) { prev.state = 'pending'; delete prev.result; } } + return ctx.step(stepId, () => ({ agentId, agentJobId: dispatchAgentJob(d, { halJobId: job.id, agentId, kind, payload, dedupeKey: `${job.id}:${kind}:${agentId}:${job.attempt}` }) }), title); +} +/** Agent 작업 만들기(HAL 작업의 자식). 같은 hal_job_id+kind+agent 의 미완료 작업이 있으면 재사용(재진입 멱등). */ +export function dispatchAgentJob(d, { halJobId, agentId, kind, payload, dedupeKey = null }) { + const agent = getAgent(d, agentId); if (!agent || agent.status !== 'active') throw new JobError('agent_unavailable', 'Agent 가 없거나 해지되었습니다'); + if (agent.kind === 'external' && !EXTERNAL_KINDS.has(kind)) throw new JobError('kind_not_allowed_external', `외부 Agent 는 ${kind} 작업을 받을 수 없습니다`); + if (agent.kind === 'external' && payload?.siteId && agent.site_id && payload.siteId !== agent.site_id) throw new JobError('site_mismatch', '이 외부 Agent 는 다른 사이트에 귀속되어 있습니다'); + const key = dedupeKey || `${halJobId}:${kind}:${agentId}`; + const ex = d.store.get('SELECT * FROM agent_jobs WHERE hal_job_id=? AND kind=? AND agent_id=? AND json_extract(payload_json, \'$._dedupe\')=?', halJobId, kind, agentId, key); + if (ex) return ex.id; + const id = newId('aj_'); + d.store.run('INSERT INTO agent_jobs(id, hal_job_id, agent_id, kind, payload_json, status, created_at, steps_json, log_json) VALUES(?,?,?,?,?,\'queued\',?,\'[]\',\'[]\')', id, halJobId, agentId, kind, toJson({ ...payload, _dedupe: key }), nowIso()); + return id; +} +export function getAgentJob(d, id) { const r = d.store.get('SELECT * FROM agent_jobs WHERE id=?', id); return r ? { ...r, payload: parseJson(r.payload_json, {}), steps: parseJson(r.steps_json, []), result: parseJson(r.result_json, null), log: parseJson(r.log_json, []) } : null; } +/** Agent 가 다음 작업을 가져간다(long-poll 은 라우트가 반복 호출). queued 중 가장 오래된 것 하나 → running */ +export function nextAgentJob(d, agentId) { + const r = d.store.get('SELECT id FROM agent_jobs WHERE agent_id=? AND status=\'queued\' ORDER BY created_at LIMIT 1', agentId); if (!r) return null; + const got = d.store.run('UPDATE agent_jobs SET status=\'running\', assigned_at=?, started_at=?, heartbeat_at=? WHERE id=? AND status=\'queued\'', nowIso(), nowIso(), nowIso(), r.id).changes; + if (!got) return null; const j = getAgentJob(d, r.id); delete j.payload._dedupe; return { id: j.id, kind: j.kind, payload: j.payload, halJobId: j.hal_job_id }; +} +/** Agent 재시작 뒤 이어서 할 작업 목록(running 으로 남은 것) */ +export function unfinishedAgentJobs(d, agentId) { return d.store.all('SELECT id, kind, payload_json, steps_json FROM agent_jobs WHERE agent_id=? AND status=\'running\'', agentId).map((r) => ({ id: r.id, kind: r.kind, payload: (() => { const p = parseJson(r.payload_json, {}); delete p._dedupe; return p; })(), steps: parseJson(r.steps_json, []) })); } +export function agentJobProgress(d, id, agentId, { progress, steps, log }) { const r = d.store.run('UPDATE agent_jobs SET progress=COALESCE(?, progress), steps_json=COALESCE(?, steps_json), log_json=COALESCE(?, log_json), heartbeat_at=? WHERE id=? AND agent_id=? AND status=\'running\'', progress ?? null, steps ? toJson(redact(steps)) : null, log ? toJson(redact(log)) : null, nowIso(), id, agentId).changes; return !!r; } +export function agentJobResult(d, id, agentId, { status, result, error, errorCode, userMessage, steps, log }) { + if (!['done', 'failed', 'needs_user'].includes(status)) throw new JobError('bad_status', 'Agent 결과 상태는 done/failed/needs_user 중 하나입니다'); + const ch = d.store.run('UPDATE agent_jobs SET status=?, result_json=?, error=?, error_code=?, user_message=?, steps_json=COALESCE(?, steps_json), log_json=COALESCE(?, log_json), progress=CASE WHEN ?=\'done\' THEN 100 ELSE progress END, ended_at=? WHERE id=? AND agent_id=? AND status=\'running\'', + status, toJson(redact(result ?? null)), error ? safeError({ message: error }) : null, errorCode || null, userMessage || null, steps ? toJson(redact(steps)) : null, log ? toJson(redact(log)) : null, status, nowIso(), id, agentId).changes; + return !!ch; +} +export const isAgentJobCancelled = (d, id) => !!d.store.get('SELECT cancel_requested FROM agent_jobs WHERE id=?', id)?.cancel_requested; +/** + * HAL 작업(lib/jobs run) 안에서 Agent 작업이 끝날 때까지 기다린다. 진행률·단계를 HAL 작업에 비춘다. + * Agent 가 오래 응답하지 않으면(heartbeat 5분) 계속 기다리되 사유를 기록하고, 서버가 offline 이면 needs_user 로 멈춘다(운영자가 서버를 살리고 '이어서 진행'). + */ +export async function awaitAgentJob(d, ctx, agentJobId, { timeoutMs = d.config.agentWaitTimeoutMs || 6 * 3600 * 1000, pollMs = d.config.agentPollMs || 1000 } = {}) { + const t0 = Date.now(); let lastProgress = -1; + while (true) { + const j = getAgentJob(d, agentJobId); if (!j) throw new JobError('agent_job_missing', 'Agent 작업 기록이 사라졌습니다'); + if (j.status === 'done') return j.result; + if (j.status === 'failed') throw new JobError(j.error_code || 'agent_failed', j.error || 'Agent 작업 실패', { retryable: j.error_code === 'transient' }); + if (j.status === 'needs_user') ctx.needsUser(j.user_message || 'Agent 가 사용자/운영자 확인을 요청합니다', [{ id: 'resume', label: '확인 후 이어서 진행' }]); + if (j.status === 'cancelled') throw new JobError('agent_job_cancelled', 'Agent 작업이 취소되었습니다'); + try { ctx.checkCancel(); } catch (e) { d.store.run('UPDATE agent_jobs SET cancel_requested=1 WHERE id=?', agentJobId); throw e; } + if (j.progress !== lastProgress) { lastProgress = j.progress; ctx.progress(j.progress); } + const agent = getAgent(d, j.agent_id); const srv = agent?.server_id ? getServer(d, agent.server_id) : null; + const silent = agent?.last_seen_at ? Date.now() - Date.parse(agent.last_seen_at) : Infinity; + if (silent > 10 * 60 * 1000) ctx.needsUser(`대상 서버(${srv?.name || agent?.label || j.agent_id})의 Agent 가 10분 넘게 응답하지 않습니다. 서버 상태를 확인한 뒤 이어서 진행하세요`, [{ id: 'resume', label: 'Agent 복구 후 이어서 진행' }]); + if (Date.now() - t0 > timeoutMs) throw new JobError('agent_timeout', 'Agent 작업이 제한 시간을 넘겼습니다'); + await new Promise((r) => setTimeout(r, pollMs)); + } +} + +// ------------------------------------------------------------------ 릴리스(§32.1) +export function registerRelease(d, { version, mainCommit, builtAt, artifacts, checks = null, notes = null, approve = false, by = null }) { + const now = nowIso(); const ex = d.store.get('SELECT version FROM releases WHERE version=?', version); + if (ex) d.store.run('UPDATE releases SET main_commit=?, built_at=?, artifacts_json=?, checks_json=?, notes=COALESCE(?, notes) WHERE version=?', mainCommit, builtAt, toJson(artifacts), toJson(checks), notes, version); + else d.store.run('INSERT INTO releases(version, main_commit, built_at, artifacts_json, checks_json, status, notes, registered_at) VALUES(?,?,?,?,?,\'registered\',?,?)', version, mainCommit, builtAt, toJson(artifacts), toJson(checks), notes, now); + if (approve) approveRelease(d, version, by); + audit(d.store, { actorType: by ? 'operator' : 'system', actorId: by, action: 'release.register', subjectType: 'release', subjectId: version, detail: { mainCommit, artifacts: Object.keys(artifacts || {}) } }); + return getRelease(d, version); +} +export function approveRelease(d, version, by) { const r = getRelease(d, version); if (!r) throw new JobError('release_not_found', '릴리스가 없습니다'); const checks = r.checks || {}; if (checks.testsPassed === false) throw new JobError('release_checks_failed', '검사에 실패한 릴리스는 승인할 수 없습니다'); d.store.run('UPDATE releases SET status=\'approved\', approved_by=?, approved_at=? WHERE version=?', by, nowIso(), version); audit(d.store, { actorType: 'operator', actorId: by, action: 'release.approve', subjectType: 'release', subjectId: version, immutable: true }); return getRelease(d, version); } +export function getRelease(d, version) { const r = d.store.get('SELECT * FROM releases WHERE version=?', version); return r ? { ...r, artifacts: parseJson(r.artifacts_json, {}), checks: parseJson(r.checks_json, null) } : null; } +export function listReleases(d) { return d.store.all('SELECT * FROM releases ORDER BY registered_at DESC').map((r) => ({ ...r, artifacts: parseJson(r.artifacts_json, {}), checks: parseJson(r.checks_json, null) })); } +export const artifactDir = (d) => join(d.config.dataDir, 'artifacts'); +/** artifact 파일 경로·존재 확인 */ +export function artifactPath(d, version, module) { const r = getRelease(d, version); const a = r && r.artifacts[module]; if (!a) return null; const p = join(artifactDir(d), version, a.file); return existsSync(p) ? { path: p, sha256: a.sha256, size: statSync(p).size } : null; } + +// ------------------------------------------------------------------ 전송(transfer) +export function transferDir(d) { const p = join(d.config.dataDir, 'transfers'); mkdirSync(p, { recursive: true, mode: 0o700 }); return p; } +export function createTransfer(d, { id = newId('tr_'), agentId, sha256: sum, size, purpose = '', meta = null }) { const path = join(transferDir(d), id + '.bin'); d.store.run('INSERT OR REPLACE INTO transfers(id, created_by_agent, path, size_bytes, sha256, created_at, expires_at, purpose, meta_json) VALUES(?,?,?,?,?,?,?,?,?)', id, agentId, path, size, sum, nowIso(), new Date(Date.now() + d.config.transferTtlMs).toISOString(), purpose, toJson(meta)); return { id, path }; } +export function getTransfer(d, id) { const t = d.store.get('SELECT * FROM transfers WHERE id=? AND deleted_at IS NULL', id); return t && existsSync(t.path) ? t : null; } + +// ------------------------------------------------------------------ 포털 이벤트 outbox(§6) +export function queuePortalEvent(d, payload) { d.store.run('INSERT INTO portal_events_outbox(id, payload_json, created_at, next_at) VALUES(?,?,?,?)', newId('ev_'), toJson(redact(payload)), nowIso(), nowIso()); } +/** outbox 전송(주기). 실패는 지수 백오프로 재시도하고 치명적으로 다루지 않는다. */ +export async function flushPortalEvents(d, { fetchImpl = fetch } = {}) { + const secret = d.vault.get('service:hal'); if (!secret) return 0; + const rows = d.store.all('SELECT * FROM portal_events_outbox WHERE delivered_at IS NULL AND next_at<=? ORDER BY created_at LIMIT 50', nowIso()); let n = 0; + for (const r of rows) { + try { const res = await serviceFetch(d.config.portalUrl, 'hal', secret, 'POST', '/internal/hal-events', parseJson(r.payload_json, {}), { timeoutMs: 8000, fetchImpl }); if (!res.ok) throw new Error(`portal ${res.status} ${res.json?.error || ''}`); d.store.run('UPDATE portal_events_outbox SET delivered_at=? WHERE id=?', nowIso(), r.id); n++; } + catch (e) { const attempts = r.attempts + 1; d.store.run('UPDATE portal_events_outbox SET attempts=?, next_at=?, last_error=? WHERE id=?', attempts, new Date(Date.now() + Math.min(600000, 5000 * 2 ** attempts)).toISOString(), safeError(e), r.id); } + } + d.store.run('DELETE FROM portal_events_outbox WHERE delivered_at IS NOT NULL AND delivered_at 0; } +export { hashPassword }; diff --git a/hal/deps.mjs b/hal/deps.mjs new file mode 100644 index 0000000..971d37a --- /dev/null +++ b/hal/deps.mjs @@ -0,0 +1,24 @@ +/** + * hal/deps.mjs — HAL 의존성 묶음(deps) 만들기. index.mjs(운영)와 테스트가 같은 함수를 쓴다. + * deps = { store, vault, config, jobs, log, version, instanceId, serviceauth, notifyLib, fetch, emergencyRunner, jobsRunning, stopping } + */ +import { openDb } from '../lib/db.mjs'; +import { openVault } from '../lib/vault.mjs'; +import { createJobEngine } from '../lib/jobs.mjs'; +import * as serviceauth from '../lib/serviceauth.mjs'; +import * as notifyLib from '../lib/notify.mjs'; +import { newId } from '../lib/crypto.mjs'; +import { MIGRATIONS } from './schema.mjs'; +import { registerKinds } from './jobs.mjs'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export const VERSION = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version; +export function createHalDeps({ config, dbFile = null, log = () => {}, fetchImpl = null, version = VERSION, concurrency = 64 } = {}) { + const store = openDb(dbFile || join(config.dataDir, 'hal.sqlite'), MIGRATIONS); + const vault = openVault(config.dataDir); + const d = { store, vault, config, log, version, instanceId: newId('hal_'), serviceauth, notifyLib, fetch: fetchImpl, emergencyRunner: null, jobsRunning: false, stopping: false }; + d.jobs = createJobEngine(store, { log, workerId: `hal@${process.pid}`, concurrency, pollMs: 500 }); + registerKinds(d.jobs, d); + return d; +} diff --git a/hal/dr.mjs b/hal/dr.mjs new file mode 100644 index 0000000..554212d --- /dev/null +++ b/hal/dr.mjs @@ -0,0 +1,90 @@ +/** + * hal/dr.mjs — 플랫폼 재해복구(§34). + * + * 복구 대상: HAL 운영 정보(hal.sqlite: 서버·Agent·릴리스·작업·모듈 배치·운영자 계정·접근 설정), 진행 중 작업, 보관소(vault.json; 키 파일은 **포함하지 않는다** — 운영자가 따로 보관), config.json, artifact 목록. + * 메일 원문·고객 사이트 데이터는 대상이 아니다(고객용 백업·메일 정책 별도). + * - 설정(dr_config): RTO·RPO·복구 사본 주기·보존 개수·외부 보관 위치. 출시 검증에서 비어 있으면 안 된다(readiness). + * - 복구 사본(snapshot): `VACUUM INTO` 로 일관된 DB 사본 + vault.json + config.json 을 tar.gz 로 묶고 sha256·읽기 검증. 외부 보관 위치로 복사(지정 시). + * - 복원(restore): 별도 승인 + "기존 제어 위치 중지" 확인이 있어야 시작. 실제 파일 복원은 deploy/hal-restore.sh 가 하고, 새 HAL 기동 때 claimLive 가 "하나의 HAL" 을 보장한다. + * 복원 뒤 checks(운영자 로그인·HAL 제어·HTTPS·고객 서비스 1개 외부 확인)와 실제 상태 비교(diff) 를 기록한다. + * - 훈련(drill): 실제 측정 RTO/RPO 와 단계 결과. 마지막 훈련이 성공해야 '정식 운영 준비 완료'. + */ +import { newId } from '../lib/crypto.mjs'; +import { nowIso, toJson, parseJson } from '../lib/db.mjs'; +import { audit } from '../lib/audit.mjs'; +import { JobError } from '../lib/jobs.mjs'; +import { createHash } from 'node:crypto'; +import { mkdirSync, existsSync, readFileSync, writeFileSync, statSync, copyFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { execFileSync } from 'node:child_process'; +import * as core from './core.mjs'; + +export function getConfig(d) { return d.store.get('SELECT * FROM dr_config WHERE id=1') || { rto_minutes: null, rpo_minutes: null, snapshot_interval_minutes: null, retention_count: null, external_store: null }; } +export function setConfig(d, patch, by) { + const c = { ...getConfig(d), ...Object.fromEntries(Object.entries(patch).filter(([k]) => ['rto_minutes', 'rpo_minutes', 'snapshot_interval_minutes', 'retention_count', 'external_store'].includes(k))) }; + for (const k of ['rto_minutes', 'rpo_minutes', 'snapshot_interval_minutes', 'retention_count']) if (c[k] != null && !(Number(c[k]) > 0)) throw new JobError('bad_value', `${k} 은(는) 0보다 커야 합니다`); + d.store.run('INSERT INTO dr_config(id, rto_minutes, rpo_minutes, snapshot_interval_minutes, retention_count, external_store, updated_at, updated_by) VALUES(1,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET rto_minutes=excluded.rto_minutes, rpo_minutes=excluded.rpo_minutes, snapshot_interval_minutes=excluded.snapshot_interval_minutes, retention_count=excluded.retention_count, external_store=excluded.external_store, updated_at=excluded.updated_at, updated_by=excluded.updated_by', c.rto_minutes, c.rpo_minutes, c.snapshot_interval_minutes, c.retention_count, c.external_store, nowIso(), by); + audit(d.store, { actorType: 'operator', actorId: by, action: 'dr.config', detail: c }); return getConfig(d); +} +export const snapshotDir = (d) => { const p = join(d.config.dataDir, 'dr'); mkdirSync(p, { recursive: true, mode: 0o700 }); return p; }; +const sha = (f) => createHash('sha256').update(readFileSync(f)).digest('hex'); +/** 복구 사본 만들기(동기, 수 초). DB 는 VACUUM INTO 로 일관 사본. */ +export function snapshot(d, { kind = 'scheduled', by = null } = {}) { + const id = newId('drs_'); const dir = snapshotDir(d); const work = join(dir, id); mkdirSync(work, { recursive: true }); + try { + const dbCopy = join(work, 'hal.sqlite'); d.store.exec(`VACUUM INTO '${dbCopy.replace(/'/g, "''")}'`); + for (const f of ['vault.json', 'config.json']) { const p = join(d.config.dataDir, f); if (existsSync(p)) copyFileSync(p, join(work, f)); } + const rel = core.listReleases(d).map((r) => ({ version: r.version, main_commit: r.main_commit, artifacts: r.artifacts, status: r.status })); const meta = { id, createdAt: nowIso(), halVersion: d.version, instanceId: d.instanceId, releases: rel, note: 'vault.key 는 포함하지 않음 — 운영자가 따로 보관' }; + writeFileSync(join(work, 'meta.json'), JSON.stringify(meta, null, 1)); + const tar = join(dir, `${id}.tar.gz`); execFileSync('tar', ['-czf', tar, '-C', work, '.']); rmSync(work, { recursive: true, force: true }); + const sum = sha(tar); const size = statSync(tar).size; + // 읽기 검증: 목록을 읽을 수 있고 meta.json 이 들어 있어야 한다 + const list = execFileSync('tar', ['-tzf', tar]).toString(); const ok = list.includes('meta.json') && list.includes('hal.sqlite'); + d.store.run('INSERT INTO dr_snapshots(id, created_at, location, size_bytes, sha256, verify_state, verified_at, kind) VALUES(?,?,?,?,?,?,?,?)', id, nowIso(), tar, size, sum, ok ? 'ok' : 'failed', nowIso(), kind); + const cfg = getConfig(d); + if (ok && cfg.external_store) { try { mkdirSync(cfg.external_store, { recursive: true }); const ext = join(cfg.external_store, `${id}.tar.gz`); copyFileSync(tar, ext); if (sha(ext) !== sum) throw new Error('외부 사본 해시 불일치'); d.store.run('UPDATE dr_snapshots SET external_copied_at=?, external_location=? WHERE id=?', nowIso(), ext, id); } catch (e) { d.store.run('UPDATE dr_snapshots SET error=? WHERE id=?', `외부 보관 실패: ${e.message}`, id); } } + prune(d); audit(d.store, { actorType: by ? 'operator' : 'system', actorId: by, action: 'dr.snapshot', subjectType: 'dr_snapshot', subjectId: id, detail: { size, ok, kind }, automatic: !by }); + return d.store.get('SELECT * FROM dr_snapshots WHERE id=?', id); + } catch (e) { rmSync(work, { recursive: true, force: true }); d.store.run('INSERT INTO dr_snapshots(id, created_at, location, verify_state, kind, error) VALUES(?,?,?,\'failed\',?,?)', id, nowIso(), '', kind, e.message); throw e; } +} +/** 보존 개수 초과 사본 삭제(외부 사본은 유지) */ +export function prune(d) { const n = getConfig(d).retention_count || 14; const rows = d.store.all('SELECT id, location FROM dr_snapshots WHERE deleted_at IS NULL AND verify_state=\'ok\' ORDER BY created_at DESC, rowid DESC'); let k = 0; for (const r of rows.slice(n)) { try { rmSync(r.location, { force: true }); } catch (_) {} d.store.run('UPDATE dr_snapshots SET deleted_at=? WHERE id=?', nowIso(), r.id); k++; } return k; } +export function listSnapshots(d) { return d.store.all('SELECT * FROM dr_snapshots ORDER BY created_at DESC, rowid DESC LIMIT 100'); } +/** 복원 요청·승인·시작. 시작에는 "기존 제어 위치 중지" 확인이 필요하다. */ +export function requestRestore(d, { snapshotId, by, note = '' }) { const s = d.store.get('SELECT * FROM dr_snapshots WHERE id=? AND deleted_at IS NULL', snapshotId); if (!s || s.verify_state !== 'ok') throw new JobError('snapshot_unusable', '검증된 복구 사본이 아닙니다'); const id = newId('drr_'); d.store.run('INSERT INTO dr_restores(id, snapshot_id, requested_by, status, note) VALUES(?,?,?,\'requested\',?)', id, snapshotId, by, note); audit(d.store, { actorType: 'operator', actorId: by, action: 'recovery.restore_request', subjectType: 'dr_restore', subjectId: id, immutable: true }); return d.store.get('SELECT * FROM dr_restores WHERE id=?', id); } +export function approveRestore(d, id, by) { d.store.run('UPDATE dr_restores SET approved_by=?, approved_at=?, status=\'approved\' WHERE id=? AND status=\'requested\'', by, nowIso(), id); audit(d.store, { actorType: 'operator', actorId: by, action: 'recovery.restore_approve', subjectType: 'dr_restore', subjectId: id, immutable: true }); return d.store.get('SELECT * FROM dr_restores WHERE id=?', id); } +export function startRestore(d, id, { by, oldControlStopped, confirmText }) { + const r = d.store.get('SELECT * FROM dr_restores WHERE id=?', id); if (!r || r.status !== 'approved') throw new JobError('restore_not_approved', '복원은 별도 승인 뒤에만 시작할 수 있습니다'); + if (oldControlStopped !== true || confirmText !== '기존 제어 위치를 중지했습니다') throw new JobError('old_control_not_confirmed', '기존 제어 위치(HAL)의 중지 확인이 필요합니다'); + const s = d.store.get('SELECT * FROM dr_snapshots WHERE id=?', r.snapshot_id); + d.store.run('UPDATE dr_restores SET old_control_stopped_confirmed_at=?, started_at=?, status=\'started\' WHERE id=?', nowIso(), nowIso(), id); + audit(d.store, { actorType: 'operator', actorId: by, action: 'recovery.restore_start', subjectType: 'dr_restore', subjectId: id, immutable: true }); + return { restore: d.store.get('SELECT * FROM dr_restores WHERE id=?', id), command: `sudo bash /srv/ariweb/hal/current/deploy/hal-restore.sh ${s.location} ${s.sha256}`, note: '새 HAL 기동 때 다른 인스턴스의 최근 임대가 있으면 기동을 거부합니다(하나의 HAL 만 운영)' }; +} +/** 복원 뒤 확인·차이 기록(§34: 운영자 로그인·HAL 제어·HTTPS·고객 서비스 1개 외부 확인 + 실제 상태 비교) */ +export function finishRestore(d, id, { by, checks }) { + const req = ['operatorLogin', 'halControl', 'https', 'customerService']; const missing = req.filter((k) => !checks || checks[k] == null); + if (missing.length) throw new JobError('checks_incomplete', `필수 확인 항목이 비었습니다: ${missing.join(', ')}`); + const dup = d.store.get('SELECT COUNT(*) AS n FROM jobs WHERE status=\'running\'').n; const drift = d.store.all('SELECT module, server_id, target_version, actual_version FROM module_targets WHERE target_version IS NOT NULL AND (actual_version IS NULL OR actual_version<>target_version)'); + const ok = req.every((k) => checks[k] === true) && dup === 0; const diff = { runningJobsAtFinish: dup, drift, offline: core.markOffline(d).map((x) => x.id) }; + d.store.run('UPDATE dr_restores SET finished_at=?, status=?, checks_json=?, diff_json=? WHERE id=?', nowIso(), ok ? 'completed' : 'completed_with_findings', toJson(checks), toJson(diff), id); + audit(d.store, { actorType: 'operator', actorId: by, action: 'recovery.restore_finish', subjectType: 'dr_restore', subjectId: id, detail: { ok, diff }, immutable: true }); + return d.store.get('SELECT * FROM dr_restores WHERE id=?', id); +} +/** 훈련 기록 */ +export function startDrill(d, { by, snapshotId = null }) { const id = newId('drd_'); d.store.run('INSERT INTO dr_drills(id, started_at, snapshot_id, performed_by, steps_json, result) VALUES(?,?,?,?,\'[]\',\'running\')', id, nowIso(), snapshotId, by); return d.store.get('SELECT * FROM dr_drills WHERE id=?', id); } +export function finishDrill(d, id, { by, steps, measuredRtoMinutes, measuredRpoMinutes, findings = [] }) { + const req = ['hal', 'operatorLogin', 'vault', 'certBase', 'httpsCustomer']; const ok = req.every((k) => steps && steps[k] === true); const cfg = getConfig(d); + const within = cfg.rto_minutes && cfg.rpo_minutes ? measuredRtoMinutes <= cfg.rto_minutes && measuredRpoMinutes <= cfg.rpo_minutes : false; + const result = ok && within ? 'success' : 'failed'; const next = new Date(Date.now() + 90 * 86400000).toISOString(); + d.store.run('UPDATE dr_drills SET finished_at=?, steps_json=?, measured_rto_minutes=?, measured_rpo_minutes=?, result=?, findings_json=?, next_due_at=? WHERE id=?', nowIso(), toJson(steps), measuredRtoMinutes, measuredRpoMinutes, result, toJson([...(findings || []), ...(within ? [] : ['측정값이 승인된 RTO/RPO 를 넘었거나 목표가 설정되지 않음'])]), next, id); + audit(d.store, { actorType: 'operator', actorId: by, action: 'recovery.drill', subjectType: 'dr_drill', subjectId: id, detail: { result, measuredRtoMinutes, measuredRpoMinutes }, immutable: true }); + return d.store.get('SELECT * FROM dr_drills WHERE id=?', id); +} +/** 정식 운영 준비 상태(§34): 설정 값이 모두 있고 마지막 훈련이 성공 */ +export function readiness(d) { + const cfg = getConfig(d); const missing = ['rto_minutes', 'rpo_minutes', 'snapshot_interval_minutes', 'retention_count'].filter((k) => !(Number(cfg[k]) > 0)); + const lastDrill = d.store.get('SELECT * FROM dr_drills WHERE result<>\'running\' ORDER BY started_at DESC, rowid DESC LIMIT 1'); const lastSnap = d.store.get('SELECT * FROM dr_snapshots WHERE verify_state=\'ok\' AND deleted_at IS NULL ORDER BY created_at DESC, rowid DESC LIMIT 1'); + const ready = !missing.length && !!lastDrill && lastDrill.result === 'success' && !!lastSnap; + return { ready, missingConfig: missing, lastDrill: lastDrill ? { id: lastDrill.id, result: lastDrill.result, at: lastDrill.finished_at, measuredRtoMinutes: lastDrill.measured_rto_minutes, measuredRpoMinutes: lastDrill.measured_rpo_minutes, nextDueAt: lastDrill.next_due_at } : null, lastSnapshot: lastSnap ? { id: lastSnap.id, at: lastSnap.created_at, external: !!lastSnap.external_copied_at } : null, config: cfg, label: ready ? '정식 운영 준비 완료' : '정식 운영 준비 전(복구 훈련 또는 설정 필요)' }; +} diff --git a/hal/emergency.mjs b/hal/emergency.mjs new file mode 100644 index 0000000..7774a27 --- /dev/null +++ b/hal/emergency.mjs @@ -0,0 +1,72 @@ +/** + * hal/emergency.mjs — 긴급 복구(§33). + * + * HAL 의 정상 작업 기능을 쓸 수 없는 실제 장애에만 허용한다: 선언(incident key + 사유) 때 HAL 자체 건강이 정상이고 열린 critical 사건이 없으면 거부한다. + * 단계: declared → approved(서로 다른 내용을 확인하는 3단계: 1 영향 범위, 2 대상 식별정보 재입력, 3 최종 실행·되돌리기 가능 여부) → credential_issued(짧은 유효 시간, 1회 표시) → + * performing(허용 범위 안의 행동만: hal.restart / hal.restore_snapshot / vault.verify / cert.reissue_internal / artifact.restore / agent.reconnect) → verified → credential_revoked → reconciled → closed. + * 운영자가 한 명이어도 되지만 세 단계의 내용과 시각이 모두 기록되어야 한다. 일반 고객 서비스 배포(module.deploy 등)는 어떤 경우에도 거부한다. + */ +import { newId, randomToken, sha256, safeEqual } from '../lib/crypto.mjs'; +import { nowIso, toJson, parseJson } from '../lib/db.mjs'; +import { audit } from '../lib/audit.mjs'; +import { JobError } from '../lib/jobs.mjs'; +import * as core from './core.mjs'; + +export const SCOPES = ['hal', 'hal_data', 'vault', 'cert_base', 'artifact_store', 'first_agent']; +export const ACTIONS = { 'hal.restart': 'hal', 'hal.restore_snapshot': 'hal_data', 'vault.verify': 'vault', 'cert.reissue_internal': 'cert_base', 'artifact.restore': 'artifact_store', 'agent.reconnect': 'first_agent' }; +const STEP_TITLES = ['영향 범위 확인', '변경되지 않는 대상 식별정보 재확인', '최종 실행과 되돌리기 가능 여부 확인']; + +function get(d, id) { const r = d.store.get('SELECT * FROM emergency_sessions WHERE id=?', id); if (!r) throw new JobError('emergency_not_found', '긴급 복구 세션이 없습니다'); return { ...r, scope: parseJson(r.scope_json, []), approvals: parseJson(r.approvals_json, []), actions: parseJson(r.actions_json, []), verify: parseJson(r.verify_json, null), reconcile: parseJson(r.reconcile_json, null) }; } +export const getSession = get; +export function listSessions(d) { return d.store.all('SELECT * FROM emergency_sessions ORDER BY declared_at DESC LIMIT 50').map((r) => get(d, r.id)); } +export function declare(d, { incidentKey, reason, scope, by, force = false }) { + if (!incidentKey || !reason) throw new JobError('declare_incomplete', '사건 번호와 복구 이유가 필요합니다'); + const bad = (scope || []).filter((s) => !SCOPES.includes(s)); if (bad.length || !scope?.length) throw new JobError('bad_scope', `허용 대상은 ${SCOPES.join(', ')} 입니다`); + const health = core.selfHealth(d); const openCritical = d.store.get('SELECT id FROM incidents WHERE severity=\'critical\' AND resolved_at IS NULL'); + if (health.ok && !openCritical && !force) throw new JobError('hal_healthy', 'HAL 정상 작업 기능을 쓸 수 있는 상태입니다 — 긴급 복구가 아니라 일반 작업을 쓰세요. 실제 장애면 사건을 먼저 등록하세요'); + const id = newId('em_'); const now = nowIso(); + d.store.run('INSERT INTO emergency_sessions(id, incident_key, reason, scope_json, status, declared_by, declared_at, hal_healthy_at_declare, updated_at) VALUES(?,?,?,?,\'declared\',?,?,?,?)', id, incidentKey, reason, toJson(scope), by, now, health.ok ? 1 : 0, now); + audit(d.store, { actorType: 'operator', actorId: by, action: 'emergency.declare', subjectType: 'emergency', subjectId: id, detail: { incidentKey, reason, scope, forced: force && health.ok }, immutable: true }); + return get(d, id); +} +/** 3단계 승인. 각 단계는 서로 다른 입력을 요구한다(같은 문구 반복 불가). */ +export function approve(d, id, { step, by, impactText, identifiers, finalConfirm, revertible }) { + const s = get(d, id); if (!['declared', 'approving'].includes(s.status)) throw new JobError('bad_state', `지금 상태(${s.status})에서는 승인할 수 없습니다`); + const n = s.approvals.length + 1; if (step !== n) throw new JobError('wrong_step', `지금은 ${n}단계(${STEP_TITLES[n - 1]})를 진행해야 합니다`); + let record; + if (n === 1) { if (!impactText || String(impactText).length < 10) throw new JobError('impact_required', '영향 범위를 구체적으로 적어야 합니다(10자 이상)'); record = { step: 1, title: STEP_TITLES[0], impactText, by, at: nowIso() }; } + else if (n === 2) { const need = s.scope.slice().sort(); const got = (identifiers || []).slice().sort(); if (JSON.stringify(need) !== JSON.stringify(got)) throw new JobError('identifiers_mismatch', `대상 식별정보가 선언한 범위(${need.join(', ')})와 다릅니다`); record = { step: 2, title: STEP_TITLES[1], identifiers: got, by, at: nowIso() }; } + else { if (finalConfirm !== `복구 실행 ${s.incident_key}`) throw new JobError('final_confirm_mismatch', `최종 확인 문구는 "복구 실행 ${s.incident_key}" 이어야 합니다`); if (typeof revertible !== 'boolean') throw new JobError('revertible_required', '되돌리기 가능 여부를 명시해야 합니다'); record = { step: 3, title: STEP_TITLES[2], finalConfirm: true, revertible, by, at: nowIso() }; } + const approvals = [...s.approvals, record]; const status = approvals.length === 3 ? 'approved' : 'approving'; + d.store.run('UPDATE emergency_sessions SET approvals_json=?, status=?, updated_at=? WHERE id=?', toJson(approvals), status, nowIso(), id); + audit(d.store, { actorType: 'operator', actorId: by, action: `emergency.approve_${n}`, subjectType: 'emergency', subjectId: id, detail: record, immutable: true }); + return get(d, id); +} +/** 자격 발급(승인 3단계 완료 뒤, 1회 표시, 짧은 유효 시간) */ +export function issueCredential(d, id, by) { + const s = get(d, id); if (s.status !== 'approved' || s.approvals.length < 3) throw new JobError('approvals_insufficient', `승인이 부족합니다(${s.approvals.length}/3)`); + const cred = randomToken(32); const exp = new Date(Date.now() + d.config.emergencyTtlMinutes * 60000).toISOString(); + d.store.run('UPDATE emergency_sessions SET credential_hash=?, credential_issued_at=?, credential_expires_at=?, status=\'credential_issued\', updated_at=? WHERE id=?', sha256(cred), nowIso(), exp, nowIso(), id); + audit(d.store, { actorType: 'operator', actorId: by, action: 'emergency.credential_issue', subjectType: 'emergency', subjectId: id, detail: { expiresAt: exp }, immutable: true }); + return { credential: cred, expiresAt: exp }; +} +function checkCredential(s, credential) { if (!s.credential_hash) throw new JobError('no_credential', '자격이 발급되지 않았습니다'); if (s.credential_revoked_at) throw new JobError('credential_revoked', '자격이 폐기되었습니다'); if (Date.parse(s.credential_expires_at) < Date.now()) throw new JobError('credential_expired', '긴급 자격이 만료되었습니다 — 새로 승인·발급하세요'); if (!safeEqual(sha256(String(credential || '')), s.credential_hash)) throw new JobError('bad_credential', '자격이 올바르지 않습니다'); } +/** 행동 수행. 허용 범위 밖·일반 배포는 거부. 실행기(d.emergencyRunner)가 실제 동작을 하고 결과를 돌려준다(테스트는 가짜). */ +export async function perform(d, id, { credential, action, target = null, by }) { + const s = get(d, id); checkCredential(s, credential); + if (!['credential_issued', 'performing'].includes(s.status)) throw new JobError('bad_state', `지금 상태(${s.status})에서는 수행할 수 없습니다`); + const scopeOf = ACTIONS[action]; + if (!scopeOf) { log(d, id, { action, target, by, result: 'rejected', reason: 'not_an_emergency_action' }); throw new JobError('action_forbidden', `긴급 복구에서 허용되지 않는 행동입니다: ${action} (일반 배포는 HAL 정상화 뒤 HAL 작업으로만)`); } + if (!s.scope.includes(scopeOf)) { log(d, id, { action, target, by, result: 'rejected', reason: 'out_of_scope' }); throw new JobError('out_of_scope', `선언한 허용 대상(${s.scope.join(', ')}) 밖의 행동입니다: ${action}`); } + let result; try { result = d.emergencyRunner ? await d.emergencyRunner(action, target, s) : { ok: true, simulated: true }; } catch (e) { log(d, id, { action, target, by, result: 'failed', error: e.message }); throw e; } + log(d, id, { action, target, by, result: 'ok', detail: result }); d.store.run('UPDATE emergency_sessions SET status=\'performing\', updated_at=? WHERE id=?', nowIso(), id); + return result; +} +function log(d, id, entry) { const s = get(d, id); const actions = [...s.actions, { at: nowIso(), ...entry }]; d.store.run('UPDATE emergency_sessions SET actions_json=?, updated_at=? WHERE id=?', toJson(actions), nowIso(), id); audit(d.store, { actorType: 'operator', actorId: entry.by, action: 'emergency.action', subjectType: 'emergency', subjectId: id, outcome: entry.result === 'ok' ? 'ok' : entry.result === 'rejected' ? 'denied' : 'failed', detail: entry, immutable: true }); } +/** 실제 결과 확인 기록 → 자격 폐기 */ +export function verifyAndRevoke(d, id, { by, verify }) { const s = get(d, id); if (!['performing', 'credential_issued'].includes(s.status)) throw new JobError('bad_state', '수행 단계가 아닙니다'); const health = core.selfHealth(d); const v = { ...(verify || {}), halHealth: health.ok, at: nowIso() }; d.store.run('UPDATE emergency_sessions SET verify_json=?, credential_revoked_at=?, status=\'verified\', updated_at=? WHERE id=?', toJson(v), nowIso(), nowIso(), id); audit(d.store, { actorType: 'operator', actorId: by, action: 'emergency.verify_revoke', subjectType: 'emergency', subjectId: id, detail: v, immutable: true }); return get(d, id); } +/** HAL 정상화 뒤 실제 상태와 원하는 상태 비교(모듈 목표·실제 버전 차이, 오프라인 서버) → 차이 목록 */ +export function reconcile(d, id, by) { const s = get(d, id); if (s.status !== 'verified') throw new JobError('bad_state', '검증·자격 폐기 뒤에 조정할 수 있습니다'); const drift = d.store.all('SELECT module, server_id, target_version, actual_version, health FROM module_targets WHERE (target_version IS NOT NULL AND (actual_version IS NULL OR actual_version<>target_version)) OR health=\'down\''); const offline = core.markOffline(d); const r = { at: nowIso(), drift, offlineServers: offline.map((x) => x.id), halHealth: core.selfHealth(d).ok }; d.store.run('UPDATE emergency_sessions SET reconcile_json=?, status=\'reconciled\', updated_at=? WHERE id=?', toJson(r), nowIso(), id); audit(d.store, { actorType: 'operator', actorId: by, action: 'emergency.reconcile', subjectType: 'emergency', subjectId: id, detail: r, immutable: true }); return r; } +export function close(d, id, by, note = '') { const s = get(d, id); if (!['reconciled', 'verified', 'declared', 'approving', 'approved'].includes(s.status)) throw new JobError('bad_state', '지금 상태에서는 종료할 수 없습니다'); if (s.credential_hash && !s.credential_revoked_at) d.store.run('UPDATE emergency_sessions SET credential_revoked_at=? WHERE id=?', nowIso(), id); d.store.run('UPDATE emergency_sessions SET status=\'closed\', closed_at=?, closed_by=?, updated_at=? WHERE id=?', nowIso(), by, nowIso(), id); audit(d.store, { actorType: 'operator', actorId: by, action: 'emergency.close', subjectType: 'emergency', subjectId: id, detail: { note }, immutable: true }); return get(d, id); } +/** 만료 자격 자동 폐기(주기) */ +export function expireCredentials(d) { return d.store.run('UPDATE emergency_sessions SET credential_revoked_at=?, updated_at=? WHERE credential_hash IS NOT NULL AND credential_revoked_at IS NULL AND credential_expires_at [--hours 8] 허용 네트워크 밖 기기용 접근권 URL 발급(§25) + * node hal/index.mjs register-release --manifest /manifest.json> [--approve] 릴리스 등록(부트스트랩·CI) + * node hal/index.mjs link-portal 포털 연동 서비스 키 생성(service:portal / service:hal) — 포털 보관소에도 같은 값을 넣는다(출력 1회) + * node hal/index.mjs bootstrap-release --version --commit --sha256 부트스트랩 때 선택한 릴리스 정보를 기록 + */ +import { loadConfig } from './config.mjs'; +import { createHalDeps, VERSION } from './deps.mjs'; +import { buildApp } from './app.mjs'; +import * as core from './core.mjs'; +import * as ops from './ops.mjs'; +import * as emergency from './emergency.mjs'; +import * as dr from './dr.mjs'; +import { purgeSessions } from '../lib/session.mjs'; +import { randomToken } from '../lib/crypto.mjs'; +import { writeFileSync, readFileSync, copyFileSync, mkdirSync, existsSync } from 'node:fs'; +import { join, dirname, basename } from 'node:path'; + +const log = (...a) => console.log(new Date().toISOString(), ...a); +const cfg = loadConfig(); const args = process.argv.slice(2); const cmd = args[0] && !args[0].startsWith('--') ? args[0] : 'serve'; +const opt = (name, def = null) => { const i = args.indexOf('--' + name); return i >= 0 ? (args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : true) : def; }; +const d = createHalDeps({ config: cfg, log }); + +if (cmd === 'setup-token') { + if (core.ensureOpsAccountExists(d)) { console.error('운영자 계정이 이미 있습니다 — 설정 토큰은 최초 1회만 쓸 수 있습니다'); process.exit(1); } + const t = randomToken(24); writeFileSync(join(cfg.dataDir, 'setup.token'), t, { mode: 0o600 }); + console.log(`설정 토큰(1회 표시, 공유 금지): ${t}\n운영자 콘솔 ${cfg.publicUrl}/ops/ 의 최초 설정 또는 curl -X POST ${cfg.publicUrl}/ops/api/setup -d '{"setupToken":"…","login":"…","password":"…","displayName":"…"}' 로 첫 운영자 계정을 만드세요`); +} else if (cmd === 'grant') { + const g = ops.issueGrant(d, { accountId: opt('login'), hours: Number(opt('hours', 8)), by: null, reason: opt('reason', 'cli') }); console.log(`접근권 URL(1회, ${g.expiresAt} 까지): ${g.url}`); +} else if (cmd === 'register-release') { + const mf = JSON.parse(readFileSync(opt('manifest'), 'utf8')); const dir = join(core.artifactDir(d), mf.version); mkdirSync(dir, { recursive: true }); + for (const [m, a] of Object.entries(mf.artifacts)) { const src = join(dirname(opt('manifest')), a.file); if (existsSync(src)) copyFileSync(src, join(dir, basename(a.file))); } + const r = core.registerRelease(d, { version: mf.version, mainCommit: mf.mainCommit, builtAt: mf.builtAt, artifacts: Object.fromEntries(Object.entries(mf.artifacts).map(([m, a]) => [m, { file: basename(a.file), sha256: a.sha256, size: a.size }])), checks: mf.checks || null, approve: !!opt('approve'), by: opt('by', 'cli') }); + console.log(`릴리스 ${r.version} 등록(${r.status}) — artifact ${Object.keys(r.artifacts).join(', ')}`); +} else if (cmd === 'link-portal') { + const p = randomToken(32), h = randomToken(32); d.vault.set('service:portal', p, { kind: 'service-key' }); d.vault.set('service:hal', h, { kind: 'service-key' }); + console.log(`포털 보관소에 넣을 값(1회 표시):\n service:hal = ${p} (포털이 HAL 을 부를 때 쓰는 키 — 포털 vault 이름 'service:hal')\n service:portal = ${h} (HAL 이 포털을 부를 때 쓰는 키 — 포털 vault 이름 'service:portal')`); +} else if (cmd === 'bootstrap-release') { + core.setBootstrap(d, { release_version: opt('version'), main_commit: opt('commit'), artifact_sha256: opt('sha256') }); console.log('부트스트랩 릴리스 정보를 기록했습니다'); +} else if (cmd === 'serve') { + // "하나의 HAL" 임대(§34). 다른 인스턴스가 살아 있으면 기동 거부(--takeover 는 재해복구 절차에서만). + try { core.claimLive(d, { instanceId: d.instanceId, version: VERSION, takeover: !!opt('takeover') }); } catch (e) { console.error(`[hal] 기동 거부: ${e.message}`); process.exit(2); } + const app = buildApp(d); d.jobs.start(); d.jobsRunning = true; + const every = (ms, fn) => setInterval(() => { try { const r = fn(); if (r && r.catch) r.catch((e) => log('[hal] 주기 작업 오류', e.message)); } catch (e) { log('[hal] 주기 작업 오류', e.message); } }, ms).unref(); + every(20000, () => core.renewLease(d, d.instanceId)); every(30000, () => core.markOffline(d)); every(3000, () => core.flushPortalEvents(d)); every(60000, () => emergency.expireCredentials(d)); + every(3600000, () => { core.purgeMetrics(d); purgeSessions(d.store); }); + every(60000, () => { const c = dr.getConfig(d); if (!c.snapshot_interval_minutes) return; const last = d.store.get('SELECT created_at FROM dr_snapshots WHERE verify_state=\'ok\' ORDER BY created_at DESC LIMIT 1'); if (!last || Date.now() - Date.parse(last.created_at) >= c.snapshot_interval_minutes * 60000) dr.snapshot(d, { kind: 'scheduled' }); }); + await app.listen(cfg.port, cfg.host); + log(`[hal] Ariweb HAL v${VERSION} — http://${cfg.host}:${cfg.port} (공개 ${cfg.publicUrl}, 데이터 ${cfg.dataDir}, 부트스트랩 ${core.bootstrapComplete(d) ? '완료' : '진행 중'})`); + process.on('SIGTERM', () => { d.stopping = true; d.jobs.stop(); setTimeout(() => process.exit(0), 500); }); +} else { console.error(`알 수 없는 명령: ${cmd}`); process.exit(1); } diff --git a/hal/jobs.mjs b/hal/jobs.mjs new file mode 100644 index 0000000..4f87a92 --- /dev/null +++ b/hal/jobs.mjs @@ -0,0 +1,167 @@ +/** + * hal/jobs.mjs — HAL 작업 종류(lib/jobs 엔진에 등록). + * + * 두 부류가 있다. + * 1) 포털이 요청하는 작업(docs/HAL-JOBS.md): site.*, domain.*, backup.* — 대상 서버(또는 외부 Agent)에 Agent 작업을 배정하고 결과를 기다린 뒤 돌려준다. + * 복합 작업(backup.create 의 원격 저장, site.relocate, backup.restore)은 여러 Agent 작업을 단계로 잇는다. 모든 단계는 ctx.step 으로 기록되어 HAL 재시작 뒤 이어서 실행된다. + * 2) 운영자 작업(§32·§29): module.deploy / module.rollback / module.health / module.reconcile — 대상 서버마다 자식 작업을 만들고 대상별 결과를 요약한다. 일부 실패는 'partial' 이며 성공으로 보이지 않는다. + * + * 대상(target) 해석: 서버 id(srv_…) → 그 서버의 Agent, Agent id(ag_…) → 그 Agent(외부 Agent 포함). + */ +import { JobError } from '../lib/jobs.mjs'; +import { nowIso, toJson } from '../lib/db.mjs'; +import { audit } from '../lib/audit.mjs'; +import { notify } from '../lib/notify.mjs'; +import { safeError } from '../lib/redact.mjs'; +import { readFileSync, existsSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as core from './core.mjs'; +import { descriptor, DEPLOY_ORDER, ARTIFACT_MODULES } from './modules/index.mjs'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); + +/** 대상 → Agent id */ +function agentFor(d, target) { + if (!target) throw new JobError('target_required', '대상 서버 또는 Agent 가 필요합니다'); + if (String(target).startsWith('ag_')) { const a = core.getAgent(d, target); if (!a || a.status !== 'active') throw new JobError('agent_unavailable', 'Agent 가 없거나 해지되었습니다'); return a.id; } + const s = core.getServer(d, target); if (!s) throw new JobError('server_not_found', `서버가 없습니다: ${target}`); + if (!s.agent_id) throw new JobError('agent_missing', `서버 ${s.name} 에 Agent 가 없습니다`); + if (s.status === 'isolated') throw new JobError('server_isolated', `서버 ${s.name} 은(는) 격리 상태입니다`); + return s.agent_id; +} +/** 외부 접속 확인(§11.2 8단계). 실제 응답을 본다. */ +async function externalCheck(d, url, { expectMarker = null } = {}) { + if (!url) return { ok: false, skipped: 'no_url' }; + const t0 = Date.now(); + try { const r = await (d.fetch || fetch)(url, { redirect: 'follow', signal: AbortSignal.timeout(15000), headers: { 'User-Agent': 'Ariweb-HAL-check/0.1' } }); const text = expectMarker ? await r.text() : ''; return { ok: r.ok && (!expectMarker || text.includes(expectMarker)), status: r.status, ms: Date.now() - t0, url, markerFound: expectMarker ? text.includes(expectMarker) : null }; } + catch (e) { return { ok: false, error: safeError(e), ms: Date.now() - t0, url }; } +} +/** 단일 Agent 작업으로 끝나는 종류를 등록하는 도우미 */ +function single(engine, d, kind, { title, agentKind = kind, steps = [], build = (s) => s.payload || {}, after = null, maxAttempts = 1, target = (s) => s.target }) { + engine.register(kind, { title, maxAttempts, steps: [{ id: 'dispatch', title: '대상 Agent 에 작업 배정' }, { id: 'await', title: '대상 서버에서 실행' }, ...steps], + async run(job, ctx) { + const disp = await core.ensureAgentJob(d, ctx, job, { agentId: agentFor(d, target(job.subject)), kind: agentKind, payload: build(job.subject, d) }); + const result = await ctx.step('await', () => core.awaitAgentJob(d, ctx, disp.agentJobId)); + return after ? after(job, ctx, result) : result; + }, + retryable: (e) => !!e.retryable, + }); +} + +export function registerKinds(engine, d) { + // ---------------- 사이트(§11) + single(engine, d, 'site.deploy', { title: '사이트 배포', steps: [{ id: 'external', title: '외부 접속 확인' }], build: (s) => ({ ...s.payload, siteId: s.payload.siteId }), + async after(job, ctx, result) { + const ext = await ctx.step('external', () => externalCheck(d, job.subject.payload.externalCheckUrl || result?.siteUrl, { expectMarker: job.subject.payload.expectMarker })); + return { ...result, externalCheck: ext }; + } }); + single(engine, d, 'site.rollback', { title: '이전 버전으로 되돌리기' }); + single(engine, d, 'site.remove', { title: '배포 제거' }); + single(engine, d, 'site.check', { title: '사이트 확인', async after(job, ctx, result) { const urls = job.subject.payload?.urls || []; const ext = await ctx.step('external', async () => { const out = []; for (const u of urls) out.push(await externalCheck(d, u, { expectMarker: job.subject.payload?.expectMarker })); return out; }, '외부 확인'); return { ...result, external: ext }; } }); + single(engine, d, 'site.usage', { title: '사용량 수집', maxAttempts: 3 }); + single(engine, d, 'domain.vhost', { title: '도메인 연결 설정' }); + single(engine, d, 'domain.cert', { title: 'HTTPS 인증서 발급', maxAttempts: 2 }); + single(engine, d, 'domain.probe', { title: 'DNS 확인', maxAttempts: 3 }); + + // ---------------- 백업(§19·§20). 저장 서버가 운영 서버와 다르면 HAL 전송을 거쳐 옮긴다. + engine.register('backup.create', { title: '백업 만들기', steps: [{ id: 'create', title: '운영 서버에서 백업 생성' }, { id: 'store', title: '백업 저장소로 복사' }, { id: 'verify', title: '읽을 수 있는지 확인' }], + async run(job, ctx) { + const s = job.subject; const p = s.payload || {}; const siteAgent = agentFor(d, s.target); const storeServer = p.storeServerId && p.storeServerId !== s.target ? p.storeServerId : null; + const created = await ctx.step('create', async () => { const id = core.dispatchAgentJob(d, { halJobId: job.id, agentId: siteAgent, kind: 'backup.create', payload: { ...p, uploadToHal: !!storeServer, transferId: `tr_bk_${job.id}` } }); return core.awaitAgentJob(d, ctx, id); }); + let location = `${s.target}:${created.path}`; let stored = created; + if (storeServer) stored = await ctx.step('store', async () => { const id = core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, storeServer), kind: 'backup.store', payload: { siteId: p.siteId, backupId: created.backupId, transferId: created.transferId, sha256: created.checksum, sizeBytes: created.sizeBytes } }); const r = await core.awaitAgentJob(d, ctx, id); location = `${storeServer}:${r.path}`; return r; }); + else await ctx.step('store', () => ({ skipped: '같은 서버에 보관(저장 서버 미지정)', warning: '운영 서버와 분리된 저장소를 권장합니다(§19.2)' })); + const verified = await ctx.step('verify', async () => { const id = core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, storeServer || s.target), kind: 'backup.verify', payload: { location: stored.path, checksum: created.checksum } }); return core.awaitAgentJob(d, ctx, id); }); + if (!verified?.ok) throw new JobError('backup_unreadable', '생성한 백업을 읽을 수 없습니다'); + return { backupId: created.backupId, location, sizeBytes: created.sizeBytes, checksum: created.checksum, verified: true, createdAt: created.createdAt, scope: created.scope }; + } }); + const splitLoc = (loc) => { const i = String(loc).indexOf(':'); return i > 0 ? { serverId: loc.slice(0, i), path: loc.slice(i + 1) } : { serverId: null, path: loc }; }; + single(engine, d, 'backup.verify', { title: '백업 검증', target: (s) => splitLoc(s.payload.location).serverId || s.target, build: (s) => ({ location: splitLoc(s.payload.location).path, checksum: s.payload.checksum }) }); + single(engine, d, 'backup.delete', { title: '백업 삭제', target: (s) => splitLoc(s.payload.location).serverId || s.target, build: (s) => ({ location: splitLoc(s.payload.location).path }) }); + engine.register('backup.restore', { title: '사이트 복원', steps: [{ id: 'fetch', title: '백업 저장소에서 가져오기' }, { id: 'protect', title: '현재 상태 보호' }, { id: 'isolated', title: '격리된 환경에서 백업 확인' }, { id: 'restore', title: '데이터 복원' }, { id: 'external', title: '외부 접속 확인' }], + async run(job, ctx) { + const s = job.subject; const p = s.payload || {}; const loc = splitLoc(p.location); const siteAgent = agentFor(d, s.target); const remote = loc.serverId && loc.serverId !== s.target; + const fetched = await ctx.step('fetch', async () => { if (!remote) return { path: loc.path }; const id = core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, loc.serverId), kind: 'backup.fetch', payload: { location: loc.path, transferId: `tr_rs_${job.id}`, checksum: p.checksum } }); return core.awaitAgentJob(d, ctx, id); }); + const r = await ctx.step('restore', async () => { const id = core.dispatchAgentJob(d, { halJobId: job.id, agentId: siteAgent, kind: 'backup.restore', payload: { ...p, location: remote ? null : fetched.path, transferId: remote ? fetched.transferId : null } }); return core.awaitAgentJob(d, ctx, id); }); + await ctx.step('protect', () => ({ protectPointLocation: r.protectPointLocation })); await ctx.step('isolated', () => ({ ok: r.isolatedVerify })); + const ext = await ctx.step('external', () => externalCheck(d, p.externalCheckUrl, { expectMarker: p.expectMarker })); + if (p.externalCheckUrl && !ext.ok) throw new JobError('restore_external_check_failed', '복원 뒤 외부 접속 확인에 실패했습니다 — 보호 시점으로 되돌릴 수 있습니다', { detail: ext }); + return { ...r, externalCheck: ext }; + }, + async revert(job, ctx) { const s = job.subject; const r = job.steps.find((x) => x.id === 'restore')?.result; if (!r?.protectPointLocation) throw new JobError('no_protect_point', '보호 시점이 없어 되돌릴 수 없습니다'); await ctx.step('revert', async () => { const id = core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, s.target), kind: 'backup.restore', payload: { siteId: s.payload.siteId, location: r.protectPointLocation, protectPoint: false, isolatedVerify: true }, dedupeKey: `${job.id}:revert` }); return core.awaitAgentJob(d, ctx, id); }, '보호 시점으로 되돌리기'); } }); + + // ---------------- 서버 이전(§21). 두 서버에 동시에 운영 상태가 되지 않도록 verify_new 뒤에만 switch 한다. + engine.register('site.relocate', { title: '서버 이전', steps: ['precheck', 'prepare', 'precopy', 'verify_copy', 'final_sync', 'verify_new', 'switch', 'external_check', 'cleanup'].map((id) => ({ id, title: { precheck: '조건 확인', prepare: '새 위치 준비', precopy: '데이터 사전 복사', verify_copy: '복사 결과 확인', final_sync: '최종 변경분 반영', verify_new: '새 위치 확인', switch: '연결 전환', external_check: '외부 접속 확인', cleanup: '이전 위치 정리' }[id] })), + async run(job, ctx) { + const p = job.subject.payload || {}; const from = agentFor(d, p.fromServerId); const to = agentFor(d, p.toServerId); const dyn = p.kind === 'dynamic'; + await ctx.step('precheck', () => { const t = core.getServer(d, p.toServerId); const f = core.getServer(d, p.fromServerId); if (!t.accepts_new || t.status !== 'ready') throw new JobError('target_not_ready', `도착 서버 ${t.name} 이(가) 신규 배치를 받을 수 없습니다`); if (f.status === 'offline') throw new JobError('source_offline', '출발 서버가 응답하지 않습니다'); const conflict = d.store.get('SELECT id FROM jobs WHERE subject_id=? AND id<>? AND kind IN (\'site.deploy\',\'backup.restore\',\'site.relocate\') AND status IN (\'queued\',\'running\',\'awaiting_external\',\'needs_user\')', p.siteId, job.id); if (conflict) throw new JobError('conflict', `같은 사이트의 다른 작업(${conflict.id})이 진행 중입니다`); return { from: f.name, to: t.name }; }); + await ctx.step('prepare', async () => core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: to, kind: 'site.prepare', payload: { siteId: p.siteId } }))); + const pre = await ctx.step('precopy', async () => core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: from, kind: 'site.export', payload: { siteId: p.siteId, transferId: `tr_rl1_${job.id}`, includeData: true } }))); + await ctx.step('verify_copy', async () => core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: to, kind: 'site.import', payload: { siteId: p.siteId, transferId: pre.transferId, checksum: pre.checksum, stage: 'pre' } }))); + const fin = await ctx.step('final_sync', async () => { if (dyn || p.writeFreeze) await core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: from, kind: 'site.freeze', payload: { siteId: p.siteId, freeze: true } })); const ex = await core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: from, kind: 'site.export', payload: { siteId: p.siteId, transferId: `tr_rl2_${job.id}`, includeData: true }, dedupeKey: `${job.id}:final` })); return core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: to, kind: 'site.import', payload: { siteId: p.siteId, transferId: ex.transferId, checksum: ex.checksum, stage: 'final' }, dedupeKey: `${job.id}:final-import` })); }); + const vn = await ctx.step('verify_new', async () => core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: to, kind: 'site.stage_check', payload: { siteId: p.siteId, expectedChecksum: fin.checksum } }))); + if (!vn?.ok) throw new JobError('verify_new_failed', '새 위치 확인에 실패해 전환하지 않습니다', { detail: vn }); + await ctx.step('switch', async () => { ctx.checkCancel(); const a = await core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: to, kind: 'site.activate', payload: { siteId: p.siteId, domains: p.domains || [], primaryDomain: p.primaryDomain, runtime: p.runtime || 'static' } })); await core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: from, kind: 'site.deactivate', payload: { siteId: p.siteId, retainDays: p.retainDays || 7 } })); return { switchedAt: nowIso(), activate: a }; }); + const ext = await ctx.step('external_check', () => externalCheck(d, p.externalCheckUrl, { expectMarker: p.expectMarker })); + if (p.externalCheckUrl && !ext.ok) throw new JobError('relocate_external_check_failed', '전환 뒤 외부 접속 확인에 실패했습니다 — 되돌리기를 검토하세요', { detail: ext }); + const cl = await ctx.step('cleanup', () => ({ oldRetainedUntil: new Date(Date.now() + (p.retainDays || 7) * 86400000).toISOString(), note: '이전 위치 사본은 보존 기간 뒤 삭제 작업(site.remove)으로 정리' })); + return { newServerId: p.toServerId, switchedAt: job.steps?.find((x) => x.id === 'switch')?.result?.switchedAt || nowIso(), oldRetainedUntil: cl.oldRetainedUntil, externalCheck: ext }; + }, + async revert(job, ctx) { const p = job.subject.payload || {}; const sw = (job.steps || []).find((x) => x.id === 'switch'); if (!sw || sw.state !== 'done') { await ctx.step('revert_unfreeze', async () => core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, p.fromServerId), kind: 'site.freeze', payload: { siteId: p.siteId, freeze: false }, dedupeKey: `${job.id}:unfreeze` }))); return; } + await ctx.step('revert_switch', async () => { await core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, p.fromServerId), kind: 'site.activate', payload: { siteId: p.siteId, domains: p.domains || [], primaryDomain: p.primaryDomain, runtime: p.runtime || 'static', reactivate: true }, dedupeKey: `${job.id}:revert-act` })); await core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, p.toServerId), kind: 'site.deactivate', payload: { siteId: p.siteId, retainDays: 1 }, dedupeKey: `${job.id}:revert-deact` })); }, '이전 위치로 연결 되돌리기'); } }); + + // ---------------- 모듈 배포(§32) + const unitTemplate = (name) => { const f = name ? join(ROOT, 'deploy', 'units', name) : null; return f && existsSync(f) ? readFileSync(f, 'utf8') : null; }; + /** 배포 전제: 릴리스 승인, artifact 존재, HAL 먼저(다른 모듈은 HAL 이 그 릴리스로 실행 중이어야), 부트스트랩 완료 */ + function precheckModule(module, version) { + const rel = core.getRelease(d, version); if (!rel) throw new JobError('release_not_found', `릴리스 ${version} 이(가) 없습니다`); + if (rel.status !== 'approved') throw new JobError('release_not_approved', `릴리스 ${version} 은(는) 승인되지 않았습니다`); + if (ARTIFACT_MODULES.includes(module) && !core.artifactPath(d, version, module)) throw new JobError('artifact_missing', `릴리스 ${version} 의 ${module} artifact 가 없습니다`); + const health = core.selfHealth(d); if (!health.ok) throw new JobError('hal_unhealthy', 'HAL 자체 건강 확인에 실패해 배포하지 않습니다'); + if (!core.bootstrapComplete(d)) throw new JobError('bootstrap_incomplete', '부트스트랩(HAL 설치·첫 Agent 등록)이 끝나지 않았습니다'); + if (module !== 'hal' && health.bootstrap.halRunningVersion !== version) throw new JobError('hal_first', `HAL 이 아직 ${version} 으로 실행되고 있지 않습니다(현재 ${health.bootstrap.halRunningVersion}). HAL 을 먼저 배포하세요`); + return rel; + } + engine.register('module.deploy.target', { title: '모듈 배포(대상)', steps: [{ id: 'dispatch', title: 'Agent 에 설치 지시' }, { id: 'await', title: '설치·재시작' }, { id: 'verify', title: '실제 버전·건강 확인' }], + async run(job, ctx) { + const { module, version, serverId, action = 'install' } = job.subject; const desc = descriptor(module); const srv = core.getServer(d, serverId); if (!srv) throw new JobError('server_not_found', '서버가 없습니다'); + const art = core.artifactPath(d, version, module); + d.store.run('INSERT INTO module_targets(module, server_id, target_version, last_job_id, updated_at) VALUES(?,?,?,?,?) ON CONFLICT(module, server_id) DO UPDATE SET target_version=excluded.target_version, last_job_id=excluded.last_job_id, updated_at=excluded.updated_at', module, serverId, version, job.id, nowIso()); + const facts = srv.facts || {}; const unitName = (facts.units && facts.units[module]) || desc.unit; + const disp = await core.ensureAgentJob(d, ctx, job, { agentId: agentFor(d, serverId), kind: action === 'rollback' ? 'module.rollback' : 'module.install', title: 'Agent 에 설치 지시', payload: { module, version, action, artifactUrl: art ? `/agent/v1/releases/${version}/artifact?module=${module}` : null, sha256: art?.sha256, size: art?.size, descriptor: { ...desc, unit: unitName }, unitTemplate: unitTemplate(desc.unitTemplate), appDir: (facts.appDirs && facts.appDirs[module]) || (desc.baseDir || `/srv/ariweb/${module}`), toVersion: job.subject.toVersion || null } }); + const r = await ctx.step('await', () => core.awaitAgentJob(d, ctx, disp.agentJobId)); + const v = await ctx.step('verify', () => { + const expected = action === 'rollback' ? (job.subject.toVersion || r.actualVersion) : version; const ok = r.actualVersion === expected && r.health?.ok !== false; + d.store.run('UPDATE module_targets SET actual_version=?, health=?, active=?, last_check_at=?, last_error=?, updated_at=? WHERE module=? AND server_id=?', r.actualVersion || null, r.health?.ok === false ? 'down' : 'ok', r.active ? 1 : 0, nowIso(), ok ? null : `기대 ${expected} 실제 ${r.actualVersion}`, nowIso(), module, serverId); + if (!ok) throw new JobError('version_mismatch', `${srv.name}: 설치 뒤 실제 버전(${r.actualVersion})·건강(${r.health?.ok})이 목표와 다릅니다`, { detail: r }); + if (module === 'hal') { d.store.run('UPDATE releases SET hal_deployed_at=? WHERE version=?', nowIso(), version); core.setBootstrap(d, { hal_running_version: expected, release_version: expected }); } + return { actualVersion: r.actualVersion, health: r.health, previousVersion: r.previousVersion }; + }); + return v; + } }); + /** 부모: 대상 서버마다 자식 작업 → 대상별 결과 요약. 일부 실패는 실패로 끝낸다. */ + async function fanout(job, ctx, { kind, mk, serverIds }) { + const ids = await ctx.step('fanout', () => serverIds.map((serverId) => engine.enqueue({ kind, subject: mk(serverId), parentId: job.id, groupId: null, subjectType: 'server', subjectId: serverId, idempotencyKey: `${job.id}:${serverId}`, createdBy: job.created_by, actorType: job.actor_type, label: `${job.label} — ${core.getServer(d, serverId)?.name || serverId}` }).job.id), '대상별 작업 만들기'); + const t0 = Date.now(); while (true) { if (Date.now() - t0 > (d.config.fanoutTimeoutMs || 12 * 3600 * 1000)) throw new JobError('fanout_timeout', '대상별 작업이 제한 시간 안에 끝나지 않았습니다'); const sum = engine.childrenSummary(job.id); ctx.progress(Math.round(100 * (sum.byStatus.done || 0) / Math.max(1, sum.total))); if (['done', 'partial', 'failed'].includes(sum.overall) && !sum.children.some((c) => ['queued', 'running', 'reverting'].includes(c.status))) { if (sum.overall !== 'done') throw new JobError(sum.overall === 'partial' ? 'partial_failure' : 'all_failed', `${sum.total}대 중 ${sum.children.filter((c) => c.status !== 'done').length}대 실패·대기 — 대상별 결과를 확인하세요`, { detail: sum }); return sum; } ctx.checkCancel(); await new Promise((r) => setTimeout(r, d.config.agentPollMs || 1000)); } + } + engine.register('module.deploy', { title: '모듈 배포', steps: [{ id: 'precheck', title: '릴리스·HAL·순서 확인' }, { id: 'fanout', title: '대상별 작업 만들기' }, { id: 'collect', title: '대상별 결과 확인' }], + async run(job, ctx) { + const { module, version, serverIds } = job.subject; await ctx.step('precheck', () => { precheckModule(module, version); if (!serverIds?.length) throw new JobError('no_targets', '대상 서버가 없습니다'); return { order: DEPLOY_ORDER.indexOf(module) }; }); + d.store.run('INSERT INTO modules(name, target_version, updated_at) VALUES(?,?,?) ON CONFLICT(name) DO UPDATE SET target_version=excluded.target_version, updated_at=excluded.updated_at', module, version, nowIso()); + const sum = await ctx.step('collect', () => fanout(job, ctx, { kind: 'module.deploy.target', serverIds, mk: (serverId) => ({ module, version, serverId, action: 'install' }) })); + audit(d.store, { actorType: job.actor_type, actorId: job.created_by, action: 'deploy.module', subjectType: 'module', subjectId: module, detail: { version, serverIds, overall: sum.overall } }); + return sum; + } }); + engine.register('module.rollback', { title: '모듈 되돌리기', steps: [{ id: 'fanout', title: '대상별 작업 만들기' }, { id: 'collect', title: '대상별 결과 확인' }], + async run(job, ctx) { const { module, toVersion, serverIds } = job.subject; const sum = await ctx.step('collect', () => fanout(job, ctx, { kind: 'module.deploy.target', serverIds, mk: (serverId) => ({ module, version: toVersion || core.getRelease(d, d.store.get('SELECT target_version FROM module_targets WHERE module=? AND server_id=?', module, serverId)?.target_version)?.version, serverId, action: 'rollback', toVersion }) })); audit(d.store, { actorType: job.actor_type, actorId: job.created_by, action: 'deploy.module_rollback', subjectType: 'module', subjectId: module, detail: { toVersion, serverIds, overall: sum.overall } }); return sum; } }); + single(engine, d, 'module.health.target', { title: '모듈 건강 확인(대상)', agentKind: 'module.health', target: (s) => s.serverId, build: (s) => ({ module: s.module, descriptor: descriptor(s.module), unit: (core.getServer(d, s.serverId)?.facts?.units || {})[s.module] || descriptor(s.module).unit }), + after(job, ctx, r) { d.store.run('UPDATE module_targets SET actual_version=COALESCE(?, actual_version), health=?, active=?, last_check_at=?, last_error=?, updated_at=? WHERE module=? AND server_id=?', r.actualVersion || null, r.health?.ok ? 'ok' : 'down', r.active ? 1 : 0, nowIso(), r.health?.ok ? null : (r.health?.error || '건강 확인 실패'), nowIso(), job.subject.module, job.subject.serverId); return r; } }); + engine.register('module.health', { title: '모듈 건강 확인', steps: [{ id: 'fanout', title: '대상별 작업 만들기' }, { id: 'collect', title: '대상별 결과 확인' }], async run(job, ctx) { const { module, serverIds } = job.subject; return ctx.step('collect', () => fanout(job, ctx, { kind: 'module.health.target', serverIds: serverIds || d.store.all('SELECT server_id FROM module_targets WHERE module=?', module).map((r) => r.server_id), mk: (serverId) => ({ module, serverId }) })); } }); + /** 다시 맞추기(§29·§34): 목표 버전과 실제 버전이 다른 대상만 다시 설치 */ + engine.register('module.reconcile', { title: '목표·실제 상태 맞추기', steps: [{ id: 'diff', title: '차이 확인' }, { id: 'fanout', title: '대상별 작업 만들기' }, { id: 'collect', title: '대상별 결과 확인' }], + async run(job, ctx) { const diff = await ctx.step('diff', () => d.store.all('SELECT module, server_id, target_version, actual_version FROM module_targets WHERE target_version IS NOT NULL AND (actual_version IS NULL OR actual_version<>target_version)' + (job.subject.module ? ' AND module=?' : ''), ...(job.subject.module ? [job.subject.module] : []))); if (!diff.length) return { drift: [], overall: 'done' }; const sum = await ctx.step('collect', () => fanout(job, ctx, { kind: 'module.deploy.target', serverIds: diff.map((x) => x.server_id), mk: (serverId) => { const x = diff.find((y) => y.server_id === serverId); return { module: x.module, version: x.target_version, serverId, action: 'install' }; } })); return { drift: diff, ...sum }; } }); + + // ---------------- 포털 이벤트: 포털이 만든 작업(subject._portal)의 상태 변화를 outbox 에 넣는다 + engine.onChange((job) => { if (job.subject && job.subject._portal) core.queuePortalEvent(d, { halJobId: job.id, kind: job.kind, status: job.status, steps: job.steps, progress: job.progress, result: job.result, error: job.error, errorCode: job.error_code, userMessage: job.user_message, updatedAt: job.updated_at, subjectType: job.subject_type, subjectId: job.subject_id }); }); +} diff --git a/hal/modules/index.mjs b/hal/modules/index.mjs new file mode 100644 index 0000000..cbafc2b --- /dev/null +++ b/hal/modules/index.mjs @@ -0,0 +1,39 @@ +/** + * hal/modules/index.mjs — 모듈 기술서(descriptor) 모음(§32). + * + * HAL 이 어떤 모듈을 어떤 서버에 어떻게 설치·갱신·건강 확인하는지를 데이터로 둔다. Agent 는 이 기술서를 작업 payload 로 받아 그대로 수행하므로 + * Agent 바이너리는 모듈별 지식을 갖지 않는다(새 모듈을 더할 때 HAL 만 바꾸면 된다). + * + * 기술서 모양 + * name 모듈 이름(릴리스 artifact 이름과 같다: -.tar.gz) + * roles 설치할 수 있는 서버 역할 + * layout 'releases'(= /srv/ariweb//releases/ + current) — 도입 모듈도 같은 틀을 쓰되 데이터 경로는 기존 것을 유지한다 + * unit systemd 유닛 이름(도입 모듈은 서버에 이미 있는 이름을 쓸 수 있어 server.facts.units 로 덮어쓴다) + * unitTemplate deploy/units/.tmpl — 없으면 기존 유닛을 그대로 두고 WorkingDirectory 만 current 심링크가 가리키게 한다 + * health { url, expect } 내부 건강 확인(127.0.0.1) + * version 실제 실행 버전을 읽는 방법: { file:'package.json' } (current/package.json 의 version) 또는 { http:'/api/health', field:'version' } + * preStop/postStart Agent 가 실행할 추가 절차 이름(agent/jobs/module.mjs 가 아는 것만) + * dataDirs 절대 건드리지 않는 데이터 경로(검증·백업 대상 안내용) + */ +export const MODULES = { + hal: { name: 'hal', roles: ['control'], layout: 'releases', unit: 'ariweb-hal', unitTemplate: 'ariweb-hal.service.tmpl', user: 'master', port: 8900, + health: { url: 'http://127.0.0.1:8900/api/health', expect: 'ok' }, version: { http: 'http://127.0.0.1:8900/api/health', field: 'version' }, dataDirs: ['/srv/ariweb/hal/data'], selfUpdate: true }, + portal: { name: 'portal', roles: ['control', 'web'], layout: 'releases', unit: 'ariweb-portal', unitTemplate: 'ariweb-portal.service.tmpl', user: 'master', port: 8910, + health: { url: 'http://127.0.0.1:8910/api/health', expect: 'ok' }, version: { http: 'http://127.0.0.1:8910/api/health', field: 'version' }, dataDirs: ['/srv/ariweb/portal/data'] }, + agent: { name: 'agent', roles: ['control', 'web', 'mail', 'backup'], layout: 'releases', unit: 'ariweb-agent', unitTemplate: 'ariweb-agent.service.tmpl', user: 'master', + health: { heartbeat: true }, version: { file: 'package.json' }, dataDirs: ['/srv/ariweb/agent/data'], selfUpdate: true }, + // 도입 모듈: Webmaker(WebHosting_AI_agent 1.7.3). 기존 서버 배치(/srv/webmaker/current, /data/webmaker, 유닛 webmaker, 포트 3080)를 존중한다 — install 은 releases 에 풀고 current 를 바꾸며, + // 유닛은 deploy/units/webmaker.service.tmpl 로 다시 쓰되 기존 유닛의 Environment 줄(DATA_ROOT 등)을 facts 로 읽어 유지한다. 원본 deploy/remote-update.sh 의 절차(백업→전송→교체→재시작→확인)를 Agent 가 같은 순서로 수행한다. + webmaker: { name: 'webmaker', roles: ['web'], layout: 'releases', unit: 'webmaker', unitTemplate: 'webmaker.service.tmpl', user: 'master', port: 3080, baseDir: '/srv/webmaker', + health: { url: 'http://127.0.0.1:3080/api/auth/csrf', expect: 'any' }, version: { file: 'package.json' }, preStop: ['webmaker.backupData'], postStart: ['webmaker.unitCheck'], dataDirs: ['/data/webmaker', '/var/www/webmaker-sites'], extraUnits: ['webmaker-setup'] }, + // 도입 모듈: AriMail 1.0.11. 서버마다 유닛 이름이 다르다(arihakdang-mail / arimail / heart-mail) — 서버 facts.arimail.unit 이 정한다. 설치는 원본 deploy/deploy.sh 와 같은 절차(테스트→릴리스 복사→DB 백업·마이그레이션→재시작→health)를 Agent 가 수행한다. + arimail: { name: 'arimail', roles: ['mail'], layout: 'releases', unit: 'arimail', unitTemplate: null, user: null, port: 8790, + health: { url: 'http://127.0.0.1:8790/api/health', expect: 'ok' }, version: { file: 'package.json' }, preStop: ['arimail.backupDb'], postStart: ['arimail.migrate', 'arimail.haraka'], dataDirs: ['/data'], copyPaths: ['server', 'web', 'lang', 'migrations', 'package.json', 'haraka'] }, + backup: { name: 'backup', roles: ['backup', 'web'], layout: 'agent-feature', unit: null, health: { feature: 'backup' }, version: { agent: true }, dataDirs: ['/srv/ariweb/backups'] }, + observability: { name: 'observability', roles: ['control', 'web', 'mail', 'backup'], layout: 'agent-feature', unit: null, health: { feature: 'metrics' }, version: { agent: true }, dataDirs: [] }, +}; +/** 릴리스 artifact 로 배포되는 모듈(agent-feature 는 Agent 안의 기능이라 별도 artifact 가 없다) */ +export const ARTIFACT_MODULES = Object.values(MODULES).filter((m) => m.layout === 'releases').map((m) => m.name); +/** HAL 이 가장 먼저이고, 그다음 agent → portal → 도입 모듈 순으로 배포한다(§32.2) */ +export const DEPLOY_ORDER = ['hal', 'agent', 'portal', 'webmaker', 'arimail', 'backup', 'observability']; +export function descriptor(name) { const d = MODULES[name]; if (!d) throw Object.assign(new Error(`알 수 없는 모듈: ${name}`), { code: 'unknown_module', status: 404 }); return d; } diff --git a/hal/ops.mjs b/hal/ops.mjs new file mode 100644 index 0000000..02378ed --- /dev/null +++ b/hal/ops.mjs @@ -0,0 +1,69 @@ +/** + * hal/ops.mjs — 운영자 계정·로그인·다중 인증·접근권(§25, §3.2, §3.3). + * + * - 운영자 계정은 고객 계정과 완전히 분리된 ops_accounts 이며 세션 종류는 'operator'(lib/session). 고객 세션으로는 /ops 를 열 수 없다. + * - 네트워크 제한: /ops/* 는 허용 CIDR(config.opsAllowCidrs) 안이거나, 유효한 접근권 쿠키(aw_ops_grant: 사용자·기기(UA 해시) 귀속, 짧은 수명)가 있어야 로그인 화면도 열린다. + * 접근권은 (1) 서버에서 CLI `node hal/index.mjs grant --login --hours 8` 로, (2) 허용 네트워크 안에서 로그인한 운영자가 API 로 만든다. 일회용 코드 URL(/ops/access?code=…)을 열면 그 기기에 쿠키가 묶인다. + * - 반복 실패 잠금(5회 → 15분), 비밀번호 변경 시 세션 세대 증가 대신 다른 세션 종료, TOTP(선택) 등록·확인·해제, 복구 코드 1회 표시. + * - 고위험 작업은 recentlyAuthed(10분) 를 요구한다. 보안 수준 표시: 'password' | 'mfa'. + */ +import { newId, randomToken, sha256, hashPassword, verifyPassword, safeEqual } from '../lib/crypto.mjs'; +import { nowIso, toJson, parseJson } from '../lib/db.mjs'; +import { audit } from '../lib/audit.mjs'; +import { JobError } from '../lib/jobs.mjs'; +import * as session from '../lib/session.mjs'; +import { cookie } from '../lib/http.mjs'; +import { ipAllowed } from './cidr.mjs'; +import { newSecret, verifyTotp, otpauthUrl } from './totp.mjs'; + +export const GRANT_COOKIE = 'aw_ops_grant'; +const MAX_FAILS = 5, LOCK_MIN = 15; +const uaHash = (ua) => sha256(String(ua || '').slice(0, 300)).slice(0, 32); + +export function createAccount(d, { login, password, displayName, by = null, mustChange = false }) { + login = String(login || '').trim().toLowerCase(); if (!/^[a-z0-9][a-z0-9._-]{2,31}$/.test(login)) throw new JobError('bad_login', '아이디는 영문 소문자·숫자로 시작하는 3~32자여야 합니다'); + if (String(password || '').length < 10) throw new JobError('weak_password', '운영자 비밀번호는 10자 이상이어야 합니다'); + if (d.store.get('SELECT id FROM ops_accounts WHERE login=?', login)) throw new JobError('login_taken', '이미 있는 아이디입니다'); + const { salt, hash } = hashPassword(password); const id = newId('op_'); const now = nowIso(); + d.store.run('INSERT INTO ops_accounts(id, login, display_name, pass_hash, pass_salt, created_at, updated_at, must_change_password) VALUES(?,?,?,?,?,?,?,?)', id, login, String(displayName || login).slice(0, 80), hash, salt, now, now, mustChange ? 1 : 0); + audit(d.store, { actorType: by ? 'operator' : 'system', actorId: by, action: 'ops.account_create', subjectType: 'ops_account', subjectId: id, detail: { login } }); + return publicAccount(d.store.get('SELECT * FROM ops_accounts WHERE id=?', id)); +} +export function publicAccount(a) { if (!a) return null; const mfa = parseJson(a.mfa_json, {}) || {}; return { id: a.id, login: a.login, displayName: a.display_name, status: a.status, lastLoginAt: a.last_login_at, mfaEnabled: !!mfa.enabledAt, securityLevel: mfa.enabledAt ? 'mfa' : 'password', securityHint: mfa.enabledAt ? '다중 인증 사용 중' : '다중 인증을 켜는 것을 권장합니다(§3.3)', mustChangePassword: !!a.must_change_password, lockedUntil: a.lock_until && Date.parse(a.lock_until) > Date.now() ? a.lock_until : null }; } +/** 1단계 로그인(비밀번호). MFA 가 켜져 있으면 세션을 만들지 않고 mfaRequired 와 임시 토큰을 돌려준다. */ +export function login(d, { login, password, totp, ip, ua }) { + const a = d.store.get('SELECT * FROM ops_accounts WHERE login=?', String(login || '').trim().toLowerCase()); + const fail = () => { if (a) { const n = a.failed_logins + 1; d.store.run('UPDATE ops_accounts SET failed_logins=?, lock_until=? WHERE id=?', n, n >= MAX_FAILS ? new Date(Date.now() + LOCK_MIN * 60000).toISOString() : null, a.id); } audit(d.store, { actorType: 'operator', action: 'ops.login', outcome: 'failed', ip, detail: { login } }); throw new JobError('bad_credentials', '아이디 또는 비밀번호가 올바르지 않습니다'); }; + if (!a || a.status !== 'active') fail(); + if (a.lock_until && Date.parse(a.lock_until) > Date.now()) throw Object.assign(new JobError('locked', `로그인 시도가 반복 실패해 ${LOCK_MIN}분 동안 잠겼습니다. 해제 시각: ${a.lock_until}`), { detail: { unlockAt: a.lock_until } }); + if (!verifyPassword(password, a.pass_salt, a.pass_hash)) fail(); + const mfa = parseJson(a.mfa_json, {}) || {}; + if (mfa.enabledAt) { + const secret = d.vault.get(`ops-totp:${a.id}`); const codes = mfa.recoveryHashes || []; + const codeOk = totp && (verifyTotp(secret, totp) || codes.includes(sha256(String(totp).replace(/\s|-/g, '')))); + if (!totp) return { mfaRequired: true, message: '인증 앱의 6자리 코드를 입력하세요' }; + if (!codeOk) fail(); + if (!verifyTotp(secret, totp)) { mfa.recoveryHashes = codes.filter((h) => h !== sha256(String(totp).replace(/\s|-/g, ''))); d.store.run('UPDATE ops_accounts SET mfa_json=? WHERE id=?', toJson(mfa), a.id); } // 복구 코드는 1회용 + } + d.store.run('UPDATE ops_accounts SET failed_logins=0, lock_until=NULL, last_login_at=? WHERE id=?', nowIso(), a.id); + const s = session.createSession(d.store, { userId: a.id, kind: 'operator', ip, ua, days: 1 }); + audit(d.store, { actorType: 'operator', actorId: a.id, action: 'ops.login', ip }); + return { session: s, account: publicAccount(a) }; +} +export function reauth(d, accountId, password, sessionId) { const a = d.store.get('SELECT * FROM ops_accounts WHERE id=?', accountId); if (!a || !verifyPassword(password, a.pass_salt, a.pass_hash)) throw new JobError('bad_credentials', '비밀번호가 올바르지 않습니다'); session.markReauth(d.store, sessionId); audit(d.store, { actorType: 'operator', actorId: accountId, action: 'ops.reauth' }); return true; } +export function changePassword(d, accountId, current, next, keepSessionId) { const a = d.store.get('SELECT * FROM ops_accounts WHERE id=?', accountId); if (!verifyPassword(current, a.pass_salt, a.pass_hash)) throw new JobError('bad_credentials', '현재 비밀번호가 올바르지 않습니다'); if (String(next).length < 10) throw new JobError('weak_password', '10자 이상이어야 합니다'); const { salt, hash } = hashPassword(next); d.store.run('UPDATE ops_accounts SET pass_hash=?, pass_salt=?, must_change_password=0, updated_at=? WHERE id=?', hash, salt, nowIso(), accountId); session.revokeOtherSessions(d.store, accountId, keepSessionId, 'operator'); audit(d.store, { actorType: 'operator', actorId: accountId, action: 'ops.password_change' }); return true; } +/** TOTP 등록 시작: 비밀을 보관소에 임시로 두고 otpauth URL 을 돌려준다(확인 전까지 enabledAt 없음) */ +export function mfaBegin(d, accountId) { const a = d.store.get('SELECT * FROM ops_accounts WHERE id=?', accountId); const secret = newSecret(); d.vault.set(`ops-totp:${accountId}`, secret, { kind: 'totp' }); const mfa = { ...(parseJson(a.mfa_json, {}) || {}), pendingAt: nowIso(), enabledAt: null }; d.store.run('UPDATE ops_accounts SET mfa_json=? WHERE id=?', toJson(mfa), accountId); return { otpauth: otpauthUrl(secret, a.login), secret }; } +/** 코드 확인 → 켜기. 복구 코드 8개를 한 번만 돌려준다. */ +export function mfaConfirm(d, accountId, code) { const secret = d.vault.get(`ops-totp:${accountId}`); if (!secret || !verifyTotp(secret, code)) throw new JobError('bad_totp', '인증 코드가 맞지 않습니다'); const codes = Array.from({ length: 8 }, () => randomToken(6).replace(/[-_]/g, 'x').slice(0, 10)); const mfa = { enabledAt: nowIso(), recoveryHashes: codes.map((c) => sha256(c)) }; d.store.run('UPDATE ops_accounts SET mfa_json=? WHERE id=?', toJson(mfa), accountId); audit(d.store, { actorType: 'operator', actorId: accountId, action: 'ops.mfa_enable' }); return { recoveryCodes: codes }; } +export function mfaDisable(d, accountId, code) { const secret = d.vault.get(`ops-totp:${accountId}`); if (!secret || !verifyTotp(secret, code)) throw new JobError('bad_totp', '인증 코드가 맞지 않습니다'); d.vault.remove(`ops-totp:${accountId}`); d.store.run('UPDATE ops_accounts SET mfa_json=NULL WHERE id=?', accountId); audit(d.store, { actorType: 'operator', actorId: accountId, action: 'ops.mfa_disable' }); return true; } +/** 접근권 발급: 일회용 코드(URL)로 기기에 묶는다 */ +export function issueGrant(d, { accountId, hours = 8, by = null, reason = '' }) { const a = d.store.get('SELECT id FROM ops_accounts WHERE id=? OR login=?', accountId, accountId); if (!a) throw new JobError('account_not_found', '운영자 계정이 없습니다'); const code = randomToken(24); const id = newId('gr_'); d.store.run('INSERT INTO ops_access_grants(id, account_id, code_hash, created_by, created_at, expires_at, reason) VALUES(?,?,?,?,?,?,?)', id, a.id, sha256(code), by, nowIso(), new Date(Date.now() + Math.min(72, Math.max(1, hours)) * 3600000).toISOString(), reason); audit(d.store, { actorType: by ? 'operator' : 'system', actorId: by, action: 'ops.grant_issue', subjectType: 'ops_account', subjectId: a.id, detail: { hours, reason } }); return { id, url: `${d.config.publicUrl}/ops/access?code=${code}`, expiresAt: new Date(Date.now() + hours * 3600000).toISOString() }; } +/** 코드로 기기 묶기 → 쿠키 값(원문). 코드는 1회용. */ +export function bindGrant(d, code, { ua, ip }) { const g = d.store.get('SELECT * FROM ops_access_grants WHERE code_hash=? AND revoked_at IS NULL', sha256(String(code || ''))); if (!g || g.bound_at || Date.parse(g.expires_at) < Date.now()) throw new JobError('bad_grant', '접근 코드가 올바르지 않거나 만료·사용되었습니다'); const cookieVal = randomToken(32); d.store.run('UPDATE ops_access_grants SET bound_at=?, cookie_hash=?, ua_hash=?, ip=? WHERE id=?', nowIso(), sha256(cookieVal), uaHash(ua), ip, g.id); return { cookie: cookieVal, expiresAt: g.expires_at }; } +export function grantValid(d, cookieVal, ua) { if (!cookieVal) return null; const g = d.store.get('SELECT * FROM ops_access_grants WHERE cookie_hash=? AND revoked_at IS NULL', sha256(cookieVal)); if (!g || Date.parse(g.expires_at) < Date.now()) return null; if (!safeEqual(g.ua_hash, uaHash(ua))) return null; return g; } +export function revokeGrant(d, id, by) { d.store.run('UPDATE ops_access_grants SET revoked_at=? WHERE id=?', nowIso(), id); audit(d.store, { actorType: 'operator', actorId: by, action: 'ops.grant_revoke', subjectType: 'grant', subjectId: id }); } +/** 네트워크 관문(§25): 허용 CIDR 또는 접근권 쿠키. 아니면 '접근 불가' — 로그인 화면도 주지 않는다. */ +export function networkGate(d, ctx) { if (ipAllowed(ctx.ip, d.config.opsAllowCidrs)) return { ok: true, via: 'network' }; const g = grantValid(d, ctx.cookies[GRANT_COOKIE], ctx.headers['user-agent']); return g ? { ok: true, via: 'grant', grant: g } : { ok: false, reason: 'network_not_allowed' }; } +export const grantCookie = (val, secure) => cookie(GRANT_COOKIE, val, { maxAge: 72 * 3600, path: '/ops', secure }); +export { ipAllowed }; diff --git a/hal/schema.mjs b/hal/schema.mjs new file mode 100644 index 0000000..feb7a6a --- /dev/null +++ b/hal/schema.mjs @@ -0,0 +1,85 @@ +/** + * hal/schema.mjs — HAL DB 마이그레이션(hal.sqlite). + * lib 공통 표(sessions·jobs·notifications·audit) 뒤에 HAL 고유 표를 둔다. docs/ARCHITECTURE.md §5·§10, Dev-Order §28·§32·§33·§34. + */ +import * as session from '../lib/session.mjs'; +import * as jobs from '../lib/jobs.mjs'; +import * as notify from '../lib/notify.mjs'; +import * as audit from '../lib/audit.mjs'; + +export const MIGRATIONS = [...session.MIGRATIONS, ...jobs.MIGRATIONS, ...notify.MIGRATIONS, ...audit.MIGRATIONS, { id: 'hal_0001', sql: ` +-- 관리 서버(§28). agent_id 는 그 서버에서 돌아가는 Node Agent. target/actual 모듈 버전은 module_targets 가 가진다. +CREATE TABLE IF NOT EXISTS servers ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, role TEXT NOT NULL, location TEXT, status TEXT NOT NULL DEFAULT 'pending', accepts_new INTEGER NOT NULL DEFAULT 0, + approved_at TEXT, approved_by TEXT, agent_id TEXT, last_seen_at TEXT, clock_skew_ms INTEGER, agent_version TEXT, metrics_json TEXT, capacity_json TEXT, facts_json TEXT, + maintenance_note TEXT, isolated_reason TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL +); +-- Node Agent(관리형) 또는 External Pull Agent(고객 외부 서버, 사이트 하나에 귀속). 비밀은 보관소('service:agent:')에, 여기엔 해시만. +CREATE TABLE IF NOT EXISTS agents ( + id TEXT PRIMARY KEY, server_id TEXT, kind TEXT NOT NULL DEFAULT 'managed', secret_hash TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'active', label TEXT, + group_id TEXT, site_id TEXT, version TEXT, registered_at TEXT NOT NULL, last_seen_at TEXT, revoked_at TEXT, facts_json TEXT +); +-- 일회용 등록 토큰(10분, 1회). 외부 Agent 토큰은 group_id/site_id 가 있다. +CREATE TABLE IF NOT EXISTS registration_tokens ( + id TEXT PRIMARY KEY, token_hash TEXT NOT NULL UNIQUE, kind TEXT NOT NULL DEFAULT 'managed', server_id TEXT, group_id TEXT, site_id TEXT, label TEXT, + created_by TEXT, created_at TEXT NOT NULL, expires_at TEXT NOT NULL, used_at TEXT, used_by_agent TEXT +); +-- 릴리스 등록부(§32.1). artifacts_json = { 모듈: { file, sha256, size } } +CREATE TABLE IF NOT EXISTS releases ( + version TEXT PRIMARY KEY, main_commit TEXT NOT NULL, built_at TEXT NOT NULL, artifacts_json TEXT NOT NULL, checks_json TEXT, status TEXT NOT NULL DEFAULT 'registered', + approved_by TEXT, approved_at TEXT, hal_deployed_at TEXT, notes TEXT, registered_at TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS modules (name TEXT PRIMARY KEY, target_version TEXT, updated_at TEXT NOT NULL); +-- 모듈 × 서버 배치 상태(목표 버전·실제 버전·건강·마지막 확인) +CREATE TABLE IF NOT EXISTS module_targets ( + module TEXT NOT NULL, server_id TEXT NOT NULL, target_version TEXT, actual_version TEXT, health TEXT NOT NULL DEFAULT 'unknown', active INTEGER, last_check_at TEXT, last_error TEXT, + last_job_id TEXT, updated_at TEXT NOT NULL, PRIMARY KEY(module, server_id) +); +-- Agent 에게 배정된 작업(HAL 작업의 자식). 상태: queued → running → done|failed|needs_user|cancelled +CREATE TABLE IF NOT EXISTS agent_jobs ( + id TEXT PRIMARY KEY, hal_job_id TEXT, agent_id TEXT NOT NULL, kind TEXT NOT NULL, payload_json TEXT, status TEXT NOT NULL DEFAULT 'queued', progress INTEGER NOT NULL DEFAULT 0, + steps_json TEXT, log_json TEXT, result_json TEXT, error TEXT, error_code TEXT, user_message TEXT, cancel_requested INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, assigned_at TEXT, started_at TEXT, ended_at TEXT, heartbeat_at TEXT +); +CREATE INDEX IF NOT EXISTS agent_jobs_agent ON agent_jobs(agent_id, status, created_at); +CREATE INDEX IF NOT EXISTS agent_jobs_hal ON agent_jobs(hal_job_id); +-- 서버 지표(90일 이상 보존, §28.1) +CREATE TABLE IF NOT EXISTS metrics_samples ( + server_id TEXT NOT NULL, at TEXT NOT NULL, cpu REAL, mem_used REAL, mem_total REAL, disk_used REAL, disk_total REAL, inode_used REAL, inode_total REAL, io_read REAL, io_write REAL, net_in REAL, net_out REAL, clock_skew_ms INTEGER, load1 REAL, + PRIMARY KEY(server_id, at) +); +-- 사건(§31): 같은 원인은 key 로 묶는다 +CREATE TABLE IF NOT EXISTS incidents ( + id TEXT PRIMARY KEY, key TEXT NOT NULL, severity TEXT NOT NULL, title TEXT NOT NULL, component TEXT, customer_impact TEXT, started_at TEXT NOT NULL, resolved_at TEXT, verified_at TEXT, + assignee TEXT, public INTEGER NOT NULL DEFAULT 0, public_note TEXT, notes_json TEXT, updated_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS incidents_key ON incidents(key, resolved_at); +-- 운영자 계정(§25). 고객 계정과 완전히 분리. mfa_json = { totp: { secret(보관소 참조 id), enabledAt } } +CREATE TABLE IF NOT EXISTS ops_accounts ( + id TEXT PRIMARY KEY, login TEXT NOT NULL UNIQUE, display_name TEXT NOT NULL, pass_hash TEXT NOT NULL, pass_salt TEXT NOT NULL, mfa_json TEXT, status TEXT NOT NULL DEFAULT 'active', + failed_logins INTEGER NOT NULL DEFAULT 0, lock_until TEXT, last_login_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, must_change_password INTEGER NOT NULL DEFAULT 0 +); +-- 짧은 수명의 사용자·기기 귀속 접근권(§25: 허용 네트워크 밖에서는 이것 없이 로그인 화면도 열리지 않음) +CREATE TABLE IF NOT EXISTS ops_access_grants ( + id TEXT PRIMARY KEY, account_id TEXT NOT NULL, code_hash TEXT UNIQUE, cookie_hash TEXT UNIQUE, ua_hash TEXT, ip TEXT, created_by TEXT, created_at TEXT NOT NULL, expires_at TEXT NOT NULL, bound_at TEXT, revoked_at TEXT, reason TEXT +); +-- 긴급 복구 세션(§33) +CREATE TABLE IF NOT EXISTS emergency_sessions ( + id TEXT PRIMARY KEY, incident_key TEXT NOT NULL, reason TEXT NOT NULL, scope_json TEXT NOT NULL, status TEXT NOT NULL, declared_by TEXT NOT NULL, declared_at TEXT NOT NULL, + approvals_json TEXT NOT NULL DEFAULT '[]', credential_hash TEXT, credential_issued_at TEXT, credential_expires_at TEXT, credential_revoked_at TEXT, + actions_json TEXT NOT NULL DEFAULT '[]', verify_json TEXT, reconcile_json TEXT, closed_at TEXT, closed_by TEXT, hal_healthy_at_declare INTEGER, updated_at TEXT NOT NULL +); +-- 재해복구(§34) +CREATE TABLE IF NOT EXISTS dr_config (id INTEGER PRIMARY KEY CHECK(id=1), rto_minutes INTEGER, rpo_minutes INTEGER, snapshot_interval_minutes INTEGER, retention_count INTEGER, external_store TEXT, updated_at TEXT, updated_by TEXT); +CREATE TABLE IF NOT EXISTS dr_snapshots (id TEXT PRIMARY KEY, created_at TEXT NOT NULL, location TEXT NOT NULL, size_bytes INTEGER, sha256 TEXT, verify_state TEXT NOT NULL DEFAULT 'pending', verified_at TEXT, external_copied_at TEXT, external_location TEXT, kind TEXT NOT NULL DEFAULT 'scheduled', error TEXT, deleted_at TEXT); +CREATE TABLE IF NOT EXISTS dr_restores (id TEXT PRIMARY KEY, snapshot_id TEXT NOT NULL, requested_by TEXT NOT NULL, approved_by TEXT, approved_at TEXT, old_control_stopped_confirmed_at TEXT, started_at TEXT, finished_at TEXT, status TEXT NOT NULL, checks_json TEXT, diff_json TEXT, note TEXT); +CREATE TABLE IF NOT EXISTS dr_drills (id TEXT PRIMARY KEY, started_at TEXT NOT NULL, finished_at TEXT, snapshot_id TEXT, performed_by TEXT, steps_json TEXT NOT NULL, measured_rto_minutes REAL, measured_rpo_minutes REAL, result TEXT NOT NULL DEFAULT 'running', findings_json TEXT, next_due_at TEXT); +-- 부트스트랩·실행 상태(§32.2). 행 하나(id=1). +CREATE TABLE IF NOT EXISTS bootstrap_state ( + id INTEGER PRIMARY KEY CHECK(id=1), first_install INTEGER NOT NULL DEFAULT 1, release_version TEXT, main_commit TEXT, artifact_sha256 TEXT, hal_running_version TEXT, hal_started_at TEXT, + first_agent_id TEXT, first_agent_registered_at TEXT, manual_install_locked INTEGER NOT NULL DEFAULT 0, live_instance_id TEXT, live_lease_at TEXT, completed_at TEXT, updated_at TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS transfers (id TEXT PRIMARY KEY, created_by_agent TEXT, path TEXT NOT NULL, size_bytes INTEGER, sha256 TEXT, created_at TEXT NOT NULL, expires_at TEXT NOT NULL, purpose TEXT, meta_json TEXT, deleted_at TEXT); +CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value_json TEXT, updated_at TEXT NOT NULL, updated_by TEXT); +CREATE TABLE IF NOT EXISTS portal_events_outbox (id TEXT PRIMARY KEY, payload_json TEXT NOT NULL, created_at TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, next_at TEXT NOT NULL, delivered_at TEXT, last_error TEXT); +` }]; diff --git a/hal/totp.mjs b/hal/totp.mjs new file mode 100644 index 0000000..6e66b03 --- /dev/null +++ b/hal/totp.mjs @@ -0,0 +1,26 @@ +/** + * hal/totp.mjs — RFC 6238 TOTP(운영자 다중 인증, §3.3·§25). 외부 의존성 없이 node:crypto 만 쓴다. + * 비밀은 base32 로 표시(인증 앱 등록용 otpauth URL)하고 저장은 보관소에만 한다. + */ +import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; + +const B32 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'; +export function base32Encode(buf) { let bits = 0, value = 0, out = ''; for (const b of buf) { value = (value << 8) | b; bits += 8; while (bits >= 5) { out += B32[(value >>> (bits - 5)) & 31]; bits -= 5; } } if (bits > 0) out += B32[(value << (5 - bits)) & 31]; return out; } +export function base32Decode(str) { const s = String(str).toUpperCase().replace(/=+$/, '').replace(/[^A-Z2-7]/g, ''); let bits = 0, value = 0; const out = []; for (const c of s) { value = (value << 5) | B32.indexOf(c); bits += 5; if (bits >= 8) { out.push((value >>> (bits - 8)) & 255); bits -= 8; } } return Buffer.from(out); } +/** 새 비밀(base32 32자) */ +export const newSecret = () => base32Encode(randomBytes(20)); +/** 코드 계산. step 30초, 6자리 */ +export function totpCode(secretB32, at = Date.now(), step = 30) { + const counter = Math.floor(at / 1000 / step); const msg = Buffer.alloc(8); msg.writeBigUInt64BE(BigInt(counter)); + const h = createHmac('sha1', base32Decode(secretB32)).update(msg).digest(); const off = h[h.length - 1] & 0xf; + const code = ((h[off] & 0x7f) << 24 | (h[off + 1] & 0xff) << 16 | (h[off + 2] & 0xff) << 8 | (h[off + 3] & 0xff)) % 1000000; + return String(code).padStart(6, '0'); +} +/** 검증(앞뒤 1단계 허용, 상수 시간 비교) */ +export function verifyTotp(secretB32, code, at = Date.now()) { + const c = String(code || '').replace(/\s/g, ''); if (!/^\d{6}$/.test(c)) return false; + for (const d of [-1, 0, 1]) { const exp = Buffer.from(totpCode(secretB32, at + d * 30000)); const got = Buffer.from(c); if (exp.length === got.length && timingSafeEqual(exp, got)) return true; } + return false; +} +/** 인증 앱 등록 URL */ +export const otpauthUrl = (secretB32, login, issuer = 'Ariweb HAL') => `otpauth://totp/${encodeURIComponent(issuer)}:${encodeURIComponent(login)}?secret=${secretB32}&issuer=${encodeURIComponent(issuer)}&digits=6&period=30`; diff --git a/lang/ko/hal.json b/lang/ko/hal.json new file mode 100644 index 0000000..8736086 --- /dev/null +++ b/lang/ko/hal.json @@ -0,0 +1,13 @@ +{ + "hal": { + "name": "HAL", + "menu": { "overview": "운영 현황", "customers": "고객과 사용자 그룹", "orders": "주문과 결제 승인", "products": "상품과 정책", "sites": "사이트와 배포", "mail": "메일 서비스", "domains": "도메인과 인증서", "servers": "서버와 용량", "backup": "백업과 복원", "relocation": "서버 이전", "releases": "배포 릴리스", "incidents": "장애와 알림", "audit": "감사 기록", "recovery": "플랫폼 복구", "emergency": "긴급 복구" }, + "server": { "status": { "pending": "등록 대기", "ready": "배치 가능", "maintenance": "유지보수", "isolated": "격리", "offline": "응답 없음" }, "role": { "web": "웹 호스팅", "mail": "메일", "backup": "백업 저장소", "control": "제어(HAL·포털)" }, "acceptsNew": "신규 배치 허용", "clockSkew": "서버 시계 차이", "lastSeen": "마지막 확인", "drift": "목표 버전과 실제 버전이 다릅니다" }, + "bootstrap": { "incomplete": "부트스트랩(HAL 설치·첫 Agent 등록)이 끝나지 않았습니다", "locked": "첫 Agent 등록이 끝나 일반 모듈의 수동 설치는 잠겼습니다 — HAL 작업으로만 배포합니다", "halFirst": "HAL 을 먼저 배포하고 건강 확인이 끝난 뒤 나머지 모듈을 배포합니다" }, + "release": { "registered": "등록됨", "approved": "승인됨", "halDeployed": "HAL 배포 완료", "checksFailed": "검사 실패" }, + "job": { "partial": "일부 대상 실패 — 대상별 결과를 확인하세요", "allFailed": "모든 대상 실패", "agentSilent": "대상 서버의 Agent 가 응답하지 않습니다" }, + "emergency": { "status": { "declared": "선언됨", "approving": "승인 진행 중", "approved": "승인 완료", "credential_issued": "자격 발급됨", "performing": "수행 중", "verified": "결과 확인·자격 폐기", "reconciled": "상태 조정 완료", "closed": "종료" }, "steps": ["영향 범위 확인", "변경되지 않는 대상 식별정보 재확인", "최종 실행과 되돌리기 가능 여부 확인"], "forbidden": "긴급 복구에서는 일반 고객 서비스 배포를 할 수 없습니다" }, + "dr": { "ready": "정식 운영 준비 완료", "notReady": "정식 운영 준비 전(복구 훈련 또는 설정 필요)", "rto": "복구 목표 시간(분)", "rpo": "허용 가능한 데이터 시점(분)", "interval": "복구 사본 생성 주기(분)", "retention": "보존 개수", "oldControlConfirm": "기존 제어 위치를 중지했습니다" }, + "access": { "denied": "접근할 수 없습니다", "reauth": "고위험 작업은 최근 10분 안의 재인증이 필요합니다" } + } +} diff --git a/portal/hal/client.mjs b/portal/hal/client.mjs new file mode 100644 index 0000000..88ded25 --- /dev/null +++ b/portal/hal/client.mjs @@ -0,0 +1,46 @@ +/** + * portal/hal/client.mjs — 포털이 HAL 을 부르는 어댑터(docs/HAL-JOBS.md 의 deps.hal). + * 서명: lib/serviceauth keyId 'portal', 비밀은 포털 보관소 'service:hal'. HAL 에 닿지 못하면 code 'service_unreachable' 을 던져 화면이 '정보 확인 지연' 을 보이고 신규 증가를 막는다(§13.4·§26.2). + * createFake(): 다른 영역의 테스트용 — HAL 없이 작업이 즉시 성공하는 가짜(결과 모양은 실제와 같다). + */ +import { serviceFetch } from '../../lib/serviceauth.mjs'; + +export function create(deps) { + const base = () => deps.config.halUrl; const secret = () => deps.vault.get('service:hal'); + async function request(method, path, body) { + const s = secret(); if (!s) { const e = new Error('HAL 연동 키(service:hal)가 보관소에 없습니다'); e.code = 'service_unreachable'; throw e; } + const r = await serviceFetch(base(), 'portal', s, method, path, body, { timeoutMs: 20000, fetchImpl: deps.fetch || fetch }); + if (!r.ok) { const e = new Error(r.json?.message || `HAL ${method} ${path} → ${r.status}`); e.code = r.json?.error || `hal_http_${r.status}`; e.status = r.status; e.detail = r.json?.detail; throw e; } + return r.json; + } + const job = (j) => ({ id: j.id, kind: j.kind, label: j.label, status: j.status, steps: j.steps, progress: j.progress, result: j.result, error: j.error, errorCode: j.errorCode, userMessage: j.userMessage, actions: j.actions, waitReason: j.waitReason, attempt: j.attempt, maxAttempts: j.maxAttempts, nextRunAt: j.nextRunAt, etaAt: j.etaAt, updatedAt: j.updatedAt, startedAt: j.startedAt, endedAt: j.endedAt, currentStep: j.currentStep, completedSteps: j.completedSteps, nextStep: j.nextStep, canCancel: j.canCancel, canRetry: j.canRetry, canResume: j.canResume, terminal: j.terminal, stale: j.stale, children: j.children }); + return { + request, + async createJob({ kind, target, payload, idempotencyKey, groupId, subjectType, subjectId, label }) { const r = await request('POST', '/portal/v1/jobs', { kind, target, payload, idempotencyKey, groupId, subjectType, subjectId, label }); return { halJobId: r.halJobId, status: r.status, created: r.created }; }, + async getJob(id) { return job(await request('GET', `/portal/v1/jobs/${encodeURIComponent(id)}`)); }, + cancelJob: (id) => request('POST', `/portal/v1/jobs/${encodeURIComponent(id)}/cancel`).then(job), retryJob: (id) => request('POST', `/portal/v1/jobs/${encodeURIComponent(id)}/retry`).then(job), revertJob: (id) => request('POST', `/portal/v1/jobs/${encodeURIComponent(id)}/revert`).then(job), resumeJob: (id) => request('POST', `/portal/v1/jobs/${encodeURIComponent(id)}/resume`).then(job), + async listServers({ role } = {}) { return (await request('GET', `/portal/v1/servers${role ? `?role=${encodeURIComponent(role)}` : ''}`)).servers; }, + async pickServer({ role = 'web', needBytes = 0 } = {}) { return (await request('POST', '/portal/v1/servers/pick', { role, needBytes })).pick; }, + async issueExternalAgentToken({ groupId, siteId, label }) { const r = await request('POST', '/portal/v1/external-agents/token', { groupId, siteId, label }); return { agentId: r.agentId, token: r.token, expiresAt: r.expiresAt, installCommand: r.installCommand }; }, + async getExternalAgent(agentId) { return (await request('GET', `/portal/v1/external-agents/${encodeURIComponent(agentId)}`)).agent; }, + async getExternalAgentBySite(siteId) { return (await request('GET', `/portal/v1/external-agents/by-site/${encodeURIComponent(siteId)}`)).agent; }, + revokeExternalAgent: (agentId) => request('POST', `/portal/v1/external-agents/${encodeURIComponent(agentId)}/revoke`), + async health() { return request('GET', '/portal/v1/health'); }, + }; +} +/** 테스트용 가짜 HAL: 작업은 만들자마자 done(결과는 종류별 그럴듯한 값). opts.fail 로 특정 kind 를 실패시킬 수 있다. */ +export function createFake(opts = {}) { + const jobs = new Map(); let n = 0; const servers = opts.servers || [{ id: 'srv_test_web', name: '테스트 웹 서버', role: 'web', location: '검증', status: 'ready', acceptsNew: true, health: 'ok', lastSeenAt: new Date().toISOString(), capacity: { diskFreeBytes: 50e9, diskTotalBytes: 100e9, memFreeBytes: 4e9, cpuLoad: 0.2, sitesHosted: 1, reservedBytes: 0 }, modules: {} }, { id: 'srv_test_backup', name: '테스트 백업 서버', role: 'backup', location: '검증', status: 'ready', acceptsNew: true, health: 'ok', lastSeenAt: new Date().toISOString(), capacity: { diskFreeBytes: 500e9, diskTotalBytes: 1000e9, memFreeBytes: 4e9, cpuLoad: 0.1, sitesHosted: 0, reservedBytes: 0 }, modules: {} }]; + const resultFor = (kind, payload) => ({ 'site.deploy': { version: `v${Date.now()}`, deployedAt: new Date().toISOString(), internalCheck: { ok: true, status: 200, ms: 12 }, externalCheck: { ok: !!payload?.externalCheckUrl, url: payload?.externalCheckUrl || null, status: 200 }, siteUrl: payload?.primaryDomain ? `https://${payload.primaryDomain}/` : `http://${payload?.siteId}.preview.ariweb.net/`, https: !!payload?.primaryDomain, leftovers: [] }, 'site.usage': { sites: Object.fromEntries((payload?.siteIds || []).map((s) => [s, { diskBytes: 12_345_678, dbBytes: 0, uploadsBytes: 1024, outBytesDay: 1000, outBytesMonth: 50000, outBytesTotal: 90000, inBytesDay: 100, inBytesMonth: 5000, inBytesTotal: 9000, req24h: 42, pv24h: 40, series7d: [], series30d: [], errorRate: 0, p95ms: 120, measuredAt: new Date().toISOString() }])) }, 'backup.create': { backupId: `bk-${++n}`, location: `srv_test_backup:/srv/ariweb/backups/${payload?.siteId}/bk-${n}.tar.gz`, sizeBytes: 4096, checksum: 'a'.repeat(64), verified: true, createdAt: new Date().toISOString() }, 'backup.verify': { ok: true }, 'backup.restore': { protectPointLocation: `srv_test_web:/srv/ariweb/backups/${payload?.siteId}/protect/p.tar.gz`, restoredVersion: 'restore-1', isolatedVerify: true, externalCheck: { ok: true } }, 'site.relocate': { newServerId: payload?.toServerId, switchedAt: new Date().toISOString(), oldRetainedUntil: new Date(Date.now() + 7 * 86400000).toISOString(), externalCheck: { ok: true } }, 'domain.cert': { issued: payload?.domains || [], failed: [], expiresAt: new Date(Date.now() + 89 * 86400000).toISOString() }, 'domain.vhost': { applied: payload?.domains || [], certs: {} }, 'domain.probe': { a: payload?.expectA || [], aOk: true, txtOk: true, resolvedAt: new Date().toISOString() }, 'site.check': { checks: (payload?.urls || []).map((u) => ({ url: u, status: 200, ok: true, ms: 10, tls: { valid: true, expiresAt: null } })) } }[kind] || { ok: true }); + const view = (j) => ({ ...j, currentStep: null, completedSteps: j.steps.filter((s) => s.state === 'done').map((s) => ({ id: s.id, title: s.title })), nextStep: null, canCancel: false, canRetry: j.status === 'failed', canResume: false, terminal: ['done', 'failed'].includes(j.status), stale: false }); + return { + calls: [], jobs, + async createJob({ kind, target, payload, idempotencyKey, label }) { const ex = idempotencyKey && [...jobs.values()].find((j) => j.idempotencyKey === idempotencyKey); if (ex) return { halJobId: ex.id, status: ex.status, created: false }; const id = `j_fake_${++n}`; const fail = opts.fail && opts.fail[kind]; const j = { id, kind, label: label || kind, target, payload, idempotencyKey, status: fail ? 'failed' : 'done', steps: [{ id: 'run', title: '실행', state: fail ? 'failed' : 'done' }], progress: fail ? 10 : 100, result: fail ? null : resultFor(kind, payload), error: fail ? (typeof fail === 'string' ? fail : '가짜 실패') : null, errorCode: fail ? 'fake_failed' : null, updatedAt: new Date().toISOString(), startedAt: new Date().toISOString(), endedAt: new Date().toISOString(), attempt: 1, maxAttempts: 1 }; jobs.set(id, j); this.calls.push({ kind, target, payload }); return { halJobId: id, status: j.status, created: true }; }, + async getJob(id) { const j = jobs.get(id); if (!j) { const e = new Error('작업이 없습니다'); e.code = 'job_not_found'; throw e; } return view(j); }, + async cancelJob(id) { const j = jobs.get(id); j.status = 'cancelled'; return view(j); }, async retryJob(id) { const j = jobs.get(id); j.status = 'done'; j.result = resultFor(j.kind, j.payload); j.error = null; return view(j); }, async revertJob(id) { const j = jobs.get(id); j.status = 'reverted'; return view(j); }, async resumeJob(id) { return view(jobs.get(id)); }, + async listServers({ role } = {}) { return servers.filter((s) => !role || s.role === role); }, async pickServer({ role = 'web' } = {}) { const s = servers.find((x) => x.role === role && x.acceptsNew); return s ? { serverId: s.id, name: s.name } : null; }, + async issueExternalAgentToken({ siteId }) { return { agentId: null, token: 'example-token', expiresAt: new Date(Date.now() + 600000).toISOString(), installCommand: `curl -fsSL https://hal.example.test/agent/v1/install.sh | bash -s -- --hal https://hal.example.test --token example-token --external --site ${siteId}` }; }, + async getExternalAgent(id) { return { id, status: 'connected', lastSeenAt: new Date().toISOString(), version: '0.1.1', lastDeploy: null, lastCheck: null }; }, async getExternalAgentBySite(siteId) { return opts.externalAgent === false ? null : { id: 'ag_fake', siteId, status: 'connected', lastSeenAt: new Date().toISOString(), version: '0.1.1', lastDeploy: null, lastCheck: null }; }, async revokeExternalAgent() { return { ok: true }; }, + async health() { if (opts.down) { const e = new Error('HAL 연결 불가'); e.code = 'service_unreachable'; throw e; } return { ok: true, version: '0.1.1' }; }, + }; +} diff --git a/scripts/release.mjs b/scripts/release.mjs new file mode 100644 index 0000000..ea708ca --- /dev/null +++ b/scripts/release.mjs @@ -0,0 +1,52 @@ +/** + * scripts/release.mjs — 릴리스 artifact 만들기·검사·등록(§32.1). + * + * node scripts/release.mjs build [--skip-tests] [--out dist] 현재 git 작업 트리에서 dist//-.tar.gz + manifest.json + * node scripts/release.mjs register --hal-data [--approve] [--dist dist] 로컬 HAL DB 에 등록(부트스트랩·CI) + * node scripts/release.mjs verify --dist dist/ manifest 의 sha256 과 파일 비교 + * 버전은 package.json, main 커밋은 git rev-parse HEAD(작업 트리가 깨끗하지 않으면 dirty 표시 — 승인 불가 검사 기록). + * 모듈별 포함 경로: hal(hal, lib, deploy, lang, package.json) portal(portal, lib, lang, package.json) agent(agent, lib, deploy, package.json) + * webmaker(modules/webmaker 전체 - data) arimail(modules/arimail 전체 - data). .git·node_modules·data·dist·test 제외. + */ +import { execFileSync, spawnSync } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, existsSync, statSync, rmSync, readdirSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { join, dirname, basename, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const args = process.argv.slice(2); const cmd = args[0] || 'build'; const opt = (n, d = null) => { const i = args.indexOf('--' + n); return i >= 0 ? (args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : true) : d; }; +const pkg = JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf8')); const VERSION = pkg.version; +const MODULES = { hal: ['hal', 'lib', 'deploy', 'lang', 'package.json', 'README.md'], portal: ['portal', 'lib', 'lang', 'package.json'], agent: ['agent', 'lib', 'deploy', 'package.json'], webmaker: ['modules/webmaker'], arimail: ['modules/arimail'] }; +const sha = (f) => createHash('sha256').update(readFileSync(f)).digest('hex'); +const git = (a) => { try { return execFileSync('git', a, { cwd: ROOT, encoding: 'utf8' }).trim(); } catch (_) { return ''; } }; + +if (cmd === 'build') { + const out = join(resolve(ROOT, opt('out', 'dist')), VERSION); rmSync(out, { recursive: true, force: true }); mkdirSync(out, { recursive: true }); + const commit = git(['rev-parse', 'HEAD']); const dirty = git(['status', '--porcelain']).length > 0; const branch = git(['rev-parse', '--abbrev-ref', 'HEAD']); + let checks = { testsPassed: null, dirty, branch }; + if (!opt('skip-tests')) { const r = spawnSync(process.execPath, ['test/run.mjs'], { cwd: ROOT, stdio: 'inherit' }); checks.testsPassed = r.status === 0; if (r.status !== 0) { console.error('테스트 실패 — artifact 를 만들지 않습니다'); process.exit(1); } } + const artifacts = {}; + for (const [m, paths] of Object.entries(MODULES)) { + const file = `${m}-${VERSION}.tar.gz`; const abs = join(out, file); const present = paths.filter((p) => existsSync(join(ROOT, p))); + const tarArgs = ['-czf', abs, '-C', ROOT, '--exclude=.git', '--exclude=node_modules', '--exclude=*/data', '--exclude=*/data/*', '--exclude=modules/*/data', '--exclude=dist', '--exclude=*.sqlite', '--exclude=*.log', '--exclude=*.tmp']; + if (m === 'webmaker' || m === 'arimail') { tarArgs.push('--transform', `s,^modules/${m},.,`); } tarArgs.push(...present); + execFileSync('tar', tarArgs); artifacts[m] = { file, sha256: sha(abs), size: statSync(abs).size, paths: present }; + } + const manifest = { version: VERSION, mainCommit: commit, branch, builtAt: new Date().toISOString(), artifacts, checks, imported: existsSync(join(ROOT, 'IMPORTED_SOURCES.json')) ? JSON.parse(readFileSync(join(ROOT, 'IMPORTED_SOURCES.json'), 'utf8')).sources.map((s) => ({ name: s.name, repo: s.repo, commit: s.commit })) : [] }; + writeFileSync(join(out, 'manifest.json'), JSON.stringify(manifest, null, 1)); + console.log(`릴리스 ${VERSION} (${commit.slice(0, 7)}${dirty ? ', dirty' : ''}) → ${out}`); for (const [m, a] of Object.entries(artifacts)) console.log(` ${m.padEnd(9)} ${a.file} ${(a.size / 1024).toFixed(0)}KB sha256 ${a.sha256.slice(0, 16)}…`); +} else if (cmd === 'verify') { + const dist = resolve(ROOT, opt('dist', join('dist', VERSION))); const mf = JSON.parse(readFileSync(join(dist, 'manifest.json'), 'utf8')); let bad = 0; + for (const [m, a] of Object.entries(mf.artifacts)) { const ok = existsSync(join(dist, a.file)) && sha(join(dist, a.file)) === a.sha256; console.log(`${ok ? '✔' : '✘'} ${m} ${a.file}`); if (!ok) bad++; } + process.exit(bad ? 1 : 0); +} else if (cmd === 'register') { + const halData = opt('hal-data'); if (!halData) { console.error('--hal-data 가 필요합니다'); process.exit(1); } const dist = resolve(ROOT, opt('dist', join('dist', VERSION))); const mf = JSON.parse(readFileSync(join(dist, 'manifest.json'), 'utf8')); + if (mf.checks?.dirty && opt('approve')) { console.error('작업 트리가 깨끗하지 않은(dirty) 빌드는 승인할 수 없습니다 — main 에 병합된 커밋에서 빌드하세요'); process.exit(1); } + const { openDb } = await import('../lib/db.mjs'); const { openVault } = await import('../lib/vault.mjs'); const { MIGRATIONS } = await import('../hal/schema.mjs'); const core = await import('../hal/core.mjs'); const { loadConfig } = await import('../hal/config.mjs'); + process.env.ARIWEB_HAL_DATA = halData; const config = loadConfig(); const d = { store: openDb(join(halData, 'hal.sqlite'), MIGRATIONS), vault: openVault(halData), config, log: () => {} }; + const dir = join(core.artifactDir(d), mf.version); mkdirSync(dir, { recursive: true }); + const artifacts = {}; for (const [m, a] of Object.entries(mf.artifacts)) { const src = join(dist, a.file); if (sha(src) !== a.sha256) { console.error(`${m} artifact 해시 불일치`); process.exit(1); } const { copyFileSync } = await import('node:fs'); copyFileSync(src, join(dir, basename(a.file))); artifacts[m] = { file: basename(a.file), sha256: a.sha256, size: a.size }; } + const r = core.registerRelease(d, { version: mf.version, mainCommit: mf.mainCommit, builtAt: mf.builtAt, artifacts, checks: mf.checks, approve: !!opt('approve'), by: opt('by', 'release-script') }); + console.log(`HAL 릴리스 등록: ${r.version} (${r.status}) main ${r.main_commit.slice(0, 7)} — artifact ${Object.keys(artifacts).join(', ')}`); d.store.close(); +} else { console.error(`알 수 없는 명령: ${cmd}`); process.exit(1); } diff --git a/test/agent.test.mjs b/test/agent.test.mjs new file mode 100644 index 0000000..357db4f --- /dev/null +++ b/test/agent.test.mjs @@ -0,0 +1,116 @@ +/** test/agent.test.mjs — Node Agent 실행기 회귀 테스트. 임시 루트 + 명령 stub(ARIWEB_AGENT_STUB_CMDS=1)으로 실제 시스템을 건드리지 않는다. tar 는 실제로 실행한다. */ +process.env.ARIWEB_AGENT_STUB_CMDS = '1'; +import { test, eq, ok, tmpDir, done } from '../lib/test.mjs'; +import { createSys } from '../agent/sys.mjs'; +import { createRunner } from '../agent/runner.mjs'; +import { KINDS } from '../agent/jobs/index.mjs'; +import { ingestLog } from '../agent/jobs/site.mjs'; +import { sample } from '../agent/metrics.mjs'; +import { mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync, readlinkSync, copyFileSync, statSync, appendFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; + +/** 가짜 HAL 클라이언트: 전송은 메모리(파일 경로) 저장, artifact 는 미리 둔 파일 */ +function fakeClient({ artifacts = {} } = {}) { + const transfers = new Map(); const calls = []; const up = join(tmpDir(), 'uploads'); mkdirSync(up, { recursive: true }); + return { calls, transfers, + async request(method, path, body) { calls.push({ method, path, body }); return { ok: true }; }, + async download(path, dest, expected) { const src = artifacts[path]; if (!src) { const e = new Error('404'); e.code = 'http_404'; throw e; } copyFileSync(src, dest); const sum = createHash('sha256').update(readFileSync(dest)).digest('hex'); if (expected && expected !== sum) throw Object.assign(new Error('checksum'), { code: 'checksum_mismatch' }); return { sha256: sum, size: statSync(dest).size }; }, + async upload(id, file) { const copy = join(up, id + '.bin'); copyFileSync(file, copy); const sum = createHash('sha256').update(readFileSync(copy)).digest('hex'); this.transfers.set(id, { path: copy, sha256: sum }); return { id, sha256: sum, size: statSync(copy).size }; }, + async downloadTransfer(id, dest, expected) { const t = this.transfers.get(id); if (!t) throw Object.assign(new Error('transfer missing'), { code: 'transfer_not_found' }); copyFileSync(t.path, dest); if (expected && String(expected).replace(/^sha256:/, '') !== t.sha256) throw Object.assign(new Error('checksum'), { code: 'checksum_mismatch' }); return { sha256: t.sha256, size: statSync(dest).size }; }, + clockSkewMs: 0 }; +} +function env(over = {}) { + const base = tmpDir(); const roots = { sites: join(base, 'sites'), srv: join(base, 'srv'), nginx: join(base, 'nginx'), backups: join(base, 'backups'), logs: join(base, 'logs'), acme: join(base, 'acme'), letsencrypt: join(base, 'le'), wildcardCert: join(base, 'le', 'wild'), platformDomain: 'ariweb.net', maintPage: '' }; + for (const r of Object.values(roots)) if (r) mkdirSync(r, { recursive: true }); mkdirSync(join(roots.nginx, 'sites-available'), { recursive: true }); mkdirSync(join(roots.nginx, 'sites-enabled'), { recursive: true }); + const sys = createSys({ stub: true }); const client = fakeClient(over); const dataDir = join(base, 'agent-data'); mkdirSync(dataDir, { recursive: true }); + const runner = createRunner({ kinds: KINDS, roots, sys, client, dataDir }); let n = 0; + return { base, roots, sys, client, dataDir, runner, run: (kind, payload, id = `aj_${++n}`) => runner.execute({ id, kind, payload }) }; +} +/** 사이트 번들 디렉터리 만들기 */ +function bundle(base, name, files = { 'index.html': '

hello marker-1

' }) { const d = join(base, 'bundles', name); mkdirSync(d, { recursive: true }); for (const [f, c] of Object.entries(files)) { mkdirSync(join(d, f, '..'), { recursive: true }); writeFileSync(join(d, f), c); } return d; } + +await test('site.deploy: 레이아웃(releases/current)·버전 기록·vhost·내부 확인·previous, rollback, 실패 시 이전 버전 유지와 leftovers, remove', async () => { + const e = env(); const b1 = bundle(e.base, 'b1'); mkdirSync(e.roots.wildcardCert, { recursive: true }); writeFileSync(join(e.roots.wildcardCert, 'fullchain.pem'), 'x'); + let r = await e.run('site.deploy', { siteId: 'site1', versionRef: 'v1', bundlePath: b1, domains: ['site1.ariweb.net'], primaryDomain: 'site1.ariweb.net', expectMarker: 'marker-1' }); eq(r.status, 'done', JSON.stringify(r)); + const sd = join(e.roots.sites, 'site1'); const cur = readlinkSync(join(sd, 'current')); ok(cur.includes('/releases/')); ok(existsSync(join(cur, 'index.html'))); ok(JSON.parse(readFileSync(join(cur, '.ariweb-version.json'), 'utf8')).versionRef === 'v1'); + ok(r.result.https, '와일드카드 인증서 적용'); eq(r.result.certs['site1.ariweb.net'].source, 'wildcard'); ok(existsSync(join(e.roots.nginx, 'sites-enabled', 'aw-site1.conf'))); + const conf = readFileSync(join(e.roots.nginx, 'sites-available', 'aw-site1.conf'), 'utf8'); ok(conf.includes('server_name site1.ariweb.net;') && conf.includes('listen 443 ssl') && conf.includes('awstats') && conf.includes('.maintenance')); + ok(e.sys.calls.some((c) => c.args.includes('nginx') && c.args.includes('-t')), 'nginx -t 실행'); eq(r.result.previousVersion, null); + // 두 번째 배포 → previous, rollback + const b2 = bundle(e.base, 'b2', { 'index.html': '

v2 marker-2

' }); r = await e.run('site.deploy', { siteId: 'site1', versionRef: 'v2', bundlePath: b2, domains: ['site1.ariweb.net'], expectMarker: 'marker-2' }); eq(r.status, 'done'); ok(r.result.previousVersion); ok(readFileSync(join(sd, 'current', 'index.html'), 'utf8').includes('v2')); + r = await e.run('site.rollback', { siteId: 'site1', domains: ['site1.ariweb.net'] }); eq(r.status, 'done', JSON.stringify(r)); ok(readFileSync(join(sd, 'current', 'index.html'), 'utf8').includes('marker-1')); + // 실패: 마커 없음 → 이전 버전 유지, leftovers 보고 + const b3 = bundle(e.base, 'b3', { 'index.html': 'broken' }); r = await e.run('site.deploy', { siteId: 'site1', versionRef: 'v3', bundlePath: b3, domains: ['site1.ariweb.net'], expectMarker: 'marker-9' }); eq(r.status, 'failed'); eq(r.errorCode, 'internal_check_failed'); ok(r.result.previousKept); ok(r.result.leftovers.length >= 1); ok(readFileSync(join(sd, 'current', 'index.html'), 'utf8').includes('marker-1'), '이전 버전 유지'); + r = await e.run('site.deploy', { siteId: '../etc', bundlePath: b1 }); eq(r.status, 'failed'); eq(r.errorCode, 'bad_site_id'); + r = await e.run('site.remove', { siteId: 'site1', keepData: true }); eq(r.status, 'done'); ok(!existsSync(join(sd, 'current'))); ok(existsSync(join(sd, 'data'))); ok(!existsSync(join(e.roots.nginx, 'sites-enabled', 'aw-site1.conf'))); +}); +await test('실행기: 단계 저장과 재시작 뒤 재개(끝난 단계 건너뜀), needs_user 보고', async () => { + const e = env(); let aRuns = 0; let fail = true; + e.runner.kinds['test.resume'] = { async run(ctx) { await ctx.step('a', () => { aRuns++; return 1; }); if (fail) throw new Error('crash'); await ctx.step('b', () => 2); return { ok: true }; } }; + let r = await e.runner.execute({ id: 'aj_r1', kind: 'test.resume', payload: {} }); eq(r.status, 'failed'); eq(aRuns, 1); + // 실패한 작업 상태 파일은 지워지지만, 재시작 시나리오(실행 중 죽음)는 상태 파일이 남는다 → 흉내: 상태 파일을 다시 써 둔다 + writeFileSync(join(e.dataDir, 'jobs', 'aj_r1.json'), JSON.stringify({ steps: [{ id: 'a', title: 'a', state: 'done', result: 1 }], log: [] })); fail = false; + r = await e.runner.execute({ id: 'aj_r1', kind: 'test.resume', payload: {} }); eq(r.status, 'done'); eq(aRuns, 1, '끝난 단계는 다시 실행하지 않음'); eq(r.steps.map((s) => s.id), ['a', 'b']); + e.runner.kinds['test.ask'] = { async run(ctx) { ctx.needsUser('sudo 가 없습니다', 'sudoers 설치'); } }; r = await e.runner.execute({ id: 'aj_r2', kind: 'test.ask', payload: {} }); eq(r.status, 'needs_user'); ok(r.userMessage.includes('sudoers')); + ok(e.client.calls.some((c) => c.path.endsWith('/result')), '결과 보고'); +}); +await test('site.usage: awstats 로그 증분 파싱·일별 집계·오류율·p95·디렉터리 크기', async () => { + const e = env(); const b = bundle(e.base, 'u', { 'index.html': 'x'.repeat(1000) }); await e.run('site.deploy', { siteId: 'u1', bundlePath: b }); const log = join(e.roots.logs, 'aw-u1.access.log'); const today = new Date().toISOString().slice(0, 10); + writeFileSync(log, `${today}T10:00:00+00:00|u1.preview.ariweb.net|300|5000|200|0.050|/\n${today}T10:00:01+00:00|u1.preview.ariweb.net|200|1000|500|2.000|/x\n`); + let r = await e.run('site.usage', { siteIds: ['u1'] }); const u = r.result.sites.u1; eq(u.req24h, 2); eq(u.outBytesDay, 6000); eq(u.inBytesDay, 500); eq(u.errorRate, 50); eq(u.p95ms, 1000); ok(u.diskBytes >= 1000); eq(u.series7d.length, 7); + appendFileSync(log, `${today}T11:00:00+00:00|u1.preview.ariweb.net|100|100|200|0.010|/\n`); r = await e.run('site.usage', { siteIds: ['u1'] }); eq(r.result.sites.u1.req24h, 3, '증분만 더함'); eq(r.result.sites.u1.outBytesDay, 6100); + const st = { days: {}, offsets: {} }; ingestLog({ roots: e.roots }, 'u1', log, st); eq(Object.values(st.days)[0].req, 3); + const m = sample(); ok(m.mem && m.mem.total > 0); ok(typeof m.load1 === 'number'); +}); +await test('backup: 생성(파일·데이터·DB·설정)·검증·원격 저장(전송)·복원(보호 시점·격리 확인)·삭제', async () => { + const e = env(); const b = bundle(e.base, 'k', { 'index.html': '

marker-k

', 'ariweb.json': '{"runtime":"static"}' }); await e.run('site.deploy', { siteId: 'k1', bundlePath: b, expectMarker: 'marker-k' }); + const sd = join(e.roots.sites, 'k1'); mkdirSync(join(sd, 'data', 'uploads'), { recursive: true }); writeFileSync(join(sd, 'data', 'uploads', 'a.txt'), 'upload-a'); mkdirSync(join(sd, 'data', 'db'), { recursive: true }); + const { DatabaseSync } = await import('node:sqlite'); const db = new DatabaseSync(join(sd, 'data', 'db', 'site.sqlite')); db.exec('CREATE TABLE t(x); INSERT INTO t VALUES(42)'); db.close(); + let r = await e.run('backup.create', { siteId: 'k1', scope: {}, uploadToHal: true, transferId: 'tr_bk_1' }); eq(r.status, 'done', JSON.stringify(r)); ok(r.result.checksum); eq(r.result.scope.db, ['site.sqlite']); ok(r.result.transferId === 'tr_bk_1'); const list = execFileSync('tar', ['-tzf', e.client.transfers.get('tr_bk_1').path]).toString(); ok(list.includes('./meta.json') && list.includes('./files.tar') && list.includes('./data.tar') && list.includes('./db/site.sqlite') && list.includes('./config/vhost.conf'), list); + // 저장 서버 역할: 전송에서 받아 보관·검증 + const e2 = env(); e2.client.transfers.set('tr_bk_1', e.client.transfers.get('tr_bk_1')); r = await e2.run('backup.store', { siteId: 'k1', backupId: 'bk-x', transferId: 'tr_bk_1', sha256: e.client.transfers.get('tr_bk_1').sha256 }); eq(r.status, 'done', JSON.stringify(r)); const loc = r.result.path; ok(existsSync(loc)); + r = await e2.run('backup.verify', { location: loc, checksum: r.result.checksum }); ok(r.result.ok); r = await e2.run('backup.verify', { location: loc, checksum: 'deadbeef' }); eq(r.result.ok, false); eq(r.result.reason, 'checksum_mismatch'); + r = await e2.run('backup.fetch', { location: loc, transferId: 'tr_rs_1' }); eq(r.status, 'done'); ok(e2.client.transfers.get('tr_rs_1')); + // 사이트를 망가뜨린 뒤 복원 + writeFileSync(join(sd, 'current', 'index.html'), 'broken'); writeFileSync(join(sd, 'data', 'uploads', 'a.txt'), 'changed'); e.client.transfers.set('tr_rs_1', e2.client.transfers.get('tr_rs_1')); + r = await e.run('backup.restore', { siteId: 'k1', transferId: 'tr_rs_1', checksum: e2.client.transfers.get('tr_rs_1').sha256, protectPoint: true, isolatedVerify: true }); eq(r.status, 'done', JSON.stringify(r)); ok(r.result.protectPointLocation && existsSync(r.result.protectPointLocation)); ok(r.result.isolatedVerify); + ok(readFileSync(join(sd, 'current', 'index.html'), 'utf8').includes('marker-k')); eq(readFileSync(join(sd, 'data', 'uploads', 'a.txt'), 'utf8'), 'upload-a'); ok(readdirSync(sd).some((x) => x.startsWith('data.pre-restore-')), '이전 데이터 보존'); const db2 = new DatabaseSync(join(sd, 'data', 'db', 'site.sqlite')); eq(db2.prepare('SELECT x FROM t').get().x, 42); db2.close(); + r = await e.run('backup.restore', { siteId: 'other', location: loc, protectPoint: false }); eq(r.status, 'failed'); eq(r.errorCode, 'backup_site_mismatch'); + r = await e2.run('backup.delete', { location: loc }); ok(r.result.deleted); ok(!existsSync(loc)); r = await e2.run('backup.delete', { location: '/etc/passwd' }); eq(r.result.deleted, false, '저장소 밖은 지우지 않음'); +}); +await test('relocate 보조: export → import → stage_check → activate(새 서버) → deactivate(이전 서버 사본 보존), freeze', async () => { + const src = env(); const dst = env(); dst.client.transfers = src.client.transfers; + const b = bundle(src.base, 'r', { 'index.html': 'reloc marker-r' }); await src.run('site.deploy', { siteId: 'r1', bundlePath: b, domains: ['r1.example.com'] }); mkdirSync(join(src.roots.sites, 'r1', 'data'), { recursive: true }); writeFileSync(join(src.roots.sites, 'r1', 'data', 'd.txt'), 'data-r'); + let r = await dst.run('site.prepare', { siteId: 'r1' }); eq(r.status, 'done'); + r = await src.run('site.freeze', { siteId: 'r1', freeze: true }); ok(existsSync(join(src.roots.sites, 'r1', '.frozen'))); + r = await src.run('site.export', { siteId: 'r1', transferId: 'tr_rl1', includeData: true }); eq(r.status, 'done', JSON.stringify(r)); const ck = r.result.checksum; + r = await dst.run('site.import', { siteId: 'r1', transferId: 'tr_rl1', checksum: ck, stage: 'final' }); eq(r.status, 'done', JSON.stringify(r)); ok(r.result.ok); + r = await dst.run('site.stage_check', { siteId: 'r1', expectedChecksum: 'wrong' }); eq(r.result.ok, false); r = await dst.run('site.stage_check', { siteId: 'r1', expectedChecksum: ck }); ok(r.result.ok); + r = await dst.run('site.activate', { siteId: 'r1', domains: ['r1.example.com'], primaryDomain: 'r1.example.com' }); eq(r.status, 'done', JSON.stringify(r)); ok(readFileSync(join(dst.roots.sites, 'r1', 'current', 'index.html'), 'utf8').includes('marker-r')); eq(readFileSync(join(dst.roots.sites, 'r1', 'data', 'd.txt'), 'utf8'), 'data-r'); ok(existsSync(join(dst.roots.nginx, 'sites-enabled', 'aw-r1.conf'))); + r = await src.run('site.deactivate', { siteId: 'r1', retainDays: 3 }); eq(r.status, 'done'); ok(!existsSync(join(src.roots.nginx, 'sites-enabled', 'aw-r1.conf'))); ok(existsSync(join(src.roots.sites, 'r1', 'current')), '사본 보존'); ok(!existsSync(join(src.roots.sites, 'r1', '.frozen'))); ok(JSON.parse(readFileSync(join(src.roots.sites, 'r1', '.retained.json'), 'utf8')).until); +}); +await test('domain.cert: 와일드카드 재사용 / certbot(stub) 뒤 HTTPS vhost / domain.probe stub, domain.vhost 유지보수', async () => { + const e = env(); mkdirSync(e.roots.wildcardCert, { recursive: true }); writeFileSync(join(e.roots.wildcardCert, 'fullchain.pem'), 'x'); const b = bundle(e.base, 'c'); await e.run('site.deploy', { siteId: 'c1', bundlePath: b }); + let r = await e.run('domain.cert', { siteId: 'c1', domains: ['c1.ariweb.net'] }); eq(r.status, 'done'); eq(r.result.source, 'wildcard'); eq(r.result.issued, ['c1.ariweb.net']); + r = await e.run('domain.cert', { siteId: 'c1', domains: ['www.example.com', 'example.com'], email: 'ops@example.test' }); eq(r.status, 'done', JSON.stringify(r)); eq(r.result.source, 'own'); ok(e.sys.calls.some((c) => c.args.includes('certbot') && c.args.includes('--webroot'))); ok(readFileSync(join(e.roots.nginx, 'sites-available', 'aw-c1.conf'), 'utf8').includes('listen 443 ssl')); + r = await e.run('domain.probe', { name: 'x.example.test', _stubAnswer: { a: ['1.2.3.4'], aOk: true, txtOk: true } }); ok(r.result.aOk); + r = await e.run('domain.vhost', { siteId: 'c1', domains: ['example.com'], primaryDomain: 'example.com', maintenance: true, redirectOld: true, oldDomains: ['old.example.com'] }); eq(r.status, 'done'); ok(existsSync(join(e.roots.sites, 'c1', '.maintenance'))); ok(readFileSync(join(e.roots.nginx, 'sites-available', 'aw-c1.conf'), 'utf8').includes('old.example.com')); +}); +await test('module.install: artifact 검증·releases/current·유닛(기존 Environment 보존)·사후 절차·실제 버전, rollback, health, 해시 불일치 실패', async () => { + const e = env(); const app = join(e.base, 'bundles', 'portal'); mkdirSync(join(app, 'portal'), { recursive: true }); writeFileSync(join(app, 'package.json'), JSON.stringify({ name: 'x', version: '0.1.1' })); writeFileSync(join(app, 'portal', 'index.mjs'), ''); const tgz = join(e.base, 'portal-0.1.1.tar.gz'); execFileSync('tar', ['-czf', tgz, '-C', app, '.']); const sum = createHash('sha256').update(readFileSync(tgz)).digest('hex'); + e.client = fakeClient({ artifacts: { '/art/portal': tgz } }); const runner = createRunner({ kinds: KINDS, roots: e.roots, sys: e.sys, client: e.client, dataDir: e.dataDir }); const appDir = join(e.roots.srv, 'portal'); + const desc = { name: 'portal', unit: 'ariweb-portal', unitTemplate: 'ariweb-portal.service.tmpl', user: 'master', health: { url: 'http://127.0.0.1:1/api/health', expect: 'ok' }, version: { file: 'package.json' }, layout: 'releases', postStart: [] }; + let r = await runner.execute({ id: 'm1', kind: 'module.install', payload: { module: 'portal', version: '0.1.1', artifactUrl: '/art/portal', sha256: 'bad', descriptor: desc, appDir, unitTemplate: 'WorkingDirectory={{APP_DIR}}/current\n{{ENV_LINES}}\nExecStart={{NODE}} x' } }); eq(r.status, 'failed'); eq(r.errorCode, 'checksum_mismatch'); + r = await runner.execute({ id: 'm2', kind: 'module.install', payload: { module: 'portal', version: '0.1.1', artifactUrl: '/art/portal', sha256: sum, descriptor: desc, appDir, unitTemplate: 'WorkingDirectory={{APP_DIR}}/current\n{{ENV_LINES}}\nExecStart={{NODE}} x' } }); eq(r.status, 'done', JSON.stringify(r)); eq(r.result.actualVersion, '0.1.1'); eq(readlinkSync(join(appDir, 'current')), join(appDir, 'releases', '0.1.1')); ok(e.sys.calls.some((c) => c.args.join(' ').includes('systemctl restart ariweb-portal'))); + const unitFile = e.sys.calls.find((c) => c.cmd === 'sudo' && c.args.includes('tee')); ok(unitFile || existsSync('/etc/systemd/system/ariweb-portal.service') === false, '유닛 파일 쓰기 시도'); + // 두 번째 버전 설치 뒤 rollback + writeFileSync(join(app, 'package.json'), JSON.stringify({ name: 'x', version: '0.1.2' })); const tgz2 = join(e.base, 'portal-0.1.2.tar.gz'); execFileSync('tar', ['-czf', tgz2, '-C', app, '.']); e.client.transfers; const sum2 = createHash('sha256').update(readFileSync(tgz2)).digest('hex'); const runner2 = createRunner({ kinds: KINDS, roots: e.roots, sys: e.sys, client: fakeClient({ artifacts: { '/art/portal2': tgz2 } }), dataDir: e.dataDir }); + r = await runner2.execute({ id: 'm3', kind: 'module.install', payload: { module: 'portal', version: '0.1.2', artifactUrl: '/art/portal2', sha256: sum2, descriptor: desc, appDir } }); eq(r.result.actualVersion, '0.1.2'); eq(r.result.previousVersion, '0.1.1'); + r = await runner2.execute({ id: 'm4', kind: 'module.rollback', payload: { module: 'portal', version: '0.1.2', toVersion: '0.1.1', descriptor: desc, appDir } }); eq(r.status, 'done', JSON.stringify(r)); eq(r.result.actualVersion, '0.1.1'); + r = await runner2.execute({ id: 'm5', kind: 'module.health', payload: { module: 'portal', descriptor: desc, appDir, unit: 'ariweb-portal' } }); eq(r.result.actualVersion, '0.1.1'); ok(r.result.health.ok); + // agent 자기 갱신은 재시작을 결과 보고 뒤로 미룬다 + r = await runner2.execute({ id: 'm6', kind: 'module.install', payload: { module: 'agent', version: '0.1.2', artifactUrl: '/art/portal2', sha256: sum2, descriptor: { ...desc, name: 'agent', unit: 'ariweb-agent' }, appDir: join(e.roots.srv, 'agent') } }); eq(r.result.restartAfterReport, 'ariweb-agent'); +}); +done(); diff --git a/test/hal.test.mjs b/test/hal.test.mjs new file mode 100644 index 0000000..073d83f --- /dev/null +++ b/test/hal.test.mjs @@ -0,0 +1,213 @@ +/** test/hal.test.mjs — HAL 제어 평면 회귀 테스트(등록·인증·heartbeat·작업 배정·모듈 배포 부분 실패·부트스트랩 관문·외부 Agent 제한·긴급 복구·재해복구·지표 보존·운영자 접근). */ +import { test, eq, ok, rejects, tmpDir, done, sleep } from '../lib/test.mjs'; +import { invoke } from '../lib/http.mjs'; +import { signHeaders } from '../lib/serviceauth.mjs'; +import { createHalDeps } from '../hal/deps.mjs'; +import { buildApp } from '../hal/app.mjs'; +import * as core from '../hal/core.mjs'; +import * as ops from '../hal/ops.mjs'; +import * as emergency from '../hal/emergency.mjs'; +import * as dr from '../hal/dr.mjs'; +import { totpCode } from '../hal/totp.mjs'; +import { ipAllowed } from '../hal/cidr.mjs'; +import { writeFileSync, mkdirSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; + +function makeHal(over = {}) { + const dataDir = join(tmpDir(), 'hal'); mkdirSync(dataDir, { recursive: true }); + const config = { dataDir, file: join(dataDir, 'config.json'), port: 0, host: '127.0.0.1', publicUrl: 'https://hal.example.test', portalUrl: 'http://127.0.0.1:1', opsAllowCidrs: ['127.0.0.1/32'], metricsRetentionDays: 90, offlineAfterMs: 90000, transferTtlMs: 3600000, emergencyTtlMinutes: 30, agentPollMs: 20, agentWaitTimeoutMs: 4000, fanoutTimeoutMs: 8000, values: {}, set(p) { Object.assign(this, p); }, ...over }; + const d = createHalDeps({ config, dbFile: ':memory:', fetchImpl: async (url) => ({ ok: true, status: 200, text: async () => 'marker-ok', json: async () => ({ ok: true }) }) }); d.jobsRunning = true; + const app = buildApp(d); return { d, app, dataDir }; +} +const sig = (keyId, secret, method, path, body) => Object.fromEntries(Object.entries(signHeaders(keyId, secret, method, path, body === undefined ? undefined : JSON.stringify(body))).map(([k, v]) => [k.toLowerCase(), v])); +const call = (app, keyId, secret, method, path, body, extra = {}) => invoke(app, method, path, { body, headers: { ...sig(keyId, secret, method, path, body), ...extra } }); +/** 서버 + Agent 등록 한 벌 */ +async function registerAgent({ d, app }, { role = 'web', name = '웹1', kind = 'managed', siteId = null, groupId = null } = {}) { + const srv = kind === 'managed' ? core.createServer(d, { name, role }) : null; + const t = core.issueRegistrationToken(d, { kind, serverId: srv?.id, siteId, groupId, label: name }); + const r = await invoke(app, 'POST', '/agent/v1/register', { body: { token: t.token, facts: { hostname: name }, version: '0.1.1' } }); ok(r.json.ok, JSON.stringify(r.json)); + const a = { agentId: r.json.agentId, secret: r.json.secret, serverId: r.json.serverId, hb: (body = {}) => call(app, `agent:${r.json.agentId}`, r.json.secret, 'POST', '/agent/v1/heartbeat', { version: '0.1.1', metrics: { cpu: 5, mem: { total: 100, used: 50 }, disk: { total: 1000, used: 100 }, load1: 0.1 }, modules: {}, clockSkewMs: 0, ...body }), next: () => call(app, `agent:${r.json.agentId}`, r.json.secret, 'GET', '/agent/v1/jobs/next?wait=0'), result: (id, body) => call(app, `agent:${r.json.agentId}`, r.json.secret, 'POST', `/agent/v1/jobs/${id}/result`, body), progress: (id, body) => call(app, `agent:${r.json.agentId}`, r.json.secret, 'POST', `/agent/v1/jobs/${id}/progress`, body) }; + await a.hb(); if (srv) core.approveServer(d, srv.id, 'op'); return a; +} +/** 부트스트랩 완료 상태로 만들기 */ +function bootstrapDone(d, version = '0.1.1') { core.claimLive(d, { instanceId: d.instanceId, version }); core.setBootstrap(d, { release_version: version, main_commit: 'abc', artifact_sha256: 'x' }); } +/** Agent 역할을 흉내: 다음 작업을 받아 결과를 보낸다 */ +async function actAgent(a, fn) { const n = await a.next(); const job = n.json.job; if (!job) return null; const out = fn ? await fn(job) : { status: 'done', result: { ok: true } }; await a.result(job.id, out); return job; } +const portalSecret = 'portal-secret-example'; +/** 작업이 끝날 때까지 tick 을 기다리지 않고 계속 던진다(부모 작업이 자식을 기다리는 동안 자식도 실행되게) */ +async function pump(h, jobId, maxMs = 8000) { const ps = []; const t0 = Date.now(); while (Date.now() - t0 < maxMs) { const st = h.d.jobs.get(jobId).status; if (['done', 'failed', 'cancelled', 'needs_user', 'awaiting_external'].includes(st) && !h.d.jobs.list({ status: ['queued', 'running'] }).length) break; ps.push(h.d.jobs.tick().catch(() => {})); await sleep(10); } await Promise.all(ps); return h.d.jobs.get(jobId); } +/** Agent 가 다음 작업이 올 때까지 기다렸다가 처리 */ +async function serveOne(a, fn, maxMs = 4000) { const t0 = Date.now(); while (Date.now() - t0 < maxMs) { const n = await a.next(); if (n.json.job) { const out = fn(n.json.job); await a.result(n.json.job.id, out); return n.json.job; } await sleep(10); } return null; } + +await test('등록 토큰: 1회만, 만료 거부, 잘못된 토큰 거부', async () => { + const h = makeHal(); const srv = core.createServer(h.d, { name: 'a', role: 'web' }); const t = core.issueRegistrationToken(h.d, { serverId: srv.id }); + let r = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: t.token, facts: {}, version: '0.1.1' } }); ok(r.json.ok); ok(r.json.secret && r.json.secret.length > 20); + r = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: t.token } }); eq(r.json.error, 'token_used'); + r = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: 'nope' } }); eq(r.json.error, 'bad_token'); + const t2 = core.issueRegistrationToken(h.d, { serverId: srv.id, ttlMs: -1 }); r = await invoke(h.app, 'POST', '/agent/v1/register', { body: { token: t2.token } }); eq(r.json.error, 'token_expired'); + ok(h.d.vault.has(`service:agent:${core.getServer(h.d, srv.id).agent_id}`), '비밀은 보관소에'); +}); +await test('Agent HMAC 인증: 정상·서명 오류·시계 차이·해지', async () => { + const h = makeHal(); const a = await registerAgent(h); + let r = await a.hb(); eq(r.status, 200, JSON.stringify(r.json)); + r = await call(h.app, `agent:${a.agentId}`, 'wrong-secret', 'POST', '/agent/v1/heartbeat', {}); eq(r.status, 401); ok(r.json.message.includes('bad_signature')); + const hdr = sig(`agent:${a.agentId}`, a.secret, 'POST', '/agent/v1/heartbeat', {}); hdr['x-service-ts'] = String(Date.now() - 10 * 60000); r = await invoke(h.app, 'POST', '/agent/v1/heartbeat', { body: {}, headers: hdr }); eq(r.status, 401); ok(r.json.message.includes('clock_skew')); + core.revokeAgent(h.d, a.agentId); r = await a.hb(); eq(r.status, 401); +}); +await test('heartbeat → 지표·용량·모듈 버전 저장, 무응답 → offline, 지표 90일 보존', async () => { + const h = makeHal({ offlineAfterMs: 50 }); const a = await registerAgent(h); + await a.hb({ modules: { webmaker: { version: '1.7.3', active: true, health: 'ok' } }, clockSkewMs: 120000 }); + const s = core.publicServer(h.d, core.getServer(h.d, a.serverId)); eq(s.modules.webmaker.actual, '1.7.3'); eq(s.capacity.diskTotalBytes, 1000); ok(core.metrics(h.d, a.serverId).length >= 1); + ok(h.d.store.get('SELECT id FROM notifications WHERE incident_key=?', `server:${a.serverId}:clock`), '시계 차이 경고'); + await sleep(80); eq(core.markOffline(h.d).length, 1); eq(core.listServers(h.d)[0].status, 'offline'); await a.hb(); eq(core.listServers(h.d)[0].status, 'ready', 'heartbeat 복귀'); + h.d.store.run('INSERT INTO metrics_samples(server_id, at, cpu) VALUES(?,?,?)', a.serverId, new Date(Date.now() - 100 * 86400000).toISOString(), 1); eq(core.purgeMetrics(h.d), 1); ok(core.metrics(h.d, a.serverId, { since: new Date(0).toISOString() }).every((m) => Date.now() - Date.parse(m.at) < 91 * 86400000)); +}); +await test('부트스트랩 관문: 첫 Agent 승인 전 포털 API 503, 승인·HAL 실행 버전 뒤 완료·수동 설치 잠김', async () => { + const h = makeHal(); h.d.vault.set('service:portal', portalSecret); + let r = await call(h.app, 'portal', portalSecret, 'GET', '/portal/v1/servers'); eq(r.status, 503); eq(r.json.error, 'bootstrap_incomplete'); + core.claimLive(h.d, { instanceId: h.d.instanceId, version: '0.1.1' }); const a = await registerAgent(h, { role: 'control' }); + ok(core.bootstrapComplete(h.d)); eq(core.bootstrapState(h.d).manual_install_locked, 1); eq(core.bootstrapState(h.d).first_agent_id, a.agentId); + r = await call(h.app, 'portal', portalSecret, 'GET', '/portal/v1/servers'); eq(r.status, 200); eq(r.json.servers.length, 1); ok(!('facts' in r.json.servers[0])); + r = await call(h.app, 'portal', 'bad', 'GET', '/portal/v1/servers'); eq(r.status, 401); +}); +await test('하나의 HAL: 다른 인스턴스의 임대가 살아 있으면 기동 거부, takeover 허용', () => { + const h = makeHal(); core.claimLive(h.d, { instanceId: 'hal_A', version: '0.1.1' }); + const e = (() => { try { core.claimLive(h.d, { instanceId: 'hal_B', version: '0.1.1' }); } catch (x) { return x; } })(); eq(e.code, 'another_hal_live'); + ok(core.claimLive(h.d, { instanceId: 'hal_B', version: '0.1.1', takeover: true })); ok(core.claimLive(h.d, { instanceId: 'hal_C', version: '0.1.1', now: Date.now() + 120000 }), '임대 만료 뒤 허용'); +}); +await test('포털 작업: 멱등 생성, Agent 배정·long-poll·결과 → HAL 작업 완료(외부 확인 포함), 이벤트 outbox', async () => { + const h = makeHal(); h.d.vault.set('service:portal', portalSecret); bootstrapDone(h.d); const a = await registerAgent(h); + let r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/jobs', { kind: 'site.deploy', target: a.serverId, payload: { siteId: 's1', externalCheckUrl: 'https://s1.example.test/', expectMarker: 'marker' }, idempotencyKey: 'dep-1', groupId: 'g', subjectType: 'site', subjectId: 's1' }); eq(r.status, 200, JSON.stringify(r.json)); ok(r.json.created); const id = r.json.halJobId; + r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/jobs', { kind: 'site.deploy', target: a.serverId, payload: {}, idempotencyKey: 'dep-1' }); eq(r.json.created, false); eq(r.json.halJobId, id); + r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/jobs', { kind: 'module.deploy', target: a.serverId }); eq(r.status, 400, '포털은 module.* 요청 불가'); + const p = h.d.jobs.tick(); await sleep(30); + let n = await a.next(); eq(n.json.job.kind, 'site.deploy'); eq(n.json.job.payload.siteId, 's1'); ok(!('_dedupe' in n.json.job.payload)); + await a.progress(n.json.job.id, { progress: 40, steps: [{ id: 'x', state: 'running' }] }); await sleep(40); ok(h.d.jobs.get(id).progress >= 40); + await a.result(n.json.job.id, { status: 'done', result: { version: 'v1', siteUrl: 'https://s1.example.test/' } }); await p; + const j = h.d.jobs.get(id); eq(j.status, 'done'); eq(j.result.version, 'v1'); ok(j.result.externalCheck.ok); eq(j.steps.map((s) => s.state), ['done', 'done', 'done']); + r = await call(h.app, 'portal', portalSecret, 'GET', `/portal/v1/jobs/${id}`); eq(r.json.status, 'done'); eq(r.json.result.version, 'v1'); + ok(h.d.store.get('SELECT COUNT(*) AS n FROM portal_events_outbox').n >= 2, '상태 변화마다 outbox'); + // 실패 결과 → failed, 재시도 API → queued + r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/jobs', { kind: 'site.check', target: a.serverId, payload: { siteId: 's1', urls: [] } }); const id2 = r.json.halJobId; const p2 = h.d.jobs.tick(); await sleep(30); await actAgent(a, () => ({ status: 'failed', error: 'disk full', errorCode: 'enospc' })); await p2; eq(h.d.jobs.get(id2).status, 'failed'); eq(h.d.jobs.get(id2).error_code, 'enospc'); + r = await call(h.app, 'portal', portalSecret, 'POST', `/portal/v1/jobs/${id2}/retry`); eq(r.json.status, 'queued'); + // needs_user 결과 → HAL 작업 needs_user + const p3 = h.d.jobs.tick(); await sleep(30); await actAgent(a, () => ({ status: 'needs_user', userMessage: 'sudo 필요' })); await p3; eq(h.d.jobs.get(id2).status, 'needs_user'); ok(h.d.jobs.get(id2).user_message.includes('sudo')); +}); +await test('모듈 배포: 릴리스 승인·HAL 먼저·대상별 자식 작업·일부 실패는 partial, 실제 버전 불일치는 실패, 되돌리기', async () => { + const h = makeHal(); bootstrapDone(h.d); const a1 = await registerAgent(h, { name: 'w1', role: 'control' }); const a2 = await registerAgent(h, { name: 'w2', role: 'web' }); + const art = join(h.dataDir, 'artifacts', '0.1.1'); mkdirSync(art, { recursive: true }); writeFileSync(join(art, 'portal-0.1.1.tar.gz'), 'x'); writeFileSync(join(art, 'hal-0.1.1.tar.gz'), 'y'); + const sum = (f) => createHash('sha256').update(f).digest('hex'); + core.registerRelease(h.d, { version: '0.1.1', mainCommit: 'abc', builtAt: new Date().toISOString(), artifacts: { portal: { file: 'portal-0.1.1.tar.gz', sha256: sum('x') }, hal: { file: 'hal-0.1.1.tar.gz', sha256: sum('y') } } }); + const enq = (module, version = '0.1.1', ids = [a1.serverId, a2.serverId]) => h.d.jobs.enqueue({ kind: 'module.deploy', subject: { module, version, serverIds: ids }, createdBy: 'op', actorType: 'operator' }).job; + let j = enq('portal'); await h.d.jobs.drain(); eq(h.d.jobs.get(j.id).error_code, 'release_not_approved'); + core.approveRelease(h.d, '0.1.1', 'op'); core.setBootstrap(h.d, { hal_running_version: '0.1.0' }); + j = enq('portal'); await h.d.jobs.drain(); eq(h.d.jobs.get(j.id).error_code, 'hal_first', 'HAL 이 먼저'); + // HAL 자체 배포 → 실행 버전 갱신 + j = enq('hal', '0.1.1', [a1.serverId]); let pp = pump(h, j.id); await serveOne(a1, () => ({ status: 'done', result: { actualVersion: '0.1.1', health: { ok: true }, active: true, previousVersion: '0.1.0' } })); await pp; + eq(h.d.jobs.get(j.id).status, 'done', JSON.stringify(h.d.jobs.get(j.id).error)); eq(core.bootstrapState(h.d).hal_running_version, '0.1.1'); ok(core.getRelease(h.d, '0.1.1').hal_deployed_at); + // 포털을 두 서버에: w1 성공, w2 실패 → partial + j = enq('portal'); pp = pump(h, j.id); await serveOne(a1, () => ({ status: 'done', result: { actualVersion: '0.1.1', health: { ok: true }, active: true } })); await serveOne(a2, () => ({ status: 'failed', error: 'health failed', errorCode: 'health_failed' })); await pp; + const parent = h.d.jobs.get(j.id); eq(parent.status, 'failed', JSON.stringify(parent.error)); eq(parent.error_code, 'partial_failure'); const sum2 = h.d.jobs.childrenSummary(j.id); eq(sum2.overall, 'partial'); eq(sum2.total, 2); + eq(h.d.store.get('SELECT health, actual_version FROM module_targets WHERE module=? AND server_id=?', 'portal', a1.serverId).actual_version, '0.1.1'); + // 버전 불일치(Agent 가 다른 버전 보고) → 실패 + j = enq('portal', '0.1.1', [a2.serverId]); pp = pump(h, j.id); await serveOne(a2, () => ({ status: 'done', result: { actualVersion: '0.1.0', health: { ok: true } } })); await pp; eq(h.d.jobs.childrenSummary(j.id).children[0].status, 'failed'); + ok(h.d.store.get('SELECT 1 AS x FROM module_targets WHERE module=? AND server_id=? AND target_version<>actual_version', 'portal', a2.serverId)); + // 되돌리기: 이전 버전으로 + j = h.d.jobs.enqueue({ kind: 'module.rollback', subject: { module: 'portal', toVersion: '0.1.0', serverIds: [a1.serverId] }, createdBy: 'op', actorType: 'operator' }).job; pp = pump(h, j.id); const rb = await serveOne(a1, () => ({ status: 'done', result: { actualVersion: '0.1.0', health: { ok: true }, active: true } })); await pp; eq(rb.kind, 'module.rollback'); eq(h.d.jobs.get(j.id).status, 'done', JSON.stringify(h.d.jobs.get(j.id).error)); +}); +await test('외부 Agent: 허용 작업만, 다른 사이트 거부, 상태 조회', async () => { + const h = makeHal(); bootstrapDone(h.d); await registerAgent(h, { role: 'control' }); const ext = await registerAgent(h, { kind: 'external', siteId: 's9', groupId: 'g1', name: '고객서버' }); + const e = (() => { try { core.dispatchAgentJob(h.d, { halJobId: 'x', agentId: ext.agentId, kind: 'backup.create', payload: {} }); } catch (x) { return x; } })(); eq(e.code, 'kind_not_allowed_external'); + const e2 = (() => { try { core.dispatchAgentJob(h.d, { halJobId: 'x', agentId: ext.agentId, kind: 'site.deploy', payload: { siteId: 'other' } }); } catch (x) { return x; } })(); eq(e2.code, 'site_mismatch'); + ok(core.dispatchAgentJob(h.d, { halJobId: 'x', agentId: ext.agentId, kind: 'site.deploy', payload: { siteId: 's9' } })); + const st = core.externalAgentStatus(h.d, ext.agentId); eq(st.status, 'connected'); eq(st.siteId, 's9'); + h.d.vault.set('service:portal', portalSecret); const r = await call(h.app, 'portal', portalSecret, 'POST', '/portal/v1/external-agents/token', { siteId: 's10', groupId: 'g1' }); ok(r.json.token); ok(r.json.installCommand.includes('--external --site s10')); +}); +await test('운영자: 최초 설정 토큰, 로그인·잠금, 네트워크 관문·접근권, 재인증 필요, CSRF, MFA', async () => { + const h = makeHal(); writeFileSync(join(h.dataDir, 'setup.token'), 'setup-example-token'); + let r = await invoke(h.app, 'POST', '/ops/api/setup', { body: { setupToken: 'wrong', login: 'ozops', password: 'password-example-1', displayName: '오즈' } }); eq(r.status, 403); + r = await invoke(h.app, 'POST', '/ops/api/setup', { body: { setupToken: 'setup-example-token', login: 'ozops', password: 'password-example-1', displayName: '오즈' } }); eq(r.status, 200, JSON.stringify(r.json)); ok(!existsSync(join(h.dataDir, 'setup.token'))); + r = await invoke(h.app, 'POST', '/ops/api/setup', { body: { setupToken: 'setup-example-token', login: 'ozops2', password: 'password-example-1' } }); eq(r.status, 403, '두 번째 설정 거부'); + // 허용 네트워크 밖: 로그인도 403 + r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' }, ip: '203.0.113.5' }); eq(r.status, 403); eq(r.json.error, 'access_denied'); + for (let i = 0; i < 5; i++) { r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'bad' } }); } eq(r.status, 400); r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' } }); eq(r.json.error, 'locked'); + h.d.store.run('UPDATE ops_accounts SET failed_logins=0, lock_until=NULL'); + r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' } }); eq(r.status, 200); const cookie = r.cookies.aw_ops; const csrf = r.json.csrf; ok(cookie && csrf); + const asOp = (method, path, body, extra = {}) => invoke(h.app, method, path, { body, cookies: { aw_ops: cookie }, headers: { 'x-csrf': csrf, ...extra } }); + r = await asOp('GET', '/ops/api/me'); eq(r.json.operator.login, 'ozops'); ok(r.json.recentlyAuthed); + r = await invoke(h.app, 'POST', '/ops/api/servers', { body: { name: 'x', role: 'web' }, cookies: { aw_ops: cookie } }); eq(r.status, 403); eq(r.json.error, 'csrf'); + r = await asOp('POST', '/ops/api/servers', { name: 'x', role: 'web' }); eq(r.status, 200); const sid = r.json.server.id; + h.d.store.run('UPDATE sessions SET reauth_at=?', new Date(Date.now() - 11 * 60000).toISOString()); + r = await asOp('PATCH', `/ops/api/servers/${sid}`, { status: 'isolated' }); eq(r.json.error, 'reauth_required'); r = await asOp('POST', '/ops/api/reauth', { password: 'password-example-1' }); eq(r.status, 200); r = await asOp('PATCH', `/ops/api/servers/${sid}`, { status: 'isolated' }); eq(r.status, 200); + // 접근권: 밖에서 URL 로 기기 묶기 → 쿠키로 로그인 화면 허용 + r = await asOp('POST', '/ops/api/access-grants', { hours: 2 }); const url = new URL(r.json.grant.url); r = await invoke(h.app, 'GET', '/ops/access?code=' + url.searchParams.get('code'), { ip: '203.0.113.5', headers: { 'user-agent': 'UA-1' } }); eq(r.status, 302); const gc = r.cookies.aw_ops_grant; ok(gc); + r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' }, ip: '203.0.113.5', cookies: { aw_ops_grant: gc }, headers: { 'user-agent': 'UA-1' } }); eq(r.status, 200); eq(r.json.via, 'grant'); + r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' }, ip: '203.0.113.5', cookies: { aw_ops_grant: gc }, headers: { 'user-agent': 'UA-2' } }); eq(r.status, 403, '다른 기기(UA) 거부'); + // MFA + r = await asOp('POST', '/ops/api/mfa/begin'); const secret = r.json.secret; r = await asOp('POST', '/ops/api/mfa/confirm', { code: '000000' }); eq(r.json.error, 'bad_totp'); r = await asOp('POST', '/ops/api/mfa/confirm', { code: totpCode(secret) }); eq(r.json.recoveryCodes.length, 8); const rc = r.json.recoveryCodes[0]; + r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1' } }); ok(r.json.mfaRequired); r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1', totp: rc } }); eq(r.status, 200); r = await invoke(h.app, 'POST', '/ops/api/login', { body: { login: 'ozops', password: 'password-example-1', totp: rc } }); eq(r.status, 400, '복구 코드는 1회'); + ok(ipAllowed('192.168.20.5', ['192.168.0.0/16']) && !ipAllowed('10.1.1.1', ['192.168.0.0/16']) && ipAllowed('::ffff:127.0.0.1', ['127.0.0.1/32'])); +}); +await test('긴급 복구: 정상 상태 선언 거부, 3단계 서로 다른 확인, 승인 부족·만료 자격·허용 범위 밖·일반 배포 거부, 정상 복구·조정·종료', async () => { + const h = makeHal(); let e = (() => { try { emergency.declare(h.d, { incidentKey: 'INC-1', reason: 'HAL DB 손상', scope: ['hal', 'hal_data'], by: 'op' }); } catch (x) { return x; } })(); eq(e.code, 'hal_healthy'); + h.d.store.run('INSERT INTO incidents(id, key, severity, title, started_at, updated_at) VALUES(?,?,?,?,?,?)', 'inc1', 'INC-1', 'critical', 'HAL 장애', new Date().toISOString(), new Date().toISOString()); + const s = emergency.declare(h.d, { incidentKey: 'INC-1', reason: 'HAL DB 손상으로 작업 엔진 사용 불가', scope: ['hal', 'hal_data'], by: 'op' }); eq(s.status, 'declared'); + e = (() => { try { emergency.issueCredential(h.d, s.id, 'op'); } catch (x) { return x; } })(); eq(e.code, 'approvals_insufficient'); + e = (() => { try { emergency.approve(h.d, s.id, { step: 2, by: 'op', identifiers: ['hal'] }); } catch (x) { return x; } })(); eq(e.code, 'wrong_step'); + emergency.approve(h.d, s.id, { step: 1, by: 'op', impactText: 'HAL 제어 평면 중단, 고객 서비스는 유지' }); + e = (() => { try { emergency.approve(h.d, s.id, { step: 2, by: 'op', identifiers: ['hal'] }); } catch (x) { return x; } })(); eq(e.code, 'identifiers_mismatch'); + emergency.approve(h.d, s.id, { step: 2, by: 'op', identifiers: ['hal_data', 'hal'] }); + e = (() => { try { emergency.approve(h.d, s.id, { step: 3, by: 'op', finalConfirm: '아무거나', revertible: true }); } catch (x) { return x; } })(); eq(e.code, 'final_confirm_mismatch'); + const s3 = emergency.approve(h.d, s.id, { step: 3, by: 'op', finalConfirm: '복구 실행 INC-1', revertible: true }); eq(s3.status, 'approved'); eq(s3.approvals.length, 3); ok(s3.approvals.every((a) => a.at)); + const cred = emergency.issueCredential(h.d, s.id, 'op'); ok(cred.credential); + let actions = []; h.d.emergencyRunner = async (action, target) => { actions.push(action); return { ok: true, action }; }; + e = await rejects(() => emergency.perform(h.d, s.id, { credential: cred.credential, action: 'module.deploy', by: 'op' }), 'action_forbidden'); + e = await rejects(() => emergency.perform(h.d, s.id, { credential: cred.credential, action: 'cert.reissue_internal', by: 'op' }), 'out_of_scope'); + e = await rejects(() => emergency.perform(h.d, s.id, { credential: 'bad', action: 'hal.restart', by: 'op' }), 'bad_credential'); + ok((await emergency.perform(h.d, s.id, { credential: cred.credential, action: 'hal.restart', by: 'op' })).ok); eq(actions, ['hal.restart']); + const rejectedLogged = emergency.getSession(h.d, s.id).actions.filter((a) => a.result === 'rejected').length; eq(rejectedLogged, 2, '거부도 기록'); + h.d.store.run('UPDATE emergency_sessions SET credential_expires_at=? WHERE id=?', new Date(Date.now() - 1000).toISOString(), s.id); + await rejects(() => emergency.perform(h.d, s.id, { credential: cred.credential, action: 'hal.restart', by: 'op' }), 'credential_expired'); + h.d.store.run('UPDATE emergency_sessions SET credential_expires_at=? WHERE id=?', new Date(Date.now() + 60000).toISOString(), s.id); + const v = emergency.verifyAndRevoke(h.d, s.id, { by: 'op', verify: { halRestarted: true } }); ok(v.credential_revoked_at); await rejects(() => emergency.perform(h.d, s.id, { credential: cred.credential, action: 'hal.restart', by: 'op' }), 'credential_revoked'); + const rc = emergency.reconcile(h.d, s.id, 'op'); ok(Array.isArray(rc.drift)); eq(emergency.close(h.d, s.id, 'op').status, 'closed'); + ok(h.d.store.all('SELECT * FROM audit_log WHERE action LIKE \'emergency.%\'').every((a) => a.immutable === 1)); +}); +await test('재해복구: 설정 검증·복구 사본 생성·검증·보존, 복원 승인·기존 제어 중지 확인, 완료 확인 항목, 훈련 → 준비 상태', async () => { + const h = makeHal(); h.d.store.exec('CREATE TABLE t(x)'); eq(dr.readiness(h.d).ready, false); eq(dr.readiness(h.d).missingConfig.length, 4); + const ext = join(tmpDir(), 'ext'); dr.setConfig(h.d, { rto_minutes: 60, rpo_minutes: 30, snapshot_interval_minutes: 60, retention_count: 2, external_store: ext }, 'op'); + // 메모리 DB 는 VACUUM INTO 가 가능하다 + const s1 = dr.snapshot(h.d, { kind: 'manual', by: 'op' }); eq(s1.verify_state, 'ok'); ok(existsSync(s1.location)); ok(s1.external_copied_at, '외부 보관'); ok(s1.sha256); + const list = execFileSync('tar', ['-tzf', s1.location]).toString(); ok(list.includes('hal.sqlite') && list.includes('meta.json') && !list.includes('vault.key')); + dr.snapshot(h.d); dr.snapshot(h.d); eq(dr.listSnapshots(h.d).filter((s) => !s.deleted_at).length, 2, '보존 개수'); + const latest = dr.listSnapshots(h.d).find((x) => !x.deleted_at); const rq = dr.requestRestore(h.d, { snapshotId: latest.id, by: 'op' }); let e = (() => { try { dr.startRestore(h.d, rq.id, { by: 'op', oldControlStopped: true, confirmText: '기존 제어 위치를 중지했습니다' }); } catch (x) { return x; } })(); eq(e.code, 'restore_not_approved'); + dr.approveRestore(h.d, rq.id, 'op2'); e = (() => { try { dr.startRestore(h.d, rq.id, { by: 'op', oldControlStopped: false, confirmText: '' }); } catch (x) { return x; } })(); eq(e.code, 'old_control_not_confirmed'); + const st = dr.startRestore(h.d, rq.id, { by: 'op', oldControlStopped: true, confirmText: '기존 제어 위치를 중지했습니다' }); ok(st.command.includes('hal-restore.sh')); + e = (() => { try { dr.finishRestore(h.d, rq.id, { by: 'op', checks: { operatorLogin: true } }); } catch (x) { return x; } })(); eq(e.code, 'checks_incomplete'); + eq(dr.finishRestore(h.d, rq.id, { by: 'op', checks: { operatorLogin: true, halControl: true, https: true, customerService: true } }).status, 'completed'); + const dl = dr.startDrill(h.d, { by: 'op', snapshotId: latest.id }); eq(dr.finishDrill(h.d, dl.id, { by: 'op', steps: { hal: true, operatorLogin: true, vault: true, certBase: true, httpsCustomer: false }, measuredRtoMinutes: 20, measuredRpoMinutes: 10 }).result, 'failed'); + const dl2 = dr.startDrill(h.d, { by: 'op' }); eq(dr.finishDrill(h.d, dl2.id, { by: 'op', steps: { hal: true, operatorLogin: true, vault: true, certBase: true, httpsCustomer: true }, measuredRtoMinutes: 20, measuredRpoMinutes: 10 }).result, 'success'); ok(dr.readiness(h.d).ready); +}); +await test('공개 상태·운영 현황·전송 업로드·artifact 내려받기', async () => { + const h = makeHal(); bootstrapDone(h.d); const a = await registerAgent(h); + let r = await invoke(h.app, 'GET', '/api/status'); ok(r.json.components.length >= 3); ok(!JSON.stringify(r.json).includes(a.serverId), '내부 id 없음'); + r = await invoke(h.app, 'GET', '/status'); ok(r.text.includes('Ariweb 서비스 상태')); + const body = Buffer.from('hello-transfer'); const sum = createHash('sha256').update(body).digest('hex'); + const low = (o) => Object.fromEntries(Object.entries(o).map(([k, v]) => [k.toLowerCase(), v])); const hdr = low(signHeaders(`agent:${a.agentId}`, a.secret, 'POST', '/agent/v1/transfers/tr_t1', '')); r = await invoke(h.app, 'POST', '/agent/v1/transfers/tr_t1', { body: body.toString(), headers: { ...hdr, 'content-type': 'application/octet-stream', 'x-transfer-sha256': sum } }); eq(r.status, 200, JSON.stringify(r.json)); eq(r.json.sha256, sum); + ok(core.getTransfer(h.d, 'tr_t1')); + r = await invoke(h.app, 'POST', '/agent/v1/transfers/tr_t2', { body: 'x', headers: { ...low(signHeaders(`agent:${a.agentId}`, a.secret, 'POST', '/agent/v1/transfers/tr_t2', '')), 'content-type': 'application/octet-stream', 'x-transfer-sha256': 'bad' } }); eq(r.status, 400); + const art = join(h.dataDir, 'artifacts', '0.1.1'); mkdirSync(art, { recursive: true }); writeFileSync(join(art, 'agent-0.1.1.tar.gz'), 'tgz'); core.registerRelease(h.d, { version: '0.1.1', mainCommit: 'abc', builtAt: 'now', artifacts: { agent: { file: 'agent-0.1.1.tar.gz', sha256: createHash('sha256').update('tgz').digest('hex') } } }); + r = await call(h.app, `agent:${a.agentId}`, a.secret, 'GET', '/agent/v1/releases/0.1.1/artifact?module=agent'); eq(r.status, 200); eq(r.text, 'tgz'); + const t = core.issueRegistrationToken(h.d, { kind: 'managed' }); r = await invoke(h.app, 'GET', '/agent/v1/bootstrap-artifact?module=agent&version=0.1.1', { headers: { 'x-registration-token': t.token } }); eq(r.status, 200); eq(r.text, 'tgz'); +}); +await test('서버 이전 작업: 단계 순서와 verify_new 실패 시 전환하지 않음', async () => { + const h = makeHal(); bootstrapDone(h.d); const a1 = await registerAgent(h, { name: 'from' }); const a2 = await registerAgent(h, { name: 'to' }); + const j = h.d.jobs.enqueue({ kind: 'site.relocate', subject: { payload: { siteId: 's1', fromServerId: a1.serverId, toServerId: a2.serverId, kind: 'static' } }, subjectType: 'site', subjectId: 's1' }).job; + const seen = []; const pp = pump(h, j.id, 10000); const serve = async (a, fn) => { const job = await serveOne(a, fn); if (job) seen.push(job.kind); return job; }; + await serve(a2, () => ({ status: 'done', result: { prepared: true } })); await serve(a1, () => ({ status: 'done', result: { transferId: 'tr_rl1', checksum: 'c1' } })); await serve(a2, () => ({ status: 'done', result: { ok: true } })); await serve(a1, () => ({ status: 'done', result: { transferId: 'tr_rl2', checksum: 'c2' } })); await serve(a2, () => ({ status: 'done', result: { ok: true, checksum: 'c2' } })); await serve(a2, () => ({ status: 'done', result: { ok: false, reason: '체크섬 다름' } })); + await pp; + eq(seen, ['site.prepare', 'site.export', 'site.import', 'site.export', 'site.import', 'site.stage_check']); eq(h.d.jobs.get(j.id).status, 'failed'); eq(h.d.jobs.get(j.id).error_code, 'verify_new_failed'); ok(!h.d.store.get('SELECT 1 AS x FROM agent_jobs WHERE hal_job_id=? AND kind=\'site.activate\'', j.id), '전환하지 않음'); +}); +done();