HAL 제어 평면(릴리스·서버·Agent·작업·모듈 배포·운영자 계정·긴급 복구·재해복구·공개 상태), Node Agent, 배포 스크립트·유닛·nginx 템플릿, 릴리스 도구, 포털 HAL 클라이언트 (§28·§29·§32~§34)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
zzabisagent-lab
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
403d0fb845
commit
6b5a014041
+186
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* hal/app.mjs — HAL HTTP 라우트.
|
||||
*
|
||||
* /api/health 공개 자체 건강(버전·DB·작업 엔진·부트스트랩 상태). 비밀 없음.
|
||||
* /status, /api/status 공개 상태 페이지 데이터(§31.3): 구성요소 상태·계획 유지보수·공개 사건. 내부 서버 이름·개인정보 없음.
|
||||
* /agent/v1/* Node Agent 프로토콜(§5). install.sh·register 외에는 Agent HMAC 인증.
|
||||
* /portal/v1/* 포털 전용 API(docs/HAL-JOBS.md). 서비스 키 'portal'. 부트스트랩 미완료면 503.
|
||||
* /ops/api/* 운영자 API(§25): 네트워크 관문 → 운영자 세션. 서버·릴리스·모듈 배포·작업·사건·긴급 복구·재해복구·계정. 화면(UI)은 별도 영역(hal/ui, 후속)에서 같은 API 를 쓴다.
|
||||
*/
|
||||
import { createApp, HttpError, cookie, esc } from '../lib/http.mjs';
|
||||
import { verifyCtx } from '../lib/serviceauth.mjs';
|
||||
import * as session from '../lib/session.mjs';
|
||||
import { describe, JobError } from '../lib/jobs.mjs';
|
||||
import { listAudit } from '../lib/audit.mjs';
|
||||
import { listNotifications } from '../lib/notify.mjs';
|
||||
import { redact } from '../lib/redact.mjs';
|
||||
import { nowIso, toJson, parseJson } from '../lib/db.mjs';
|
||||
import { newId, sha256 } from '../lib/crypto.mjs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createReadStream, createWriteStream, statSync, readFileSync, existsSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import * as core from './core.mjs';
|
||||
import * as ops from './ops.mjs';
|
||||
import * as emergency from './emergency.mjs';
|
||||
import * as dr from './dr.mjs';
|
||||
import { MODULES, DEPLOY_ORDER } from './modules/index.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const bad = (e) => { if (e instanceof HttpError) throw e; if (e instanceof JobError || (e && e.code)) throw new HttpError(e.status || (e.code === 'quota_insufficient' ? 409 : 400), e.code, e.message, e.detail); throw e; };
|
||||
const wrap = (fn) => async (ctx) => { try { return await fn(ctx); } catch (e) { bad(e); } };
|
||||
|
||||
export function buildApp(d) {
|
||||
const app = createApp({ log: d.log });
|
||||
const uiDir = join(ROOT, 'hal', 'ui', 'public'); if (existsSync(uiDir)) app.static('/ops/ui', uiDir);
|
||||
|
||||
// ---------------- 공개
|
||||
app.get('/api/health', () => core.selfHealth(d));
|
||||
app.get('/api/status', () => publicStatus(d));
|
||||
app.get('/status', (ctx) => ctx.html(statusPage(d)));
|
||||
app.get('/agent/v1/install.sh', (ctx) => { const f = join(ROOT, 'deploy', 'agent-install.sh'); return ctx.text(existsSync(f) ? readFileSync(f, 'utf8') : 'echo "install script missing"; exit 1'); });
|
||||
|
||||
// ---------------- Agent 프로토콜
|
||||
app.post('/agent/v1/register', wrap(async (ctx) => { const b = await ctx.body(); const r = core.registerAgent(d, { token: b.token, facts: b.facts || {}, version: b.version || '' }); return { ok: true, ...r, halUrl: d.config.publicUrl, serverTime: nowIso() }; }));
|
||||
/** Agent 인증 미들웨어: /agent/v1/* (register·install.sh 제외) */
|
||||
const agentAuth = async (ctx) => {
|
||||
const streaming = ctx.method === 'POST' && /^\/agent\/v1\/transfers\//.test(ctx.path);
|
||||
const r = streaming ? verifyStreaming(ctx) : await verifyCtx(ctx, (keyId) => keyId.startsWith('agent:') ? d.vault.get(`service:${keyId}`) : null);
|
||||
if (!r.ok) throw new HttpError(401, 'agent_auth_failed', `Agent 인증 실패(${r.reason})`);
|
||||
const agent = core.getAgent(d, r.keyId.slice(6)); if (!agent || agent.status !== 'active') throw new HttpError(401, 'agent_revoked', 'Agent 가 해지되었습니다');
|
||||
ctx.agent = agent; ctx.outHeaders['X-HAL-Time'] = String(Date.now());
|
||||
};
|
||||
function verifyStreaming(ctx) { const { verifyRequest } = d.serviceauth; return verifyRequest({ method: ctx.method, path: ctx.path + (ctx.url.search || ''), headers: ctx.headers, rawBody: '' }, (keyId) => keyId.startsWith('agent:') ? d.vault.get(`service:${keyId}`) : null); }
|
||||
/** 등록 전 Agent 코드 내려받기: 아직 서명을 만들 수 없으므로 유효한(미사용·미만료) 등록 토큰을 헤더로 확인한다. 토큰은 소비하지 않는다(등록 때 1회 소비). */
|
||||
app.get('/agent/v1/bootstrap-artifact', wrap((ctx) => { const tok = String(ctx.headers['x-registration-token'] || ''); const t = d.store.get('SELECT * FROM registration_tokens WHERE token_hash=?', sha256(tok)); if (!t || t.used_at || Date.parse(t.expires_at) < Date.now()) throw new HttpError(401, 'bad_token', '등록 토큰이 올바르지 않거나 만료되었습니다'); const version = ctx.query.version || core.bootstrapState(d).release_version; const a = core.artifactPath(d, version, ctx.query.module || 'agent'); if (!a) throw new HttpError(404, 'artifact_not_found', 'artifact 가 없습니다'); ctx.ended = true; ctx.res.writeHead(200, { 'Content-Type': 'application/gzip', 'Content-Length': a.size, 'X-Sha256': a.sha256, 'Cache-Control': 'no-store' }); createReadStream(a.path).pipe(ctx.res); }));
|
||||
app.use(async (ctx) => { if (ctx.path.startsWith('/agent/v1/') && !['/agent/v1/register', '/agent/v1/install.sh', '/agent/v1/bootstrap-artifact'].includes(ctx.path)) await agentAuth(ctx); });
|
||||
app.post('/agent/v1/heartbeat', wrap(async (ctx) => core.heartbeat(d, ctx.agent, await ctx.body())));
|
||||
app.get('/agent/v1/jobs/next', wrap(async (ctx) => { const wait = Math.min(25, Number(ctx.query.wait || 0)); const until = Date.now() + wait * 1000; while (true) { const j = core.nextAgentJob(d, ctx.agent.id); if (j || Date.now() >= until || d.stopping) return { ok: true, job: j || null, serverTime: nowIso() }; await new Promise((r) => setTimeout(r, 500)); } }));
|
||||
app.get('/agent/v1/jobs/unfinished', wrap((ctx) => ({ ok: true, jobs: core.unfinishedAgentJobs(d, ctx.agent.id) })));
|
||||
app.post('/agent/v1/jobs/:id/progress', wrap(async (ctx) => { const b = await ctx.body(); const ok = core.agentJobProgress(d, ctx.params.id, ctx.agent.id, b); return { ok, cancelRequested: core.isAgentJobCancelled(d, ctx.params.id) }; }));
|
||||
app.post('/agent/v1/jobs/:id/result', wrap(async (ctx) => { const b = await ctx.body(); const ok = core.agentJobResult(d, ctx.params.id, ctx.agent.id, b); if (!ok) throw new HttpError(409, 'job_not_running', '실행 중인 작업이 아닙니다'); return { ok }; }));
|
||||
app.get('/agent/v1/releases/:version/artifact', wrap((ctx) => { const a = core.artifactPath(d, ctx.params.version, ctx.query.module); if (!a) throw new HttpError(404, 'artifact_not_found', 'artifact 가 없습니다'); ctx.ended = true; ctx.res.writeHead(200, { 'Content-Type': 'application/gzip', 'Content-Length': a.size, 'X-Sha256': a.sha256, 'Cache-Control': 'no-store' }); createReadStream(a.path).pipe(ctx.res); }));
|
||||
app.post('/agent/v1/transfers/:id', wrap((ctx) => new Promise((resolve, reject) => {
|
||||
const id = ctx.params.id; if (!/^tr_[A-Za-z0-9_.-]{2,80}$/.test(id)) return reject(new HttpError(400, 'bad_transfer_id', '전송 id 형식이 올바르지 않습니다'));
|
||||
const expected = String(ctx.headers['x-transfer-sha256'] || ''); const path = join(core.transferDir(d), id + '.bin'); const hash = createHash('sha256'); let size = 0; const limit = 20 * 1024 ** 3;
|
||||
const out = createWriteStream(path, { mode: 0o600 }); ctx.req.on('data', (c) => { size += c.length; hash.update(c); if (size > limit) ctx.req.destroy(new Error('too large')); }); ctx.req.on('error', (e) => { out.destroy(); reject(new HttpError(413, 'transfer_too_large', e.message)); });
|
||||
ctx.req.pipe(out); out.on('finish', () => { const sum = hash.digest('hex'); if (expected && expected !== sum) return reject(new HttpError(400, 'checksum_mismatch', '전송 파일 해시가 다릅니다')); core.createTransfer(d, { id, agentId: ctx.agent.id, sha256: sum, size, purpose: String(ctx.query.purpose || '') }); resolve({ ok: true, id, sha256: sum, size }); });
|
||||
})));
|
||||
app.get('/agent/v1/transfers/:id', wrap((ctx) => { const t = core.getTransfer(d, ctx.params.id); if (!t) throw new HttpError(404, 'transfer_not_found', '전송 파일이 없거나 만료되었습니다'); ctx.ended = true; ctx.res.writeHead(200, { 'Content-Type': 'application/octet-stream', 'Content-Length': statSync(t.path).size, 'X-Sha256': t.sha256, 'Cache-Control': 'no-store' }); createReadStream(t.path).pipe(ctx.res); }));
|
||||
|
||||
// ---------------- 포털 API
|
||||
app.use(async (ctx) => { if (!ctx.path.startsWith('/portal/v1/')) return; const r = await verifyCtx(ctx, (k) => k === 'portal' ? d.vault.get('service:portal') : null); if (!r.ok) throw new HttpError(401, 'service_auth_failed', `서비스 인증 실패(${r.reason})`); if (ctx.path !== '/portal/v1/health' && !core.bootstrapComplete(d)) throw new HttpError(503, 'bootstrap_incomplete', 'HAL 부트스트랩(HAL 설치·첫 Agent 등록)이 끝나지 않았습니다'); });
|
||||
app.get('/portal/v1/health', () => ({ ok: true, ...core.selfHealth(d) }));
|
||||
app.post('/portal/v1/jobs', wrap(async (ctx) => { const b = await ctx.body(); if (!d.jobs.kinds().includes(b.kind) || b.kind.startsWith('module.')) throw new HttpError(400, 'bad_kind', `포털이 요청할 수 없는 작업 종류: ${b.kind}`); const { job, created } = d.jobs.enqueue({ kind: b.kind, subject: { target: b.target, payload: b.payload || {}, _portal: true }, idempotencyKey: b.idempotencyKey || null, createdBy: 'portal', actorType: 'system', groupId: b.groupId || null, subjectType: b.subjectType || null, subjectId: b.subjectId || null, label: b.label || null }); return { ok: true, halJobId: job.id, status: job.status, created }; }));
|
||||
const jobView = (j) => { if (!j) throw new HttpError(404, 'job_not_found', '작업이 없습니다'); const dsc = describe(j); return { ...dsc, result: j.result, children: j.kind.startsWith('module.') ? d.jobs.childrenSummary(j.id) : undefined }; };
|
||||
app.get('/portal/v1/jobs/:id', wrap((ctx) => ({ ok: true, ...jobView(d.jobs.get(ctx.params.id)) })));
|
||||
for (const act of ['cancel', 'retry', 'resume', 'revert']) app.post(`/portal/v1/jobs/:id/${act}`, wrap(async (ctx) => { const j = await d.jobs[act](ctx.params.id, 'portal'); return { ok: true, ...jobView(j) }; }));
|
||||
app.get('/portal/v1/servers', wrap((ctx) => ({ ok: true, servers: core.listServers(d, { role: ctx.query.role || null }).map(stripFacts) })));
|
||||
app.post('/portal/v1/servers/pick', wrap(async (ctx) => { const b = await ctx.body(); return { ok: true, pick: core.pickServer(d, { role: b.role || 'web', needBytes: Number(b.needBytes || 0) }) }; }));
|
||||
app.post('/portal/v1/external-agents/token', wrap(async (ctx) => { const b = await ctx.body(); if (!b.siteId || !b.groupId) throw new HttpError(400, 'site_required', 'siteId·groupId 가 필요합니다'); const t = core.issueRegistrationToken(d, { kind: 'external', groupId: b.groupId, siteId: b.siteId, label: b.label || '', by: null }); return { ok: true, agentId: null, token: t.token, expiresAt: t.expiresAt, installCommand: t.installCommand }; }));
|
||||
app.get('/portal/v1/external-agents/by-site/:siteId', wrap((ctx) => { const a = d.store.get('SELECT id FROM agents WHERE kind=\'external\' AND site_id=? AND status=\'active\' ORDER BY registered_at DESC LIMIT 1', ctx.params.siteId); return { ok: true, agent: a ? core.externalAgentStatus(d, a.id) : null }; }));
|
||||
app.get('/portal/v1/external-agents/:id', wrap((ctx) => { const s = core.externalAgentStatus(d, ctx.params.id); if (!s) throw new HttpError(404, 'agent_not_found', '외부 Agent 가 없습니다'); return { ok: true, agent: s }; }));
|
||||
app.post('/portal/v1/external-agents/:id/revoke', wrap((ctx) => { core.revokeAgent(d, ctx.params.id, 'portal'); return { ok: true }; }));
|
||||
|
||||
// ---------------- 운영자 API(§25)
|
||||
app.use(async (ctx) => {
|
||||
if (!ctx.path.startsWith('/ops')) return;
|
||||
const gate = ops.networkGate(d, ctx);
|
||||
if (ctx.path === '/ops/access') { const code = ctx.query.code; try { const g = ops.bindGrant(d, code, { ua: ctx.headers['user-agent'], ip: ctx.ip }); ctx.setCookie(ops.grantCookie(g.cookie, ctx.secure)); return ctx.redirect('/ops/'); } catch (e) { throw new HttpError(403, 'access_denied', '접근할 수 없습니다'); } }
|
||||
if (!gate.ok) throw new HttpError(403, 'access_denied', '접근할 수 없습니다'); // 로그인 화면도 주지 않는다(§25)
|
||||
ctx.opsVia = gate.via;
|
||||
const tok = ctx.cookies[session.cookieName('operator')]; ctx.session = session.resolveSession(d.store, tok, 'operator'); ctx.operator = ctx.session ? ops.publicAccount(d.store.get('SELECT * FROM ops_accounts WHERE id=?', ctx.session.user_id)) : null;
|
||||
if (ctx.path.startsWith('/ops/api/') && !['/ops/api/login', '/ops/api/setup', '/ops/api/me'].includes(ctx.path)) { if (!ctx.operator) throw new HttpError(401, 'unauthenticated', '운영자 로그인이 필요합니다'); if (!['GET', 'HEAD'].includes(ctx.method) && !session.csrfOk(ctx.session, ctx.headers['x-csrf'])) throw new HttpError(403, 'csrf', '보안 토큰이 맞지 않습니다'); }
|
||||
});
|
||||
const requireRecent = (ctx) => { if (!session.recentlyAuthed(ctx.session)) throw new HttpError(403, 'reauth_required', '고위험 작업은 최근 10분 안의 재인증이 필요합니다', { action: 'reauth' }); };
|
||||
/** 고위험 작업 3단계 확인(§25): 영향 범위 확인 → 대상 식별정보 재입력 → 최종 실행 */
|
||||
const threeStep = (b, expectIds) => { if (b.confirmImpact !== true) throw new HttpError(400, 'confirm_impact_required', '1단계: 영향 범위를 확인해야 합니다'); const ids = [].concat(b.confirmIdentifiers || []).sort(); if (JSON.stringify(ids) !== JSON.stringify([].concat(expectIds).sort())) throw new HttpError(400, 'identifiers_mismatch', '2단계: 대상 식별정보가 다릅니다'); if (b.confirmExecute !== true) throw new HttpError(400, 'confirm_execute_required', '3단계: 최종 실행을 확인해야 합니다'); };
|
||||
app.post('/ops/api/setup', wrap(async (ctx) => { const b = await ctx.body(); const st = core.bootstrapState(d); const tokFile = join(d.config.dataDir, 'setup.token'); const expected = existsSync(tokFile) ? readFileSync(tokFile, 'utf8').trim() : ''; if (core.ensureOpsAccountExists(d) || !expected || b.setupToken !== expected) throw new HttpError(403, 'setup_closed', '최초 설정은 설정 토큰으로 한 번만 할 수 있습니다'); const a = ops.createAccount(d, { login: b.login, password: b.password, displayName: b.displayName, by: null }); try { (await import('node:fs')).unlinkSync(tokFile); } catch (_) {} return { ok: true, account: a, bootstrap: st }; }));
|
||||
app.post('/ops/api/login', wrap(async (ctx) => { const b = await ctx.body(); const r = ops.login(d, { login: b.login, password: b.password, totp: b.totp, ip: ctx.ip, ua: ctx.headers['user-agent'] }); if (r.mfaRequired) return { ok: true, mfaRequired: true, message: r.message }; ctx.setCookie(cookie(session.cookieName('operator'), r.session.token, { maxAge: 86400, path: '/ops', secure: ctx.secure })); return { ok: true, account: r.account, csrf: r.session.csrf, via: ctx.opsVia }; }));
|
||||
app.get('/ops/api/me', (ctx) => ({ ok: true, operator: ctx.operator, csrf: ctx.session?.csrf || null, via: ctx.opsVia, recentlyAuthed: session.recentlyAuthed(ctx.session), bootstrap: core.selfHealth(d).bootstrap, setupNeeded: !core.ensureOpsAccountExists(d) }));
|
||||
app.post('/ops/api/logout', (ctx) => { session.revokeSession(d.store, ctx.session.id, ctx.session.user_id); ctx.setCookie(cookie(session.cookieName('operator'), '', { path: '/ops' })); return { ok: true }; });
|
||||
app.post('/ops/api/reauth', wrap(async (ctx) => { const b = await ctx.body(); ops.reauth(d, ctx.operator.id, b.password, ctx.session.id); return { ok: true }; }));
|
||||
app.post('/ops/api/password', wrap(async (ctx) => { const b = await ctx.body(); ops.changePassword(d, ctx.operator.id, b.current, b.next, ctx.session.id); return { ok: true }; }));
|
||||
app.post('/ops/api/mfa/begin', wrap((ctx) => ({ ok: true, ...ops.mfaBegin(d, ctx.operator.id) })));
|
||||
app.post('/ops/api/mfa/confirm', wrap(async (ctx) => ({ ok: true, ...ops.mfaConfirm(d, ctx.operator.id, (await ctx.body()).code) })));
|
||||
app.post('/ops/api/mfa/disable', wrap(async (ctx) => { requireRecent(ctx); ops.mfaDisable(d, ctx.operator.id, (await ctx.body()).code); return { ok: true }; }));
|
||||
app.get('/ops/api/sessions', (ctx) => ({ ok: true, sessions: session.listSessions(d.store, ctx.operator.id, 'operator').map((s) => ({ ...s, current: s.id === ctx.session.id })) }));
|
||||
app.post('/ops/api/sessions/revoke', wrap(async (ctx) => { const b = await ctx.body(); const n = b.all ? session.revokeOtherSessions(d.store, ctx.operator.id, ctx.session.id, 'operator') : session.revokeSession(d.store, b.id, ctx.operator.id); return { ok: true, revoked: n }; }));
|
||||
app.get('/ops/api/accounts', (ctx) => ({ ok: true, accounts: d.store.all('SELECT * FROM ops_accounts ORDER BY created_at').map(ops.publicAccount) }));
|
||||
app.post('/ops/api/accounts', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); return { ok: true, account: ops.createAccount(d, { ...b, by: ctx.operator.id, mustChange: true }) }; }));
|
||||
app.post('/ops/api/access-grants', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); return { ok: true, grant: ops.issueGrant(d, { accountId: b.accountId || ctx.operator.id, hours: b.hours || 8, by: ctx.operator.id, reason: b.reason || '' }) }; }));
|
||||
app.get('/ops/api/access-grants', (ctx) => ({ ok: true, grants: d.store.all('SELECT id, account_id, created_by, created_at, expires_at, bound_at, revoked_at, reason, ip FROM ops_access_grants ORDER BY created_at DESC LIMIT 200') }));
|
||||
app.post('/ops/api/access-grants/:id/revoke', wrap((ctx) => { ops.revokeGrant(d, ctx.params.id, ctx.operator.id); return { ok: true }; }));
|
||||
// 운영 현황(§26.1)
|
||||
app.get('/ops/api/overview', (ctx) => overview(d));
|
||||
app.get('/ops/api/config', () => ({ ok: true, config: { publicUrl: d.config.publicUrl, portalUrl: d.config.portalUrl, opsAllowCidrs: d.config.opsAllowCidrs, metricsRetentionDays: d.config.metricsRetentionDays, offlineAfterMs: d.config.offlineAfterMs, emergencyTtlMinutes: d.config.emergencyTtlMinutes } }));
|
||||
app.put('/ops/api/config', wrap(async (ctx) => { requireRecent(ctx); d.config.set(await ctx.body(), ctx.operator.id); return { ok: true }; }));
|
||||
// 서버(§28)
|
||||
app.get('/ops/api/servers', () => ({ ok: true, servers: core.listServers(d) }));
|
||||
app.post('/ops/api/servers', wrap(async (ctx) => { const b = await ctx.body(); const s = core.createServer(d, { ...b, by: ctx.operator.id }); return { ok: true, server: core.publicServer(d, s) }; }));
|
||||
app.get('/ops/api/servers/:id', wrap((ctx) => { const s = core.getServer(d, ctx.params.id); if (!s) throw new HttpError(404, 'server_not_found', '서버가 없습니다'); return { ok: true, server: core.publicServer(d, s), jobs: d.jobs.list({ subjectType: 'server', subjectId: s.id, limit: 50 }).map(describe), agentJobs: d.store.all('SELECT id, kind, status, progress, created_at, ended_at, error FROM agent_jobs WHERE agent_id=? ORDER BY created_at DESC LIMIT 50', s.agent_id || '') }; }));
|
||||
app.get('/ops/api/servers/:id/metrics', (ctx) => ({ ok: true, samples: core.metrics(d, ctx.params.id, { since: ctx.query.since, until: ctx.query.until, limit: Number(ctx.query.limit || 5000) }) }));
|
||||
app.patch('/ops/api/servers/:id', wrap(async (ctx) => { const b = await ctx.body(); if (b.status === 'isolated' || b.role) requireRecent(ctx); return { ok: true, server: core.publicServer(d, core.updateServer(d, ctx.params.id, b, ctx.operator.id)) }; }));
|
||||
app.post('/ops/api/servers/:id/token', wrap((ctx) => ({ ok: true, ...core.issueRegistrationToken(d, { kind: 'managed', serverId: ctx.params.id, by: ctx.operator.id }) })));
|
||||
app.post('/ops/api/servers/:id/approve', wrap((ctx) => ({ ok: true, server: core.publicServer(d, core.approveServer(d, ctx.params.id, ctx.operator.id)) })));
|
||||
app.post('/ops/api/servers/:id/maintenance', wrap(async (ctx) => { const b = await ctx.body(); const s = core.updateServer(d, ctx.params.id, b.end ? { status: 'ready', accepts_new: b.acceptsNew === false ? 0 : 1, maintenance_note: null } : { status: 'maintenance', accepts_new: 0, maintenance_note: b.note || '' }, ctx.operator.id); return { ok: true, server: core.publicServer(d, s) }; }));
|
||||
app.post('/ops/api/servers/:id/delete', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); const s = core.getServer(d, ctx.params.id); if (!s) throw new HttpError(404, 'server_not_found', '서버가 없습니다'); threeStep(b, [s.id]); const hosted = (s.capacity && s.capacity.sitesHosted) || 0; if (hosted > 0 && !b.force) throw new HttpError(409, 'server_in_use', `이 서버에 ${hosted}개 서비스가 배치되어 있습니다 — 먼저 이전하세요`); if (s.agent_id) core.revokeAgent(d, s.agent_id, ctx.operator.id); d.store.run('DELETE FROM servers WHERE id=?', s.id); d.store.run('DELETE FROM module_targets WHERE server_id=?', s.id); listAudit; return { ok: true }; }));
|
||||
// 릴리스·모듈 배포(§32)
|
||||
app.get('/ops/api/releases', () => ({ ok: true, releases: core.listReleases(d), running: core.selfHealth(d).bootstrap, modules: Object.keys(MODULES), deployOrder: DEPLOY_ORDER, targets: d.store.all('SELECT * FROM module_targets ORDER BY module, server_id') }));
|
||||
app.post('/ops/api/releases/:version/approve', wrap((ctx) => { requireRecent(ctx); return { ok: true, release: core.approveRelease(d, ctx.params.version, ctx.operator.id) }; }));
|
||||
app.post('/ops/api/modules/:name/deploy', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); const r = d.jobs.enqueue({ kind: 'module.deploy', subject: { module: ctx.params.name, version: b.version, serverIds: b.serverIds || [] }, idempotencyKey: b.idempotencyKey || `module.deploy:${ctx.params.name}:${b.version}:${(b.serverIds || []).slice().sort().join(',')}:${b.nonce || ''}`, createdBy: ctx.operator.id, actorType: 'operator', subjectType: 'module', subjectId: ctx.params.name, label: `${ctx.params.name} ${b.version} 배포` }); return { ok: true, job: describe(r.job), created: r.created }; }));
|
||||
app.post('/ops/api/modules/:name/rollback', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); threeStep(b, b.serverIds || []); const r = d.jobs.enqueue({ kind: 'module.rollback', subject: { module: ctx.params.name, toVersion: b.toVersion || null, serverIds: b.serverIds || [] }, createdBy: ctx.operator.id, actorType: 'operator', subjectType: 'module', subjectId: ctx.params.name, label: `${ctx.params.name} 되돌리기` }); return { ok: true, job: describe(r.job) }; }));
|
||||
app.post('/ops/api/modules/:name/health', wrap(async (ctx) => { const b = await ctx.body(); const r = d.jobs.enqueue({ kind: 'module.health', subject: { module: ctx.params.name, serverIds: b.serverIds || null }, createdBy: ctx.operator.id, actorType: 'operator', subjectType: 'module', subjectId: ctx.params.name, label: `${ctx.params.name} 건강 확인` }); return { ok: true, job: describe(r.job) }; }));
|
||||
app.post('/ops/api/modules/reconcile', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); const r = d.jobs.enqueue({ kind: 'module.reconcile', subject: { module: b.module || null }, createdBy: ctx.operator.id, actorType: 'operator', label: '목표·실제 상태 맞추기' }); return { ok: true, job: describe(r.job) }; }));
|
||||
// 작업(§29)
|
||||
app.get('/ops/api/jobs', (ctx) => ({ ok: true, jobs: d.jobs.list({ status: ctx.query.status ? ctx.query.status.split(',') : null, kind: ctx.query.kind || null, limit: Number(ctx.query.limit || 200) }).map((j) => ({ ...describe(j), subject: redact(j.subject), createdBy: j.created_by, actorType: j.actor_type, groupId: j.group_id })) }));
|
||||
app.get('/ops/api/jobs/:id', wrap((ctx) => { const j = d.jobs.get(ctx.params.id); if (!j) throw new HttpError(404, 'job_not_found', '작업이 없습니다'); return { ok: true, job: { ...describe(j), subject: redact(j.subject), result: j.result, log: j.log, createdBy: j.created_by, actorType: j.actor_type, groupId: j.group_id }, children: d.jobs.childrenSummary(j.id), agentJobs: d.store.all('SELECT id, agent_id, kind, status, progress, error, user_message, created_at, ended_at FROM agent_jobs WHERE hal_job_id=? ORDER BY created_at', j.id) }; }));
|
||||
for (const act of ['cancel', 'retry', 'resume', 'revert', 'pause']) app.post(`/ops/api/jobs/:id/${act}`, wrap(async (ctx) => { if (act === 'revert') requireRecent(ctx); const j = await d.jobs[act](ctx.params.id, ctx.operator.id); return { ok: true, job: describe(j) }; }));
|
||||
// 사건·알림(§31)
|
||||
app.get('/ops/api/incidents', () => ({ ok: true, incidents: d.store.all('SELECT * FROM incidents ORDER BY started_at DESC LIMIT 200'), notifications: listNotifications(d.store, { groupId: null, limit: 200 }) }));
|
||||
app.post('/ops/api/incidents', wrap(async (ctx) => { const b = await ctx.body(); const open = d.store.get('SELECT id FROM incidents WHERE key=? AND resolved_at IS NULL', b.key); if (open) { d.store.run('UPDATE incidents SET severity=?, title=?, customer_impact=?, updated_at=? WHERE id=?', b.severity || 'warning', b.title, b.customerImpact || null, nowIso(), open.id); return { ok: true, id: open.id, merged: true }; } const id = newId('inc_'); d.store.run('INSERT INTO incidents(id, key, severity, title, component, customer_impact, started_at, assignee, public, public_note, notes_json, updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,\'[]\',?)', id, b.key, b.severity || 'warning', b.title, b.component || null, b.customerImpact || null, nowIso(), ctx.operator.id, b.public ? 1 : 0, b.publicNote || null, nowIso()); return { ok: true, id }; }));
|
||||
app.patch('/ops/api/incidents/:id', wrap(async (ctx) => { const b = await ctx.body(); const inc = d.store.get('SELECT * FROM incidents WHERE id=?', ctx.params.id); if (!inc) throw new HttpError(404, 'not_found', '사건이 없습니다'); if (b.resolve && !b.verifiedUserPath) throw new HttpError(400, 'verify_required', '실제 사용자 경로 확인 뒤에 종료할 수 있습니다(§31.2)'); const notes = parseJson(inc.notes_json, []); if (b.note) notes.push({ at: nowIso(), by: ctx.operator.id, note: b.note }); d.store.run('UPDATE incidents SET severity=COALESCE(?, severity), assignee=COALESCE(?, assignee), public=COALESCE(?, public), public_note=COALESCE(?, public_note), notes_json=?, resolved_at=CASE WHEN ? THEN ? ELSE resolved_at END, verified_at=CASE WHEN ? THEN ? ELSE verified_at END, updated_at=? WHERE id=?', b.severity || null, b.assignee || null, b.public == null ? null : (b.public ? 1 : 0), b.publicNote || null, toJson(notes), b.resolve ? 1 : 0, nowIso(), b.resolve ? 1 : 0, nowIso(), nowIso(), inc.id); return { ok: true }; }));
|
||||
app.get('/ops/api/audit', (ctx) => ({ ok: true, entries: listAudit(d.store, { action: ctx.query.action, subjectType: ctx.query.subjectType, subjectId: ctx.query.subjectId, since: ctx.query.since, limit: Number(ctx.query.limit || 200) }) }));
|
||||
// 긴급 복구(§33)
|
||||
app.get('/ops/api/emergency', () => ({ ok: true, sessions: emergency.listSessions(d), scopes: emergency.SCOPES, actions: emergency.ACTIONS }));
|
||||
app.post('/ops/api/emergency', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); return { ok: true, session: emergency.declare(d, { ...b, by: ctx.operator.id }) }; }));
|
||||
app.post('/ops/api/emergency/:id/approve', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); return { ok: true, session: emergency.approve(d, ctx.params.id, { ...b, by: ctx.operator.id }) }; }));
|
||||
app.post('/ops/api/emergency/:id/credential', wrap((ctx) => { requireRecent(ctx); return { ok: true, ...emergency.issueCredential(d, ctx.params.id, ctx.operator.id) }; }));
|
||||
app.post('/ops/api/emergency/:id/perform', wrap(async (ctx) => { const b = await ctx.body(); return { ok: true, result: await emergency.perform(d, ctx.params.id, { ...b, by: ctx.operator.id }) }; }));
|
||||
app.post('/ops/api/emergency/:id/verify', wrap(async (ctx) => ({ ok: true, session: emergency.verifyAndRevoke(d, ctx.params.id, { by: ctx.operator.id, verify: (await ctx.body()).verify }) })));
|
||||
app.post('/ops/api/emergency/:id/reconcile', wrap((ctx) => ({ ok: true, result: emergency.reconcile(d, ctx.params.id, ctx.operator.id) })));
|
||||
app.post('/ops/api/emergency/:id/close', wrap(async (ctx) => ({ ok: true, session: emergency.close(d, ctx.params.id, ctx.operator.id, (await ctx.body()).note) })));
|
||||
// 재해복구(§34)
|
||||
app.get('/ops/api/dr', () => ({ ok: true, readiness: dr.readiness(d), snapshots: dr.listSnapshots(d), restores: d.store.all('SELECT * FROM dr_restores ORDER BY rowid DESC LIMIT 50'), drills: d.store.all('SELECT * FROM dr_drills ORDER BY started_at DESC LIMIT 50') }));
|
||||
app.put('/ops/api/dr/config', wrap(async (ctx) => { requireRecent(ctx); return { ok: true, config: dr.setConfig(d, await ctx.body(), ctx.operator.id) }; }));
|
||||
app.post('/ops/api/dr/snapshots', wrap((ctx) => ({ ok: true, snapshot: dr.snapshot(d, { kind: 'manual', by: ctx.operator.id }) })));
|
||||
app.post('/ops/api/dr/restores', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); return { ok: true, restore: dr.requestRestore(d, { snapshotId: b.snapshotId, by: ctx.operator.id, note: b.note }) }; }));
|
||||
app.post('/ops/api/dr/restores/:id/approve', wrap((ctx) => { requireRecent(ctx); return { ok: true, restore: dr.approveRestore(d, ctx.params.id, ctx.operator.id) }; }));
|
||||
app.post('/ops/api/dr/restores/:id/start', wrap(async (ctx) => { requireRecent(ctx); const b = await ctx.body(); return { ok: true, ...dr.startRestore(d, ctx.params.id, { by: ctx.operator.id, oldControlStopped: b.oldControlStopped, confirmText: b.confirmText }) }; }));
|
||||
app.post('/ops/api/dr/restores/:id/finish', wrap(async (ctx) => ({ ok: true, restore: dr.finishRestore(d, ctx.params.id, { by: ctx.operator.id, checks: (await ctx.body()).checks }) })));
|
||||
app.post('/ops/api/dr/drills', wrap(async (ctx) => ({ ok: true, drill: dr.startDrill(d, { by: ctx.operator.id, snapshotId: (await ctx.body()).snapshotId }) })));
|
||||
app.post('/ops/api/dr/drills/:id/finish', wrap(async (ctx) => ({ ok: true, drill: dr.finishDrill(d, ctx.params.id, { by: ctx.operator.id, ...(await ctx.body()) }) })));
|
||||
// 포털 운영 화면 중계(§25·§26.2): 포털의 /internal/ops/* 를 운영자 세션으로 대신 부른다
|
||||
app.route('*', '/ops/api/portal/*', wrap(async (ctx) => { const secret = d.vault.get('service:hal'); if (!secret) throw new HttpError(503, 'portal_not_linked', '포털 연동 키가 없습니다'); if (!['GET', 'HEAD'].includes(ctx.method)) requireRecent(ctx); const rest = ctx.path.replace(/^\/ops\/api\/portal/, '') + (ctx.url.search || ''); const body = ['GET', 'HEAD'].includes(ctx.method) ? undefined : { ...(await ctx.body()), _operator: { id: ctx.operator.id, login: ctx.operator.login } }; const { serviceFetch } = d.serviceauth; const r = await serviceFetch(d.config.portalUrl, 'hal', secret, ctx.method, '/internal/ops' + rest, body, { timeoutMs: 20000, fetchImpl: d.fetch || fetch }).catch((e) => { throw new HttpError(503, 'portal_unreachable', '포털에 연결할 수 없습니다 — 정보 확인 지연'); }); ctx.status = r.status; return r.json || { ok: false, error: 'portal_bad_response' }; }));
|
||||
app.get('/ops', (ctx) => ctx.redirect('/ops/')); app.get('/ops/', (ctx) => { const f = join(uiDir, 'index.html'); return ctx.html(existsSync(f) ? readFileSync(f, 'utf8') : `<!doctype html><html lang="ko"><meta charset="utf-8"><title>Ariweb HAL</title><body style="font-family:sans-serif;padding:40px"><h1>Ariweb HAL 운영자 콘솔</h1><p>화면(UI)은 아직 설치되지 않았습니다. API: /ops/api/me</p></body></html>`); });
|
||||
return app;
|
||||
}
|
||||
function stripFacts(s) { const { facts, ...rest } = s; return rest; }
|
||||
/** 운영 현황(§26.1) */
|
||||
function overview(d) {
|
||||
const servers = core.listServers(d); const health = core.selfHealth(d);
|
||||
const jobs = d.jobs.list({ status: ['queued', 'running', 'awaiting_external', 'needs_user', 'reverting'], limit: 500 });
|
||||
const comp = (name, ok, extra = {}) => ({ name, status: ok === null ? 'unknown' : ok ? 'ok' : 'down', ...extra });
|
||||
const modState = (m) => { const t = d.store.all('SELECT health, actual_version, last_check_at FROM module_targets WHERE module=?', m); if (!t.length) return null; const stale = t.some((x) => !x.last_check_at || Date.now() - Date.parse(x.last_check_at) > 10 * 60 * 1000); return { ok: t.every((x) => x.health === 'ok'), stale, versions: [...new Set(t.map((x) => x.actual_version))] }; };
|
||||
const components = ['hal', 'portal', 'webmaker', 'arimail', 'backup', 'observability'].map((m) => { if (m === 'hal') return comp('HAL', health.ok, { version: health.version }); const s = modState(m); return s ? comp(m, s.stale ? null : s.ok, { versions: s.versions, stale: s.stale }) : comp(m, null, { note: '배치 없음' }); });
|
||||
return { ok: true, hal: health, components, servers: { total: servers.length, offline: servers.filter((s) => s.status === 'offline').length, pending: servers.filter((s) => s.status === 'pending').length, maintenance: servers.filter((s) => s.rawStatus === 'maintenance').length, diskWarning: servers.filter((s) => s.capacity.diskTotalBytes && (s.capacity.diskTotalBytes - s.capacity.diskFreeBytes) / s.capacity.diskTotalBytes >= 0.9).length },
|
||||
jobs: { running: jobs.filter((j) => j.status === 'running').length, waiting: jobs.filter((j) => ['awaiting_external', 'queued'].includes(j.status)).length, needsUser: jobs.filter((j) => j.status === 'needs_user').length, byKind: jobs.reduce((a, j) => { a[j.kind] = (a[j.kind] || 0) + 1; return a; }, {}) },
|
||||
incidents: d.store.all('SELECT id, key, severity, title, started_at, customer_impact FROM incidents WHERE resolved_at IS NULL ORDER BY started_at DESC'), drift: d.store.all('SELECT module, server_id, target_version, actual_version FROM module_targets WHERE target_version IS NOT NULL AND (actual_version IS NULL OR actual_version<>target_version)'), dr: dr.readiness(d), emergencyOpen: d.store.all('SELECT id, status, incident_key FROM emergency_sessions WHERE status<>\'closed\''), generatedAt: nowIso() };
|
||||
}
|
||||
/** 공개 상태(§31.3): 내부 이름·고객 정보 없음. 고객 영향이 없는 내부 경고는 올리지 않는다. */
|
||||
function publicStatus(d) {
|
||||
const o = overview(d);
|
||||
const label = { HAL: '관리 플랫폼', portal: '고객 포털', webmaker: 'Webmaker', arimail: '메일', backup: '백업', observability: '모니터링' };
|
||||
return { ok: true, generatedAt: o.generatedAt, components: o.components.map((c) => ({ name: label[c.name] || c.name, status: c.status === 'ok' ? 'ok' : c.status === 'down' ? 'down' : 'unknown', label: c.status === 'ok' ? '정상' : c.status === 'down' ? '장애' : '확인 중' })), incidents: o.incidents.filter((i) => d.store.get('SELECT public FROM incidents WHERE id=?', i.id)?.public).map((i) => ({ severity: i.severity, title: i.title, startedAt: i.started_at, note: d.store.get('SELECT public_note FROM incidents WHERE id=?', i.id)?.public_note || '' })), maintenance: d.store.all('SELECT name, maintenance_note FROM servers WHERE status=\'maintenance\'').map((s) => ({ note: s.maintenance_note || '계획된 유지보수' })) };
|
||||
}
|
||||
function statusPage(d) { const s = publicStatus(d); return `<!doctype html><html lang="ko"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Ariweb 서비스 상태</title><style>body{font-family:-apple-system,"Noto Sans KR",sans-serif;max-width:720px;margin:40px auto;padding:0 16px;color:#1f2933}li{padding:8px 0;border-bottom:1px solid #eee}.ok{color:#1a7f4b}.down{color:#b3261e}.unknown{color:#b26a00}</style></head><body><h1>Ariweb 서비스 상태</h1><p>갱신 ${esc(s.generatedAt)}</p><ul>${s.components.map((c) => `<li><b class="${c.status}">${c.status === 'ok' ? '●' : c.status === 'down' ? '✘' : '◔'} ${esc(c.label)}</b> — ${esc(c.name)}</li>`).join('')}</ul><h2>진행 중인 사건</h2>${s.incidents.length ? `<ul>${s.incidents.map((i) => `<li><b>${esc(i.title)}</b> (${esc(i.startedAt)})<br>${esc(i.note)}</li>`).join('')}</ul>` : '<p>진행 중인 사건이 없습니다.</p>'}<h2>계획된 유지보수</h2>${s.maintenance.length ? `<ul>${s.maintenance.map((m) => `<li>${esc(m.note)}</li>`).join('')}</ul>` : '<p>예정된 유지보수가 없습니다.</p>'}</body></html>`; }
|
||||
@@ -0,0 +1,21 @@
|
||||
/** hal/cidr.mjs — IP 허용 목록(CIDR) 검사(§25 운영자 콘솔 네트워크 제한). IPv4·IPv6·::ffff: 로 감싼 IPv4 를 다룬다. */
|
||||
import { isIP } from 'node:net';
|
||||
|
||||
function toBytes(ip) {
|
||||
let s = String(ip || '').trim(); if (s.startsWith('::ffff:') && isIP(s.slice(7)) === 4) s = s.slice(7);
|
||||
if (isIP(s) === 4) return { v: 4, b: s.split('.').map(Number) };
|
||||
if (isIP(s) !== 6) return null;
|
||||
const [head, tail] = s.split('::'); const h = head ? head.split(':') : []; const t = tail ? tail.split(':') : []; const parts = [...h, ...Array(8 - h.length - t.length).fill('0'), ...t];
|
||||
const b = []; for (const p of parts) { const n = parseInt(p || '0', 16); b.push(n >> 8, n & 255); } return { v: 6, b };
|
||||
}
|
||||
/** ip 가 cidr 목록 중 하나에 속하면 true. 목록이 비면 false(허용 네트워크 없음). */
|
||||
export function ipAllowed(ip, cidrs) {
|
||||
const a = toBytes(ip); if (!a) return false;
|
||||
for (const c of cidrs || []) {
|
||||
const [net, lenS] = String(c).split('/'); const n = toBytes(net); if (!n || n.v !== a.v) continue;
|
||||
const len = lenS === undefined ? (n.v === 4 ? 32 : 128) : Number(lenS); if (!(len >= 0 && len <= (n.v === 4 ? 32 : 128))) continue;
|
||||
let ok = true; for (let i = 0; i < len; i++) { const byte = i >> 3, bit = 7 - (i & 7); if (((a.b[byte] >> bit) & 1) !== ((n.b[byte] >> bit) & 1)) { ok = false; break; } }
|
||||
if (ok) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* hal/config.mjs — HAL 실행 설정.
|
||||
*
|
||||
* 배포 때 정해지는 값(포트·데이터 디렉터리)은 환경 변수로, 운영 중 바꾸는 값(공개 주소·포털 주소·허용 네트워크·메트릭 보존 일수)은
|
||||
* data/config.json 에 둔다. 비밀값(서비스 키·Agent 비밀)은 설정이 아니라 보관소(lib/vault)에만 둔다.
|
||||
*/
|
||||
import { readFileSync, existsSync, writeFileSync, mkdirSync, renameSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
export function loadConfig() {
|
||||
const dataDir = process.env.ARIWEB_HAL_DATA || process.env.DATA_DIR || join(process.cwd(), 'data');
|
||||
mkdirSync(dataDir, { recursive: true, mode: 0o700 });
|
||||
const file = join(dataDir, 'config.json');
|
||||
let values = {}; try { values = existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')).values || {} : {}; } catch (_) { values = {}; }
|
||||
const cfg = {
|
||||
dataDir, file,
|
||||
port: Number(process.env.PORT || 8900), host: process.env.HOST || '127.0.0.1',
|
||||
/** HAL 공개 주소(Agent 가 접속하는 주소) */
|
||||
publicUrl: values.publicUrl || process.env.HAL_PUBLIC_URL || 'https://hal.ariweb.net',
|
||||
/** 포털 내부 주소(작업 이벤트를 미는 곳) */
|
||||
portalUrl: values.portalUrl || process.env.PORTAL_URL || 'http://127.0.0.1:8910',
|
||||
/** 운영자 콘솔 접근 허용 네트워크(CIDR 목록, §25). 비어 있으면 접근권(grant) 쿠키만 허용 */
|
||||
opsAllowCidrs: values.opsAllowCidrs || (process.env.OPS_ALLOW_CIDRS ? process.env.OPS_ALLOW_CIDRS.split(',').map((s) => s.trim()).filter(Boolean) : ['127.0.0.1/32', '::1/128', '10.0.0.0/8', '192.168.0.0/16']),
|
||||
/** 메트릭 보존 일수(최소 90, §28.1) */
|
||||
metricsRetentionDays: Math.max(90, Number(values.metricsRetentionDays || 90)),
|
||||
/** heartbeat 가 이만큼 없으면 서버를 offline 으로 본다 */
|
||||
offlineAfterMs: Number(values.offlineAfterMs || 90 * 1000),
|
||||
/** 전송(transfer) 파일 보관 시간 */
|
||||
transferTtlMs: Number(values.transferTtlMs || 24 * 3600 * 1000),
|
||||
/** 긴급 자격 최대 유효 시간(분) */
|
||||
emergencyTtlMinutes: Math.min(60, Number(values.emergencyTtlMinutes || 30)),
|
||||
values,
|
||||
/** 설정 저장(운영자 화면·CLI). 비밀값은 거부한다. */
|
||||
set(patch, by = 'ops') {
|
||||
for (const k of Object.keys(patch)) if (/key|secret|token|pass/i.test(k)) throw new Error(`비밀값(${k})은 설정 파일에 둘 수 없습니다 — 보관소를 쓰세요`);
|
||||
Object.assign(values, patch); const tmp = file + '.tmp';
|
||||
writeFileSync(tmp, JSON.stringify({ version: 1, updatedAt: new Date().toISOString(), updatedBy: by, values }, null, 1), { mode: 0o600 }); renameSync(tmp, file); Object.assign(cfg, patch);
|
||||
},
|
||||
};
|
||||
return cfg;
|
||||
}
|
||||
+249
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* hal/core.mjs — HAL 핵심 서비스: 서버·Agent 등록부, Agent 작업 배정·대기, 릴리스 등록부, 부트스트랩 상태, 지표, 포털 이벤트 outbox.
|
||||
* app.mjs(라우트)와 jobs.mjs(작업 종류)가 함께 쓴다. 모든 함수는 deps(store·vault·config·log·jobs) 를 받는다.
|
||||
*/
|
||||
import { newId, randomToken, sha256, hashPassword } from '../lib/crypto.mjs';
|
||||
import { nowIso, toJson, parseJson } from '../lib/db.mjs';
|
||||
import { audit } from '../lib/audit.mjs';
|
||||
import { notify } from '../lib/notify.mjs';
|
||||
import { redact, safeError } from '../lib/redact.mjs';
|
||||
import { serviceFetch } from '../lib/serviceauth.mjs';
|
||||
import { JobError } from '../lib/jobs.mjs';
|
||||
import { existsSync, statSync, mkdirSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
export const EXTERNAL_KINDS = new Set(['site.deploy', 'site.remove', 'site.check', 'site.rollback']); // 외부 Agent 가 받을 수 있는 작업(§10.2)
|
||||
export const SERVER_ROLES = ['web', 'mail', 'backup', 'control'];
|
||||
|
||||
// ------------------------------------------------------------------ 부트스트랩 상태(§32.2)
|
||||
export function bootstrapState(d) { let r = d.store.get('SELECT * FROM bootstrap_state WHERE id=1'); if (!r) { d.store.run('INSERT INTO bootstrap_state(id, updated_at) VALUES(1, ?)', nowIso()); r = d.store.get('SELECT * FROM bootstrap_state WHERE id=1'); } return r; }
|
||||
export function setBootstrap(d, patch) { const sets = Object.keys(patch).map((k) => `${k}=?`); d.store.run(`UPDATE bootstrap_state SET ${sets.join(', ')}, updated_at=? WHERE id=1`, ...Object.values(patch), nowIso()); return bootstrapState(d); }
|
||||
/** 부트스트랩 완료 여부: HAL 실행 버전 기록 + 첫 Agent 등록·승인 */
|
||||
export function bootstrapComplete(d) { const b = bootstrapState(d); return !!(b.hal_running_version && b.first_agent_id && b.completed_at); }
|
||||
/** HAL 기동 때: 실행 버전·인스턴스 임대 기록. 다른 인스턴스의 최근 임대가 있으면 거부(§34 "하나의 HAL 만 운영"). */
|
||||
export function claimLive(d, { instanceId, version, takeover = false, now = Date.now() }) {
|
||||
const b = bootstrapState(d);
|
||||
if (b.live_instance_id && b.live_instance_id !== instanceId && b.live_lease_at && now - Date.parse(b.live_lease_at) < 60000 && !takeover) {
|
||||
throw Object.assign(new Error(`다른 HAL 인스턴스(${b.live_instance_id})가 ${b.live_lease_at} 에 운영 중입니다. 기존 제어 위치를 먼저 중지하세요`), { code: 'another_hal_live' });
|
||||
}
|
||||
return setBootstrap(d, { live_instance_id: instanceId, live_lease_at: new Date(now).toISOString(), hal_running_version: version, hal_started_at: nowIso() });
|
||||
}
|
||||
export function renewLease(d, instanceId) { d.store.run('UPDATE bootstrap_state SET live_lease_at=? WHERE id=1 AND live_instance_id=?', nowIso(), instanceId); }
|
||||
/** 첫 Agent 가 승인되면 부트스트랩을 닫고 수동 설치를 잠근다 */
|
||||
export function maybeCompleteBootstrap(d, agentId) {
|
||||
const b = bootstrapState(d); if (b.completed_at) return b;
|
||||
if (!b.first_agent_id) setBootstrap(d, { first_agent_id: agentId, first_agent_registered_at: nowIso() });
|
||||
if (bootstrapState(d).hal_running_version) { const r = setBootstrap(d, { completed_at: nowIso(), manual_install_locked: 1, first_install: 0 }); audit(d.store, { actorType: 'system', action: 'bootstrap.complete', subjectType: 'agent', subjectId: agentId, immutable: true }); return r; }
|
||||
return bootstrapState(d);
|
||||
}
|
||||
/** HAL 자체 건강(§32.2 4단계) */
|
||||
export function selfHealth(d) {
|
||||
const b = bootstrapState(d); let dbOk = true; try { d.store.get('SELECT 1 AS x'); } catch (_) { dbOk = false; }
|
||||
return { ok: dbOk && !!d.jobsRunning, version: d.version, uptimeSec: Math.round(process.uptime()), db: dbOk, jobs: !!d.jobsRunning, bootstrap: { complete: bootstrapComplete(d), firstInstall: !!b.first_install, releaseVersion: b.release_version, mainCommit: b.main_commit, artifactSha256: b.artifact_sha256, halRunningVersion: b.hal_running_version, firstAgent: b.first_agent_id, manualInstallLocked: !!b.manual_install_locked }, instanceId: d.instanceId };
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ 서버·Agent
|
||||
export function createServer(d, { name, role, location = '', by = null }) {
|
||||
if (!SERVER_ROLES.includes(role)) throw new JobError('bad_role', `서버 역할은 ${SERVER_ROLES.join('/')} 중 하나입니다`);
|
||||
const id = newId('srv_'); const now = nowIso();
|
||||
d.store.run('INSERT INTO servers(id, name, role, location, status, created_at, updated_at) VALUES(?,?,?,?,\'pending\',?,?)', id, String(name).slice(0, 80), role, String(location).slice(0, 120), now, now);
|
||||
audit(d.store, { actorType: 'operator', actorId: by, action: 'server.create', subjectType: 'server', subjectId: id, detail: { name, role, location } });
|
||||
return getServer(d, id);
|
||||
}
|
||||
export function getServer(d, id) { const s = d.store.get('SELECT * FROM servers WHERE id=?', id); return s ? { ...s, metrics: parseJson(s.metrics_json, null), capacity: parseJson(s.capacity_json, null), facts: parseJson(s.facts_json, {}) } : null; }
|
||||
export function listServers(d, { role = null } = {}) { return d.store.all(`SELECT * FROM servers ${role ? 'WHERE role=?' : ''} ORDER BY created_at`, ...(role ? [role] : [])).map((s) => publicServer(d, s)); }
|
||||
/** 화면·포털용 서버 요약(비밀 없음). offline 판정은 last_seen 기준(§28.1 heartbeat). */
|
||||
export function publicServer(d, s) {
|
||||
const seen = s.last_seen_at ? Date.now() - Date.parse(s.last_seen_at) : Infinity; const offline = seen > d.config.offlineAfterMs;
|
||||
const status = s.status === 'pending' ? 'pending' : offline ? 'offline' : s.status;
|
||||
const mods = d.store.all('SELECT module, target_version, actual_version, health, active, last_check_at, last_error FROM module_targets WHERE server_id=?', s.id);
|
||||
const capacity = parseJson(s.capacity_json, {}) || {}; const metrics = parseJson(s.metrics_json, null);
|
||||
return { id: s.id, name: s.name, role: s.role, location: s.location, status, rawStatus: s.status, acceptsNew: !!s.accepts_new && status === 'ready', approvedAt: s.approved_at, agentId: s.agent_id, agentVersion: s.agent_version, lastSeenAt: s.last_seen_at, stale: offline, clockSkewMs: s.clock_skew_ms,
|
||||
health: status === 'offline' ? 'down' : mods.some((m) => m.health === 'down') ? 'partial_outage' : mods.some((m) => m.health === 'degraded') ? 'degraded' : status === 'ready' ? 'ok' : 'unknown',
|
||||
metrics, capacity: { diskFreeBytes: capacity.diskFreeBytes || 0, diskTotalBytes: capacity.diskTotalBytes || 0, memFreeBytes: capacity.memFreeBytes || 0, cpuLoad: capacity.cpuLoad || 0, sitesHosted: capacity.sitesHosted || 0, reservedBytes: capacity.reservedBytes || 0, safetyMarginBytes: capacity.safetyMarginBytes || Math.round((capacity.diskTotalBytes || 0) * 0.1), exhaustEta: capacity.exhaustEta || null },
|
||||
modules: Object.fromEntries(mods.map((m) => [m.module, { target: m.target_version, actual: m.actual_version, drift: !!(m.target_version && m.actual_version && m.target_version !== m.actual_version), health: m.health, active: !!m.active, lastCheckAt: m.last_check_at, lastError: m.last_error }])),
|
||||
maintenanceNote: s.maintenance_note, isolatedReason: s.isolated_reason, facts: parseJson(s.facts_json, {}) };
|
||||
}
|
||||
export function updateServer(d, id, patch, by = null) {
|
||||
const allowed = ['name', 'location', 'status', 'accepts_new', 'maintenance_note', 'isolated_reason', 'role'];
|
||||
const keys = Object.keys(patch).filter((k) => allowed.includes(k)); if (!keys.length) return getServer(d, id);
|
||||
d.store.run(`UPDATE servers SET ${keys.map((k) => `${k}=?`).join(', ')}, updated_at=? WHERE id=?`, ...keys.map((k) => patch[k]), nowIso(), id);
|
||||
audit(d.store, { actorType: 'operator', actorId: by, action: 'server.update', subjectType: 'server', subjectId: id, detail: patch });
|
||||
return getServer(d, id);
|
||||
}
|
||||
/** 운영자 승인(§28.2 6단계): pending → ready, 신규 배치 허용 */
|
||||
export function approveServer(d, id, by) { const s = getServer(d, id); if (!s) throw new JobError('server_not_found', '서버가 없습니다'); if (!s.agent_id) throw new JobError('agent_missing', 'Agent 가 아직 연결되지 않았습니다'); updateServer(d, id, { status: 'ready', accepts_new: 1 }, by); d.store.run('UPDATE servers SET approved_at=?, approved_by=? WHERE id=?', nowIso(), by, id); maybeCompleteBootstrap(d, s.agent_id); return getServer(d, id); }
|
||||
/** 신규 배치 가능 서버 고르기(§28.3): ready·accepts_new·여유 용량(안전 여유 제외) */
|
||||
export function pickServer(d, { role = 'web', needBytes = 0 } = {}) {
|
||||
const cands = listServers(d, { role }).filter((s) => s.acceptsNew && s.status === 'ready' && (s.capacity.diskFreeBytes - s.capacity.reservedBytes - s.capacity.safetyMarginBytes) >= needBytes);
|
||||
cands.sort((a, b) => (b.capacity.diskFreeBytes - b.capacity.reservedBytes) - (a.capacity.diskFreeBytes - a.capacity.reservedBytes));
|
||||
return cands[0] ? { serverId: cands[0].id, name: cands[0].name } : null;
|
||||
}
|
||||
/** 일회용 등록 토큰(10분). 토큰 원문은 이 반환값에서 한 번만 보인다. */
|
||||
export function issueRegistrationToken(d, { kind = 'managed', serverId = null, groupId = null, siteId = null, label = '', by = null, ttlMs = 10 * 60 * 1000 }) {
|
||||
if (kind === 'managed' && serverId && !getServer(d, serverId)) throw new JobError('server_not_found', '서버가 없습니다');
|
||||
const token = randomToken(32); const id = newId('rt_'); const expiresAt = new Date(Date.now() + ttlMs).toISOString();
|
||||
d.store.run('INSERT INTO registration_tokens(id, token_hash, kind, server_id, group_id, site_id, label, created_by, created_at, expires_at) VALUES(?,?,?,?,?,?,?,?,?,?)', id, sha256(token), kind, serverId, groupId, siteId, label, by, nowIso(), expiresAt);
|
||||
audit(d.store, { actorType: by ? 'operator' : 'system', actorId: by, action: 'agent.token_issue', subjectType: kind === 'external' ? 'site' : 'server', subjectId: siteId || serverId, detail: { kind, label } });
|
||||
return { id, token, expiresAt, installCommand: `curl -fsSL ${d.config.publicUrl}/agent/v1/install.sh | bash -s -- --hal ${d.config.publicUrl} --token ${token}${kind === 'external' ? ` --external --site ${siteId}` : ''}` };
|
||||
}
|
||||
/**
|
||||
* Agent 등록(토큰 1회 사용). 비밀은 보관소에 두고 응답에서 한 번만 돌려준다.
|
||||
* @returns {{agentId, secret, serverId, kind}}
|
||||
*/
|
||||
export function registerAgent(d, { token, facts = {}, version = '' }) {
|
||||
const t = d.store.get('SELECT * FROM registration_tokens WHERE token_hash=?', sha256(String(token || '')));
|
||||
if (!t) throw new JobError('bad_token', '등록 토큰이 올바르지 않습니다');
|
||||
if (t.used_at) throw new JobError('token_used', '이미 사용한 등록 토큰입니다 — 새 토큰을 발급받으세요');
|
||||
if (Date.parse(t.expires_at) < Date.now()) throw new JobError('token_expired', '등록 토큰이 만료되었습니다 — 새 토큰을 발급받으세요');
|
||||
const agentId = newId('ag_'); const secret = randomToken(32); const now = nowIso();
|
||||
return d.store.tx(() => {
|
||||
const used = d.store.run('UPDATE registration_tokens SET used_at=?, used_by_agent=? WHERE id=? AND used_at IS NULL', now, agentId, t.id).changes;
|
||||
if (!used) throw new JobError('token_used', '이미 사용한 등록 토큰입니다');
|
||||
let serverId = t.server_id;
|
||||
if (t.kind === 'managed' && !serverId) { serverId = createServer(d, { name: facts.hostname || agentId, role: facts.role || 'web', location: facts.location || '' }).id; }
|
||||
d.store.run('INSERT INTO agents(id, server_id, kind, secret_hash, status, label, group_id, site_id, version, registered_at, last_seen_at, facts_json) VALUES(?,?,?,?,\'active\',?,?,?,?,?,?,?)', agentId, serverId, t.kind, sha256(secret), t.label || facts.hostname || '', t.group_id, t.site_id, version, now, now, toJson(redact(facts)));
|
||||
d.vault.set(`service:agent:${agentId}`, secret, { kind: 'agent-secret' });
|
||||
if (serverId) d.store.run('UPDATE servers SET agent_id=?, last_seen_at=?, agent_version=?, facts_json=?, updated_at=? WHERE id=?', agentId, now, version, toJson(redact(facts)), now, serverId);
|
||||
audit(d.store, { actorType: 'agent', actorId: agentId, action: 'agent.register', subjectType: t.kind === 'external' ? 'site' : 'server', subjectId: t.site_id || serverId, detail: { kind: t.kind, version, hostname: facts.hostname } });
|
||||
return { agentId, secret, serverId, kind: t.kind, siteId: t.site_id, groupId: t.group_id };
|
||||
});
|
||||
}
|
||||
export function getAgent(d, id) { return d.store.get('SELECT * FROM agents WHERE id=?', id) || null; }
|
||||
export function revokeAgent(d, id, by = null) { d.store.run('UPDATE agents SET status=\'revoked\', revoked_at=? WHERE id=?', nowIso(), id); d.vault.remove(`service:agent:${id}`); d.store.run('UPDATE agent_jobs SET status=\'cancelled\', ended_at=? WHERE agent_id=? AND status IN (\'queued\',\'running\')', nowIso(), id); audit(d.store, { actorType: by ? 'operator' : 'system', actorId: by, action: 'agent.revoke', subjectType: 'agent', subjectId: id, immutable: true }); }
|
||||
/** 외부 Agent 상태(포털 화면용, §8.1) */
|
||||
export function externalAgentStatus(d, id) {
|
||||
const a = getAgent(d, id); if (!a || a.kind !== 'external') return null;
|
||||
const seen = a.last_seen_at ? Date.now() - Date.parse(a.last_seen_at) : Infinity;
|
||||
const last = d.store.get('SELECT * FROM agent_jobs WHERE agent_id=? AND kind=\'site.deploy\' ORDER BY created_at DESC LIMIT 1', id);
|
||||
const lastCheck = d.store.get('SELECT * FROM agent_jobs WHERE agent_id=? AND kind=\'site.check\' AND status=\'done\' ORDER BY created_at DESC LIMIT 1', id);
|
||||
return { id: a.id, status: a.status === 'revoked' ? 'revoked' : !a.last_seen_at ? 'pending' : seen > d.config.offlineAfterMs ? 'offline' : 'connected', lastSeenAt: a.last_seen_at, version: a.version, siteId: a.site_id, facts: parseJson(a.facts_json, {}),
|
||||
lastDeploy: last ? { id: last.id, status: last.status, at: last.ended_at || last.started_at || last.created_at, result: parseJson(last.result_json, null), error: last.error } : null,
|
||||
lastCheck: lastCheck ? { at: lastCheck.ended_at, result: parseJson(lastCheck.result_json, null) } : null };
|
||||
}
|
||||
/** heartbeat 처리: 지표 저장, 서버 last_seen·시계 차이·모듈 실제 버전·용량 갱신 */
|
||||
export function heartbeat(d, agent, body) {
|
||||
const now = nowIso(); const m = body.metrics || {}; const skew = Number.isFinite(body.clockSkewMs) ? Math.round(body.clockSkewMs) : null;
|
||||
d.store.run('UPDATE agents SET last_seen_at=?, version=?, facts_json=COALESCE(?, facts_json) WHERE id=?', now, body.version || agent.version, body.facts ? toJson(redact(body.facts)) : null, agent.id);
|
||||
if (!agent.server_id) return { ok: true };
|
||||
const capacity = { diskFreeBytes: m.disk ? m.disk.total - m.disk.used : 0, diskTotalBytes: m.disk?.total || 0, memFreeBytes: m.mem ? m.mem.total - m.mem.used : 0, cpuLoad: m.load1 || 0, sitesHosted: body.sitesHosted || 0, reservedBytes: body.reservedBytes || 0, exhaustEta: body.exhaustEta || null };
|
||||
d.store.run('UPDATE servers SET last_seen_at=?, clock_skew_ms=?, agent_version=?, metrics_json=?, capacity_json=?, facts_json=COALESCE(?, facts_json), status=CASE WHEN status=\'offline\' THEN \'ready\' ELSE status END, updated_at=? WHERE id=?', now, skew, body.version || null, toJson(m), toJson(capacity), body.facts ? toJson(redact(body.facts)) : null, now, agent.server_id);
|
||||
d.store.run('INSERT OR REPLACE INTO metrics_samples(server_id, at, cpu, mem_used, mem_total, disk_used, disk_total, inode_used, inode_total, io_read, io_write, net_in, net_out, clock_skew_ms, load1) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
|
||||
agent.server_id, now, m.cpu ?? null, m.mem?.used ?? null, m.mem?.total ?? null, m.disk?.used ?? null, m.disk?.total ?? null, m.inode?.used ?? null, m.inode?.total ?? null, m.io?.read ?? null, m.io?.write ?? null, m.net?.in ?? null, m.net?.out ?? null, skew, m.load1 ?? null);
|
||||
for (const [name, mod] of Object.entries(body.modules || {})) {
|
||||
d.store.run('INSERT INTO module_targets(module, server_id, actual_version, health, active, last_check_at, updated_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(module, server_id) DO UPDATE SET actual_version=excluded.actual_version, health=excluded.health, active=excluded.active, last_check_at=excluded.last_check_at, updated_at=excluded.updated_at',
|
||||
name, agent.server_id, mod.version || null, mod.health || 'unknown', mod.active ? 1 : 0, now, now);
|
||||
}
|
||||
// 시계 차이·디스크 고갈 경고(§28.1)
|
||||
if (skew != null && Math.abs(skew) > 60000) notify(d.store, { category: 'ops', severity: 'warning', incidentKey: `server:${agent.server_id}:clock`, title: `서버 시계 차이 ${Math.round(skew / 1000)}초`, link: `/ops/servers/${agent.server_id}` });
|
||||
else { try { const { resolveIncident } = d.notifyLib; resolveIncident(d.store, `server:${agent.server_id}:clock`); } catch (_) {} }
|
||||
if (m.disk && m.disk.total && m.disk.used / m.disk.total >= 0.9) notify(d.store, { category: 'ops', severity: m.disk.used / m.disk.total >= 0.95 ? 'critical' : 'warning', incidentKey: `server:${agent.server_id}:disk`, title: `서버 디스크 ${Math.round(m.disk.used / m.disk.total * 100)}% 사용`, link: `/ops/servers/${agent.server_id}` });
|
||||
return { ok: true, serverTime: now, pendingJobs: d.store.get('SELECT COUNT(*) AS n FROM agent_jobs WHERE agent_id=? AND status=\'queued\'', agent.id).n };
|
||||
}
|
||||
/** 오래 heartbeat 가 없는 서버를 offline 으로 표시(주기 작업). 변경된 서버 목록 반환 */
|
||||
export function markOffline(d) {
|
||||
const cut = new Date(Date.now() - d.config.offlineAfterMs).toISOString();
|
||||
const rows = d.store.all('SELECT id, name FROM servers WHERE status IN (\'ready\',\'maintenance\') AND agent_id IS NOT NULL AND (last_seen_at IS NULL OR last_seen_at<?)', cut);
|
||||
for (const r of rows) { d.store.run('UPDATE servers SET status=\'offline\', updated_at=? WHERE id=?', nowIso(), r.id); notify(d.store, { category: 'incident', severity: 'critical', mandatory: false, incidentKey: `server:${r.id}:offline`, title: `서버 ${r.name} 응답 없음(Node Agent heartbeat 중단)`, link: `/ops/servers/${r.id}` }); }
|
||||
return rows;
|
||||
}
|
||||
export function purgeMetrics(d) { const cut = new Date(Date.now() - d.config.metricsRetentionDays * 86400000).toISOString(); return d.store.run('DELETE FROM metrics_samples WHERE at<?', cut).changes; }
|
||||
export function metrics(d, serverId, { since = null, until = null, limit = 5000 } = {}) { return d.store.all('SELECT * FROM metrics_samples WHERE server_id=? AND at>=? AND at<=? ORDER BY at DESC LIMIT ?', serverId, since || new Date(Date.now() - 90 * 86400000).toISOString(), until || nowIso(), Math.min(limit, 50000)); }
|
||||
|
||||
// ------------------------------------------------------------------ Agent 작업
|
||||
/**
|
||||
* 재진입 가능한 배정: 'dispatch' 단계에 저장된 Agent 작업이 실패·취소로 끝났으면(재시도) 새 Agent 작업을 만든다.
|
||||
* 호출자는 ctx.step('dispatch', …) 대신 이 함수를 쓴다.
|
||||
*/
|
||||
export async function ensureAgentJob(d, ctx, job, { stepId = 'dispatch', agentId, kind, payload, title = '대상 Agent 에 작업 배정' }) {
|
||||
const prev = ctx.steps().find((x) => x.id === stepId);
|
||||
if (prev && prev.state === 'done' && prev.result?.agentJobId) { const aj = getAgentJob(d, prev.result.agentJobId); if (aj && ['failed', 'cancelled'].includes(aj.status)) { prev.state = 'pending'; delete prev.result; } }
|
||||
return ctx.step(stepId, () => ({ agentId, agentJobId: dispatchAgentJob(d, { halJobId: job.id, agentId, kind, payload, dedupeKey: `${job.id}:${kind}:${agentId}:${job.attempt}` }) }), title);
|
||||
}
|
||||
/** Agent 작업 만들기(HAL 작업의 자식). 같은 hal_job_id+kind+agent 의 미완료 작업이 있으면 재사용(재진입 멱등). */
|
||||
export function dispatchAgentJob(d, { halJobId, agentId, kind, payload, dedupeKey = null }) {
|
||||
const agent = getAgent(d, agentId); if (!agent || agent.status !== 'active') throw new JobError('agent_unavailable', 'Agent 가 없거나 해지되었습니다');
|
||||
if (agent.kind === 'external' && !EXTERNAL_KINDS.has(kind)) throw new JobError('kind_not_allowed_external', `외부 Agent 는 ${kind} 작업을 받을 수 없습니다`);
|
||||
if (agent.kind === 'external' && payload?.siteId && agent.site_id && payload.siteId !== agent.site_id) throw new JobError('site_mismatch', '이 외부 Agent 는 다른 사이트에 귀속되어 있습니다');
|
||||
const key = dedupeKey || `${halJobId}:${kind}:${agentId}`;
|
||||
const ex = d.store.get('SELECT * FROM agent_jobs WHERE hal_job_id=? AND kind=? AND agent_id=? AND json_extract(payload_json, \'$._dedupe\')=?', halJobId, kind, agentId, key);
|
||||
if (ex) return ex.id;
|
||||
const id = newId('aj_');
|
||||
d.store.run('INSERT INTO agent_jobs(id, hal_job_id, agent_id, kind, payload_json, status, created_at, steps_json, log_json) VALUES(?,?,?,?,?,\'queued\',?,\'[]\',\'[]\')', id, halJobId, agentId, kind, toJson({ ...payload, _dedupe: key }), nowIso());
|
||||
return id;
|
||||
}
|
||||
export function getAgentJob(d, id) { const r = d.store.get('SELECT * FROM agent_jobs WHERE id=?', id); return r ? { ...r, payload: parseJson(r.payload_json, {}), steps: parseJson(r.steps_json, []), result: parseJson(r.result_json, null), log: parseJson(r.log_json, []) } : null; }
|
||||
/** Agent 가 다음 작업을 가져간다(long-poll 은 라우트가 반복 호출). queued 중 가장 오래된 것 하나 → running */
|
||||
export function nextAgentJob(d, agentId) {
|
||||
const r = d.store.get('SELECT id FROM agent_jobs WHERE agent_id=? AND status=\'queued\' ORDER BY created_at LIMIT 1', agentId); if (!r) return null;
|
||||
const got = d.store.run('UPDATE agent_jobs SET status=\'running\', assigned_at=?, started_at=?, heartbeat_at=? WHERE id=? AND status=\'queued\'', nowIso(), nowIso(), nowIso(), r.id).changes;
|
||||
if (!got) return null; const j = getAgentJob(d, r.id); delete j.payload._dedupe; return { id: j.id, kind: j.kind, payload: j.payload, halJobId: j.hal_job_id };
|
||||
}
|
||||
/** Agent 재시작 뒤 이어서 할 작업 목록(running 으로 남은 것) */
|
||||
export function unfinishedAgentJobs(d, agentId) { return d.store.all('SELECT id, kind, payload_json, steps_json FROM agent_jobs WHERE agent_id=? AND status=\'running\'', agentId).map((r) => ({ id: r.id, kind: r.kind, payload: (() => { const p = parseJson(r.payload_json, {}); delete p._dedupe; return p; })(), steps: parseJson(r.steps_json, []) })); }
|
||||
export function agentJobProgress(d, id, agentId, { progress, steps, log }) { const r = d.store.run('UPDATE agent_jobs SET progress=COALESCE(?, progress), steps_json=COALESCE(?, steps_json), log_json=COALESCE(?, log_json), heartbeat_at=? WHERE id=? AND agent_id=? AND status=\'running\'', progress ?? null, steps ? toJson(redact(steps)) : null, log ? toJson(redact(log)) : null, nowIso(), id, agentId).changes; return !!r; }
|
||||
export function agentJobResult(d, id, agentId, { status, result, error, errorCode, userMessage, steps, log }) {
|
||||
if (!['done', 'failed', 'needs_user'].includes(status)) throw new JobError('bad_status', 'Agent 결과 상태는 done/failed/needs_user 중 하나입니다');
|
||||
const ch = d.store.run('UPDATE agent_jobs SET status=?, result_json=?, error=?, error_code=?, user_message=?, steps_json=COALESCE(?, steps_json), log_json=COALESCE(?, log_json), progress=CASE WHEN ?=\'done\' THEN 100 ELSE progress END, ended_at=? WHERE id=? AND agent_id=? AND status=\'running\'',
|
||||
status, toJson(redact(result ?? null)), error ? safeError({ message: error }) : null, errorCode || null, userMessage || null, steps ? toJson(redact(steps)) : null, log ? toJson(redact(log)) : null, status, nowIso(), id, agentId).changes;
|
||||
return !!ch;
|
||||
}
|
||||
export const isAgentJobCancelled = (d, id) => !!d.store.get('SELECT cancel_requested FROM agent_jobs WHERE id=?', id)?.cancel_requested;
|
||||
/**
|
||||
* HAL 작업(lib/jobs run) 안에서 Agent 작업이 끝날 때까지 기다린다. 진행률·단계를 HAL 작업에 비춘다.
|
||||
* Agent 가 오래 응답하지 않으면(heartbeat 5분) 계속 기다리되 사유를 기록하고, 서버가 offline 이면 needs_user 로 멈춘다(운영자가 서버를 살리고 '이어서 진행').
|
||||
*/
|
||||
export async function awaitAgentJob(d, ctx, agentJobId, { timeoutMs = d.config.agentWaitTimeoutMs || 6 * 3600 * 1000, pollMs = d.config.agentPollMs || 1000 } = {}) {
|
||||
const t0 = Date.now(); let lastProgress = -1;
|
||||
while (true) {
|
||||
const j = getAgentJob(d, agentJobId); if (!j) throw new JobError('agent_job_missing', 'Agent 작업 기록이 사라졌습니다');
|
||||
if (j.status === 'done') return j.result;
|
||||
if (j.status === 'failed') throw new JobError(j.error_code || 'agent_failed', j.error || 'Agent 작업 실패', { retryable: j.error_code === 'transient' });
|
||||
if (j.status === 'needs_user') ctx.needsUser(j.user_message || 'Agent 가 사용자/운영자 확인을 요청합니다', [{ id: 'resume', label: '확인 후 이어서 진행' }]);
|
||||
if (j.status === 'cancelled') throw new JobError('agent_job_cancelled', 'Agent 작업이 취소되었습니다');
|
||||
try { ctx.checkCancel(); } catch (e) { d.store.run('UPDATE agent_jobs SET cancel_requested=1 WHERE id=?', agentJobId); throw e; }
|
||||
if (j.progress !== lastProgress) { lastProgress = j.progress; ctx.progress(j.progress); }
|
||||
const agent = getAgent(d, j.agent_id); const srv = agent?.server_id ? getServer(d, agent.server_id) : null;
|
||||
const silent = agent?.last_seen_at ? Date.now() - Date.parse(agent.last_seen_at) : Infinity;
|
||||
if (silent > 10 * 60 * 1000) ctx.needsUser(`대상 서버(${srv?.name || agent?.label || j.agent_id})의 Agent 가 10분 넘게 응답하지 않습니다. 서버 상태를 확인한 뒤 이어서 진행하세요`, [{ id: 'resume', label: 'Agent 복구 후 이어서 진행' }]);
|
||||
if (Date.now() - t0 > timeoutMs) throw new JobError('agent_timeout', 'Agent 작업이 제한 시간을 넘겼습니다');
|
||||
await new Promise((r) => setTimeout(r, pollMs));
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ 릴리스(§32.1)
|
||||
export function registerRelease(d, { version, mainCommit, builtAt, artifacts, checks = null, notes = null, approve = false, by = null }) {
|
||||
const now = nowIso(); const ex = d.store.get('SELECT version FROM releases WHERE version=?', version);
|
||||
if (ex) d.store.run('UPDATE releases SET main_commit=?, built_at=?, artifacts_json=?, checks_json=?, notes=COALESCE(?, notes) WHERE version=?', mainCommit, builtAt, toJson(artifacts), toJson(checks), notes, version);
|
||||
else d.store.run('INSERT INTO releases(version, main_commit, built_at, artifacts_json, checks_json, status, notes, registered_at) VALUES(?,?,?,?,?,\'registered\',?,?)', version, mainCommit, builtAt, toJson(artifacts), toJson(checks), notes, now);
|
||||
if (approve) approveRelease(d, version, by);
|
||||
audit(d.store, { actorType: by ? 'operator' : 'system', actorId: by, action: 'release.register', subjectType: 'release', subjectId: version, detail: { mainCommit, artifacts: Object.keys(artifacts || {}) } });
|
||||
return getRelease(d, version);
|
||||
}
|
||||
export function approveRelease(d, version, by) { const r = getRelease(d, version); if (!r) throw new JobError('release_not_found', '릴리스가 없습니다'); const checks = r.checks || {}; if (checks.testsPassed === false) throw new JobError('release_checks_failed', '검사에 실패한 릴리스는 승인할 수 없습니다'); d.store.run('UPDATE releases SET status=\'approved\', approved_by=?, approved_at=? WHERE version=?', by, nowIso(), version); audit(d.store, { actorType: 'operator', actorId: by, action: 'release.approve', subjectType: 'release', subjectId: version, immutable: true }); return getRelease(d, version); }
|
||||
export function getRelease(d, version) { const r = d.store.get('SELECT * FROM releases WHERE version=?', version); return r ? { ...r, artifacts: parseJson(r.artifacts_json, {}), checks: parseJson(r.checks_json, null) } : null; }
|
||||
export function listReleases(d) { return d.store.all('SELECT * FROM releases ORDER BY registered_at DESC').map((r) => ({ ...r, artifacts: parseJson(r.artifacts_json, {}), checks: parseJson(r.checks_json, null) })); }
|
||||
export const artifactDir = (d) => join(d.config.dataDir, 'artifacts');
|
||||
/** artifact 파일 경로·존재 확인 */
|
||||
export function artifactPath(d, version, module) { const r = getRelease(d, version); const a = r && r.artifacts[module]; if (!a) return null; const p = join(artifactDir(d), version, a.file); return existsSync(p) ? { path: p, sha256: a.sha256, size: statSync(p).size } : null; }
|
||||
|
||||
// ------------------------------------------------------------------ 전송(transfer)
|
||||
export function transferDir(d) { const p = join(d.config.dataDir, 'transfers'); mkdirSync(p, { recursive: true, mode: 0o700 }); return p; }
|
||||
export function createTransfer(d, { id = newId('tr_'), agentId, sha256: sum, size, purpose = '', meta = null }) { const path = join(transferDir(d), id + '.bin'); d.store.run('INSERT OR REPLACE INTO transfers(id, created_by_agent, path, size_bytes, sha256, created_at, expires_at, purpose, meta_json) VALUES(?,?,?,?,?,?,?,?,?)', id, agentId, path, size, sum, nowIso(), new Date(Date.now() + d.config.transferTtlMs).toISOString(), purpose, toJson(meta)); return { id, path }; }
|
||||
export function getTransfer(d, id) { const t = d.store.get('SELECT * FROM transfers WHERE id=? AND deleted_at IS NULL', id); return t && existsSync(t.path) ? t : null; }
|
||||
|
||||
// ------------------------------------------------------------------ 포털 이벤트 outbox(§6)
|
||||
export function queuePortalEvent(d, payload) { d.store.run('INSERT INTO portal_events_outbox(id, payload_json, created_at, next_at) VALUES(?,?,?,?)', newId('ev_'), toJson(redact(payload)), nowIso(), nowIso()); }
|
||||
/** outbox 전송(주기). 실패는 지수 백오프로 재시도하고 치명적으로 다루지 않는다. */
|
||||
export async function flushPortalEvents(d, { fetchImpl = fetch } = {}) {
|
||||
const secret = d.vault.get('service:hal'); if (!secret) return 0;
|
||||
const rows = d.store.all('SELECT * FROM portal_events_outbox WHERE delivered_at IS NULL AND next_at<=? ORDER BY created_at LIMIT 50', nowIso()); let n = 0;
|
||||
for (const r of rows) {
|
||||
try { const res = await serviceFetch(d.config.portalUrl, 'hal', secret, 'POST', '/internal/hal-events', parseJson(r.payload_json, {}), { timeoutMs: 8000, fetchImpl }); if (!res.ok) throw new Error(`portal ${res.status} ${res.json?.error || ''}`); d.store.run('UPDATE portal_events_outbox SET delivered_at=? WHERE id=?', nowIso(), r.id); n++; }
|
||||
catch (e) { const attempts = r.attempts + 1; d.store.run('UPDATE portal_events_outbox SET attempts=?, next_at=?, last_error=? WHERE id=?', attempts, new Date(Date.now() + Math.min(600000, 5000 * 2 ** attempts)).toISOString(), safeError(e), r.id); }
|
||||
}
|
||||
d.store.run('DELETE FROM portal_events_outbox WHERE delivered_at IS NOT NULL AND delivered_at<?', new Date(Date.now() - 7 * 86400000).toISOString());
|
||||
return n;
|
||||
}
|
||||
export function ensureOpsAccountExists(d) { return d.store.get('SELECT COUNT(*) AS n FROM ops_accounts').n > 0; }
|
||||
export { hashPassword };
|
||||
@@ -0,0 +1,24 @@
|
||||
/**
|
||||
* hal/deps.mjs — HAL 의존성 묶음(deps) 만들기. index.mjs(운영)와 테스트가 같은 함수를 쓴다.
|
||||
* deps = { store, vault, config, jobs, log, version, instanceId, serviceauth, notifyLib, fetch, emergencyRunner, jobsRunning, stopping }
|
||||
*/
|
||||
import { openDb } from '../lib/db.mjs';
|
||||
import { openVault } from '../lib/vault.mjs';
|
||||
import { createJobEngine } from '../lib/jobs.mjs';
|
||||
import * as serviceauth from '../lib/serviceauth.mjs';
|
||||
import * as notifyLib from '../lib/notify.mjs';
|
||||
import { newId } from '../lib/crypto.mjs';
|
||||
import { MIGRATIONS } from './schema.mjs';
|
||||
import { registerKinds } from './jobs.mjs';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
export const VERSION = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version;
|
||||
export function createHalDeps({ config, dbFile = null, log = () => {}, fetchImpl = null, version = VERSION, concurrency = 64 } = {}) {
|
||||
const store = openDb(dbFile || join(config.dataDir, 'hal.sqlite'), MIGRATIONS);
|
||||
const vault = openVault(config.dataDir);
|
||||
const d = { store, vault, config, log, version, instanceId: newId('hal_'), serviceauth, notifyLib, fetch: fetchImpl, emergencyRunner: null, jobsRunning: false, stopping: false };
|
||||
d.jobs = createJobEngine(store, { log, workerId: `hal@${process.pid}`, concurrency, pollMs: 500 });
|
||||
registerKinds(d.jobs, d);
|
||||
return d;
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
/**
|
||||
* hal/dr.mjs — 플랫폼 재해복구(§34).
|
||||
*
|
||||
* 복구 대상: HAL 운영 정보(hal.sqlite: 서버·Agent·릴리스·작업·모듈 배치·운영자 계정·접근 설정), 진행 중 작업, 보관소(vault.json; 키 파일은 **포함하지 않는다** — 운영자가 따로 보관), config.json, artifact 목록.
|
||||
* 메일 원문·고객 사이트 데이터는 대상이 아니다(고객용 백업·메일 정책 별도).
|
||||
* - 설정(dr_config): RTO·RPO·복구 사본 주기·보존 개수·외부 보관 위치. 출시 검증에서 비어 있으면 안 된다(readiness).
|
||||
* - 복구 사본(snapshot): `VACUUM INTO` 로 일관된 DB 사본 + vault.json + config.json 을 tar.gz 로 묶고 sha256·읽기 검증. 외부 보관 위치로 복사(지정 시).
|
||||
* - 복원(restore): 별도 승인 + "기존 제어 위치 중지" 확인이 있어야 시작. 실제 파일 복원은 deploy/hal-restore.sh 가 하고, 새 HAL 기동 때 claimLive 가 "하나의 HAL" 을 보장한다.
|
||||
* 복원 뒤 checks(운영자 로그인·HAL 제어·HTTPS·고객 서비스 1개 외부 확인)와 실제 상태 비교(diff) 를 기록한다.
|
||||
* - 훈련(drill): 실제 측정 RTO/RPO 와 단계 결과. 마지막 훈련이 성공해야 '정식 운영 준비 완료'.
|
||||
*/
|
||||
import { newId } from '../lib/crypto.mjs';
|
||||
import { nowIso, toJson, parseJson } from '../lib/db.mjs';
|
||||
import { audit } from '../lib/audit.mjs';
|
||||
import { JobError } from '../lib/jobs.mjs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdirSync, existsSync, readFileSync, writeFileSync, statSync, copyFileSync, rmSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import * as core from './core.mjs';
|
||||
|
||||
export function getConfig(d) { return d.store.get('SELECT * FROM dr_config WHERE id=1') || { rto_minutes: null, rpo_minutes: null, snapshot_interval_minutes: null, retention_count: null, external_store: null }; }
|
||||
export function setConfig(d, patch, by) {
|
||||
const c = { ...getConfig(d), ...Object.fromEntries(Object.entries(patch).filter(([k]) => ['rto_minutes', 'rpo_minutes', 'snapshot_interval_minutes', 'retention_count', 'external_store'].includes(k))) };
|
||||
for (const k of ['rto_minutes', 'rpo_minutes', 'snapshot_interval_minutes', 'retention_count']) if (c[k] != null && !(Number(c[k]) > 0)) throw new JobError('bad_value', `${k} 은(는) 0보다 커야 합니다`);
|
||||
d.store.run('INSERT INTO dr_config(id, rto_minutes, rpo_minutes, snapshot_interval_minutes, retention_count, external_store, updated_at, updated_by) VALUES(1,?,?,?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET rto_minutes=excluded.rto_minutes, rpo_minutes=excluded.rpo_minutes, snapshot_interval_minutes=excluded.snapshot_interval_minutes, retention_count=excluded.retention_count, external_store=excluded.external_store, updated_at=excluded.updated_at, updated_by=excluded.updated_by', c.rto_minutes, c.rpo_minutes, c.snapshot_interval_minutes, c.retention_count, c.external_store, nowIso(), by);
|
||||
audit(d.store, { actorType: 'operator', actorId: by, action: 'dr.config', detail: c }); return getConfig(d);
|
||||
}
|
||||
export const snapshotDir = (d) => { const p = join(d.config.dataDir, 'dr'); mkdirSync(p, { recursive: true, mode: 0o700 }); return p; };
|
||||
const sha = (f) => createHash('sha256').update(readFileSync(f)).digest('hex');
|
||||
/** 복구 사본 만들기(동기, 수 초). DB 는 VACUUM INTO 로 일관 사본. */
|
||||
export function snapshot(d, { kind = 'scheduled', by = null } = {}) {
|
||||
const id = newId('drs_'); const dir = snapshotDir(d); const work = join(dir, id); mkdirSync(work, { recursive: true });
|
||||
try {
|
||||
const dbCopy = join(work, 'hal.sqlite'); d.store.exec(`VACUUM INTO '${dbCopy.replace(/'/g, "''")}'`);
|
||||
for (const f of ['vault.json', 'config.json']) { const p = join(d.config.dataDir, f); if (existsSync(p)) copyFileSync(p, join(work, f)); }
|
||||
const rel = core.listReleases(d).map((r) => ({ version: r.version, main_commit: r.main_commit, artifacts: r.artifacts, status: r.status })); const meta = { id, createdAt: nowIso(), halVersion: d.version, instanceId: d.instanceId, releases: rel, note: 'vault.key 는 포함하지 않음 — 운영자가 따로 보관' };
|
||||
writeFileSync(join(work, 'meta.json'), JSON.stringify(meta, null, 1));
|
||||
const tar = join(dir, `${id}.tar.gz`); execFileSync('tar', ['-czf', tar, '-C', work, '.']); rmSync(work, { recursive: true, force: true });
|
||||
const sum = sha(tar); const size = statSync(tar).size;
|
||||
// 읽기 검증: 목록을 읽을 수 있고 meta.json 이 들어 있어야 한다
|
||||
const list = execFileSync('tar', ['-tzf', tar]).toString(); const ok = list.includes('meta.json') && list.includes('hal.sqlite');
|
||||
d.store.run('INSERT INTO dr_snapshots(id, created_at, location, size_bytes, sha256, verify_state, verified_at, kind) VALUES(?,?,?,?,?,?,?,?)', id, nowIso(), tar, size, sum, ok ? 'ok' : 'failed', nowIso(), kind);
|
||||
const cfg = getConfig(d);
|
||||
if (ok && cfg.external_store) { try { mkdirSync(cfg.external_store, { recursive: true }); const ext = join(cfg.external_store, `${id}.tar.gz`); copyFileSync(tar, ext); if (sha(ext) !== sum) throw new Error('외부 사본 해시 불일치'); d.store.run('UPDATE dr_snapshots SET external_copied_at=?, external_location=? WHERE id=?', nowIso(), ext, id); } catch (e) { d.store.run('UPDATE dr_snapshots SET error=? WHERE id=?', `외부 보관 실패: ${e.message}`, id); } }
|
||||
prune(d); audit(d.store, { actorType: by ? 'operator' : 'system', actorId: by, action: 'dr.snapshot', subjectType: 'dr_snapshot', subjectId: id, detail: { size, ok, kind }, automatic: !by });
|
||||
return d.store.get('SELECT * FROM dr_snapshots WHERE id=?', id);
|
||||
} catch (e) { rmSync(work, { recursive: true, force: true }); d.store.run('INSERT INTO dr_snapshots(id, created_at, location, verify_state, kind, error) VALUES(?,?,?,\'failed\',?,?)', id, nowIso(), '', kind, e.message); throw e; }
|
||||
}
|
||||
/** 보존 개수 초과 사본 삭제(외부 사본은 유지) */
|
||||
export function prune(d) { const n = getConfig(d).retention_count || 14; const rows = d.store.all('SELECT id, location FROM dr_snapshots WHERE deleted_at IS NULL AND verify_state=\'ok\' ORDER BY created_at DESC, rowid DESC'); let k = 0; for (const r of rows.slice(n)) { try { rmSync(r.location, { force: true }); } catch (_) {} d.store.run('UPDATE dr_snapshots SET deleted_at=? WHERE id=?', nowIso(), r.id); k++; } return k; }
|
||||
export function listSnapshots(d) { return d.store.all('SELECT * FROM dr_snapshots ORDER BY created_at DESC, rowid DESC LIMIT 100'); }
|
||||
/** 복원 요청·승인·시작. 시작에는 "기존 제어 위치 중지" 확인이 필요하다. */
|
||||
export function requestRestore(d, { snapshotId, by, note = '' }) { const s = d.store.get('SELECT * FROM dr_snapshots WHERE id=? AND deleted_at IS NULL', snapshotId); if (!s || s.verify_state !== 'ok') throw new JobError('snapshot_unusable', '검증된 복구 사본이 아닙니다'); const id = newId('drr_'); d.store.run('INSERT INTO dr_restores(id, snapshot_id, requested_by, status, note) VALUES(?,?,?,\'requested\',?)', id, snapshotId, by, note); audit(d.store, { actorType: 'operator', actorId: by, action: 'recovery.restore_request', subjectType: 'dr_restore', subjectId: id, immutable: true }); return d.store.get('SELECT * FROM dr_restores WHERE id=?', id); }
|
||||
export function approveRestore(d, id, by) { d.store.run('UPDATE dr_restores SET approved_by=?, approved_at=?, status=\'approved\' WHERE id=? AND status=\'requested\'', by, nowIso(), id); audit(d.store, { actorType: 'operator', actorId: by, action: 'recovery.restore_approve', subjectType: 'dr_restore', subjectId: id, immutable: true }); return d.store.get('SELECT * FROM dr_restores WHERE id=?', id); }
|
||||
export function startRestore(d, id, { by, oldControlStopped, confirmText }) {
|
||||
const r = d.store.get('SELECT * FROM dr_restores WHERE id=?', id); if (!r || r.status !== 'approved') throw new JobError('restore_not_approved', '복원은 별도 승인 뒤에만 시작할 수 있습니다');
|
||||
if (oldControlStopped !== true || confirmText !== '기존 제어 위치를 중지했습니다') throw new JobError('old_control_not_confirmed', '기존 제어 위치(HAL)의 중지 확인이 필요합니다');
|
||||
const s = d.store.get('SELECT * FROM dr_snapshots WHERE id=?', r.snapshot_id);
|
||||
d.store.run('UPDATE dr_restores SET old_control_stopped_confirmed_at=?, started_at=?, status=\'started\' WHERE id=?', nowIso(), nowIso(), id);
|
||||
audit(d.store, { actorType: 'operator', actorId: by, action: 'recovery.restore_start', subjectType: 'dr_restore', subjectId: id, immutable: true });
|
||||
return { restore: d.store.get('SELECT * FROM dr_restores WHERE id=?', id), command: `sudo bash /srv/ariweb/hal/current/deploy/hal-restore.sh ${s.location} ${s.sha256}`, note: '새 HAL 기동 때 다른 인스턴스의 최근 임대가 있으면 기동을 거부합니다(하나의 HAL 만 운영)' };
|
||||
}
|
||||
/** 복원 뒤 확인·차이 기록(§34: 운영자 로그인·HAL 제어·HTTPS·고객 서비스 1개 외부 확인 + 실제 상태 비교) */
|
||||
export function finishRestore(d, id, { by, checks }) {
|
||||
const req = ['operatorLogin', 'halControl', 'https', 'customerService']; const missing = req.filter((k) => !checks || checks[k] == null);
|
||||
if (missing.length) throw new JobError('checks_incomplete', `필수 확인 항목이 비었습니다: ${missing.join(', ')}`);
|
||||
const dup = d.store.get('SELECT COUNT(*) AS n FROM jobs WHERE status=\'running\'').n; const drift = d.store.all('SELECT module, server_id, target_version, actual_version FROM module_targets WHERE target_version IS NOT NULL AND (actual_version IS NULL OR actual_version<>target_version)');
|
||||
const ok = req.every((k) => checks[k] === true) && dup === 0; const diff = { runningJobsAtFinish: dup, drift, offline: core.markOffline(d).map((x) => x.id) };
|
||||
d.store.run('UPDATE dr_restores SET finished_at=?, status=?, checks_json=?, diff_json=? WHERE id=?', nowIso(), ok ? 'completed' : 'completed_with_findings', toJson(checks), toJson(diff), id);
|
||||
audit(d.store, { actorType: 'operator', actorId: by, action: 'recovery.restore_finish', subjectType: 'dr_restore', subjectId: id, detail: { ok, diff }, immutable: true });
|
||||
return d.store.get('SELECT * FROM dr_restores WHERE id=?', id);
|
||||
}
|
||||
/** 훈련 기록 */
|
||||
export function startDrill(d, { by, snapshotId = null }) { const id = newId('drd_'); d.store.run('INSERT INTO dr_drills(id, started_at, snapshot_id, performed_by, steps_json, result) VALUES(?,?,?,?,\'[]\',\'running\')', id, nowIso(), snapshotId, by); return d.store.get('SELECT * FROM dr_drills WHERE id=?', id); }
|
||||
export function finishDrill(d, id, { by, steps, measuredRtoMinutes, measuredRpoMinutes, findings = [] }) {
|
||||
const req = ['hal', 'operatorLogin', 'vault', 'certBase', 'httpsCustomer']; const ok = req.every((k) => steps && steps[k] === true); const cfg = getConfig(d);
|
||||
const within = cfg.rto_minutes && cfg.rpo_minutes ? measuredRtoMinutes <= cfg.rto_minutes && measuredRpoMinutes <= cfg.rpo_minutes : false;
|
||||
const result = ok && within ? 'success' : 'failed'; const next = new Date(Date.now() + 90 * 86400000).toISOString();
|
||||
d.store.run('UPDATE dr_drills SET finished_at=?, steps_json=?, measured_rto_minutes=?, measured_rpo_minutes=?, result=?, findings_json=?, next_due_at=? WHERE id=?', nowIso(), toJson(steps), measuredRtoMinutes, measuredRpoMinutes, result, toJson([...(findings || []), ...(within ? [] : ['측정값이 승인된 RTO/RPO 를 넘었거나 목표가 설정되지 않음'])]), next, id);
|
||||
audit(d.store, { actorType: 'operator', actorId: by, action: 'recovery.drill', subjectType: 'dr_drill', subjectId: id, detail: { result, measuredRtoMinutes, measuredRpoMinutes }, immutable: true });
|
||||
return d.store.get('SELECT * FROM dr_drills WHERE id=?', id);
|
||||
}
|
||||
/** 정식 운영 준비 상태(§34): 설정 값이 모두 있고 마지막 훈련이 성공 */
|
||||
export function readiness(d) {
|
||||
const cfg = getConfig(d); const missing = ['rto_minutes', 'rpo_minutes', 'snapshot_interval_minutes', 'retention_count'].filter((k) => !(Number(cfg[k]) > 0));
|
||||
const lastDrill = d.store.get('SELECT * FROM dr_drills WHERE result<>\'running\' ORDER BY started_at DESC, rowid DESC LIMIT 1'); const lastSnap = d.store.get('SELECT * FROM dr_snapshots WHERE verify_state=\'ok\' AND deleted_at IS NULL ORDER BY created_at DESC, rowid DESC LIMIT 1');
|
||||
const ready = !missing.length && !!lastDrill && lastDrill.result === 'success' && !!lastSnap;
|
||||
return { ready, missingConfig: missing, lastDrill: lastDrill ? { id: lastDrill.id, result: lastDrill.result, at: lastDrill.finished_at, measuredRtoMinutes: lastDrill.measured_rto_minutes, measuredRpoMinutes: lastDrill.measured_rpo_minutes, nextDueAt: lastDrill.next_due_at } : null, lastSnapshot: lastSnap ? { id: lastSnap.id, at: lastSnap.created_at, external: !!lastSnap.external_copied_at } : null, config: cfg, label: ready ? '정식 운영 준비 완료' : '정식 운영 준비 전(복구 훈련 또는 설정 필요)' };
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* hal/emergency.mjs — 긴급 복구(§33).
|
||||
*
|
||||
* HAL 의 정상 작업 기능을 쓸 수 없는 실제 장애에만 허용한다: 선언(incident key + 사유) 때 HAL 자체 건강이 정상이고 열린 critical 사건이 없으면 거부한다.
|
||||
* 단계: declared → approved(서로 다른 내용을 확인하는 3단계: 1 영향 범위, 2 대상 식별정보 재입력, 3 최종 실행·되돌리기 가능 여부) → credential_issued(짧은 유효 시간, 1회 표시) →
|
||||
* performing(허용 범위 안의 행동만: hal.restart / hal.restore_snapshot / vault.verify / cert.reissue_internal / artifact.restore / agent.reconnect) → verified → credential_revoked → reconciled → closed.
|
||||
* 운영자가 한 명이어도 되지만 세 단계의 내용과 시각이 모두 기록되어야 한다. 일반 고객 서비스 배포(module.deploy 등)는 어떤 경우에도 거부한다.
|
||||
*/
|
||||
import { newId, randomToken, sha256, safeEqual } from '../lib/crypto.mjs';
|
||||
import { nowIso, toJson, parseJson } from '../lib/db.mjs';
|
||||
import { audit } from '../lib/audit.mjs';
|
||||
import { JobError } from '../lib/jobs.mjs';
|
||||
import * as core from './core.mjs';
|
||||
|
||||
export const SCOPES = ['hal', 'hal_data', 'vault', 'cert_base', 'artifact_store', 'first_agent'];
|
||||
export const ACTIONS = { 'hal.restart': 'hal', 'hal.restore_snapshot': 'hal_data', 'vault.verify': 'vault', 'cert.reissue_internal': 'cert_base', 'artifact.restore': 'artifact_store', 'agent.reconnect': 'first_agent' };
|
||||
const STEP_TITLES = ['영향 범위 확인', '변경되지 않는 대상 식별정보 재확인', '최종 실행과 되돌리기 가능 여부 확인'];
|
||||
|
||||
function get(d, id) { const r = d.store.get('SELECT * FROM emergency_sessions WHERE id=?', id); if (!r) throw new JobError('emergency_not_found', '긴급 복구 세션이 없습니다'); return { ...r, scope: parseJson(r.scope_json, []), approvals: parseJson(r.approvals_json, []), actions: parseJson(r.actions_json, []), verify: parseJson(r.verify_json, null), reconcile: parseJson(r.reconcile_json, null) }; }
|
||||
export const getSession = get;
|
||||
export function listSessions(d) { return d.store.all('SELECT * FROM emergency_sessions ORDER BY declared_at DESC LIMIT 50').map((r) => get(d, r.id)); }
|
||||
export function declare(d, { incidentKey, reason, scope, by, force = false }) {
|
||||
if (!incidentKey || !reason) throw new JobError('declare_incomplete', '사건 번호와 복구 이유가 필요합니다');
|
||||
const bad = (scope || []).filter((s) => !SCOPES.includes(s)); if (bad.length || !scope?.length) throw new JobError('bad_scope', `허용 대상은 ${SCOPES.join(', ')} 입니다`);
|
||||
const health = core.selfHealth(d); const openCritical = d.store.get('SELECT id FROM incidents WHERE severity=\'critical\' AND resolved_at IS NULL');
|
||||
if (health.ok && !openCritical && !force) throw new JobError('hal_healthy', 'HAL 정상 작업 기능을 쓸 수 있는 상태입니다 — 긴급 복구가 아니라 일반 작업을 쓰세요. 실제 장애면 사건을 먼저 등록하세요');
|
||||
const id = newId('em_'); const now = nowIso();
|
||||
d.store.run('INSERT INTO emergency_sessions(id, incident_key, reason, scope_json, status, declared_by, declared_at, hal_healthy_at_declare, updated_at) VALUES(?,?,?,?,\'declared\',?,?,?,?)', id, incidentKey, reason, toJson(scope), by, now, health.ok ? 1 : 0, now);
|
||||
audit(d.store, { actorType: 'operator', actorId: by, action: 'emergency.declare', subjectType: 'emergency', subjectId: id, detail: { incidentKey, reason, scope, forced: force && health.ok }, immutable: true });
|
||||
return get(d, id);
|
||||
}
|
||||
/** 3단계 승인. 각 단계는 서로 다른 입력을 요구한다(같은 문구 반복 불가). */
|
||||
export function approve(d, id, { step, by, impactText, identifiers, finalConfirm, revertible }) {
|
||||
const s = get(d, id); if (!['declared', 'approving'].includes(s.status)) throw new JobError('bad_state', `지금 상태(${s.status})에서는 승인할 수 없습니다`);
|
||||
const n = s.approvals.length + 1; if (step !== n) throw new JobError('wrong_step', `지금은 ${n}단계(${STEP_TITLES[n - 1]})를 진행해야 합니다`);
|
||||
let record;
|
||||
if (n === 1) { if (!impactText || String(impactText).length < 10) throw new JobError('impact_required', '영향 범위를 구체적으로 적어야 합니다(10자 이상)'); record = { step: 1, title: STEP_TITLES[0], impactText, by, at: nowIso() }; }
|
||||
else if (n === 2) { const need = s.scope.slice().sort(); const got = (identifiers || []).slice().sort(); if (JSON.stringify(need) !== JSON.stringify(got)) throw new JobError('identifiers_mismatch', `대상 식별정보가 선언한 범위(${need.join(', ')})와 다릅니다`); record = { step: 2, title: STEP_TITLES[1], identifiers: got, by, at: nowIso() }; }
|
||||
else { if (finalConfirm !== `복구 실행 ${s.incident_key}`) throw new JobError('final_confirm_mismatch', `최종 확인 문구는 "복구 실행 ${s.incident_key}" 이어야 합니다`); if (typeof revertible !== 'boolean') throw new JobError('revertible_required', '되돌리기 가능 여부를 명시해야 합니다'); record = { step: 3, title: STEP_TITLES[2], finalConfirm: true, revertible, by, at: nowIso() }; }
|
||||
const approvals = [...s.approvals, record]; const status = approvals.length === 3 ? 'approved' : 'approving';
|
||||
d.store.run('UPDATE emergency_sessions SET approvals_json=?, status=?, updated_at=? WHERE id=?', toJson(approvals), status, nowIso(), id);
|
||||
audit(d.store, { actorType: 'operator', actorId: by, action: `emergency.approve_${n}`, subjectType: 'emergency', subjectId: id, detail: record, immutable: true });
|
||||
return get(d, id);
|
||||
}
|
||||
/** 자격 발급(승인 3단계 완료 뒤, 1회 표시, 짧은 유효 시간) */
|
||||
export function issueCredential(d, id, by) {
|
||||
const s = get(d, id); if (s.status !== 'approved' || s.approvals.length < 3) throw new JobError('approvals_insufficient', `승인이 부족합니다(${s.approvals.length}/3)`);
|
||||
const cred = randomToken(32); const exp = new Date(Date.now() + d.config.emergencyTtlMinutes * 60000).toISOString();
|
||||
d.store.run('UPDATE emergency_sessions SET credential_hash=?, credential_issued_at=?, credential_expires_at=?, status=\'credential_issued\', updated_at=? WHERE id=?', sha256(cred), nowIso(), exp, nowIso(), id);
|
||||
audit(d.store, { actorType: 'operator', actorId: by, action: 'emergency.credential_issue', subjectType: 'emergency', subjectId: id, detail: { expiresAt: exp }, immutable: true });
|
||||
return { credential: cred, expiresAt: exp };
|
||||
}
|
||||
function checkCredential(s, credential) { if (!s.credential_hash) throw new JobError('no_credential', '자격이 발급되지 않았습니다'); if (s.credential_revoked_at) throw new JobError('credential_revoked', '자격이 폐기되었습니다'); if (Date.parse(s.credential_expires_at) < Date.now()) throw new JobError('credential_expired', '긴급 자격이 만료되었습니다 — 새로 승인·발급하세요'); if (!safeEqual(sha256(String(credential || '')), s.credential_hash)) throw new JobError('bad_credential', '자격이 올바르지 않습니다'); }
|
||||
/** 행동 수행. 허용 범위 밖·일반 배포는 거부. 실행기(d.emergencyRunner)가 실제 동작을 하고 결과를 돌려준다(테스트는 가짜). */
|
||||
export async function perform(d, id, { credential, action, target = null, by }) {
|
||||
const s = get(d, id); checkCredential(s, credential);
|
||||
if (!['credential_issued', 'performing'].includes(s.status)) throw new JobError('bad_state', `지금 상태(${s.status})에서는 수행할 수 없습니다`);
|
||||
const scopeOf = ACTIONS[action];
|
||||
if (!scopeOf) { log(d, id, { action, target, by, result: 'rejected', reason: 'not_an_emergency_action' }); throw new JobError('action_forbidden', `긴급 복구에서 허용되지 않는 행동입니다: ${action} (일반 배포는 HAL 정상화 뒤 HAL 작업으로만)`); }
|
||||
if (!s.scope.includes(scopeOf)) { log(d, id, { action, target, by, result: 'rejected', reason: 'out_of_scope' }); throw new JobError('out_of_scope', `선언한 허용 대상(${s.scope.join(', ')}) 밖의 행동입니다: ${action}`); }
|
||||
let result; try { result = d.emergencyRunner ? await d.emergencyRunner(action, target, s) : { ok: true, simulated: true }; } catch (e) { log(d, id, { action, target, by, result: 'failed', error: e.message }); throw e; }
|
||||
log(d, id, { action, target, by, result: 'ok', detail: result }); d.store.run('UPDATE emergency_sessions SET status=\'performing\', updated_at=? WHERE id=?', nowIso(), id);
|
||||
return result;
|
||||
}
|
||||
function log(d, id, entry) { const s = get(d, id); const actions = [...s.actions, { at: nowIso(), ...entry }]; d.store.run('UPDATE emergency_sessions SET actions_json=?, updated_at=? WHERE id=?', toJson(actions), nowIso(), id); audit(d.store, { actorType: 'operator', actorId: entry.by, action: 'emergency.action', subjectType: 'emergency', subjectId: id, outcome: entry.result === 'ok' ? 'ok' : entry.result === 'rejected' ? 'denied' : 'failed', detail: entry, immutable: true }); }
|
||||
/** 실제 결과 확인 기록 → 자격 폐기 */
|
||||
export function verifyAndRevoke(d, id, { by, verify }) { const s = get(d, id); if (!['performing', 'credential_issued'].includes(s.status)) throw new JobError('bad_state', '수행 단계가 아닙니다'); const health = core.selfHealth(d); const v = { ...(verify || {}), halHealth: health.ok, at: nowIso() }; d.store.run('UPDATE emergency_sessions SET verify_json=?, credential_revoked_at=?, status=\'verified\', updated_at=? WHERE id=?', toJson(v), nowIso(), nowIso(), id); audit(d.store, { actorType: 'operator', actorId: by, action: 'emergency.verify_revoke', subjectType: 'emergency', subjectId: id, detail: v, immutable: true }); return get(d, id); }
|
||||
/** HAL 정상화 뒤 실제 상태와 원하는 상태 비교(모듈 목표·실제 버전 차이, 오프라인 서버) → 차이 목록 */
|
||||
export function reconcile(d, id, by) { const s = get(d, id); if (s.status !== 'verified') throw new JobError('bad_state', '검증·자격 폐기 뒤에 조정할 수 있습니다'); const drift = d.store.all('SELECT module, server_id, target_version, actual_version, health FROM module_targets WHERE (target_version IS NOT NULL AND (actual_version IS NULL OR actual_version<>target_version)) OR health=\'down\''); const offline = core.markOffline(d); const r = { at: nowIso(), drift, offlineServers: offline.map((x) => x.id), halHealth: core.selfHealth(d).ok }; d.store.run('UPDATE emergency_sessions SET reconcile_json=?, status=\'reconciled\', updated_at=? WHERE id=?', toJson(r), nowIso(), id); audit(d.store, { actorType: 'operator', actorId: by, action: 'emergency.reconcile', subjectType: 'emergency', subjectId: id, detail: r, immutable: true }); return r; }
|
||||
export function close(d, id, by, note = '') { const s = get(d, id); if (!['reconciled', 'verified', 'declared', 'approving', 'approved'].includes(s.status)) throw new JobError('bad_state', '지금 상태에서는 종료할 수 없습니다'); if (s.credential_hash && !s.credential_revoked_at) d.store.run('UPDATE emergency_sessions SET credential_revoked_at=? WHERE id=?', nowIso(), id); d.store.run('UPDATE emergency_sessions SET status=\'closed\', closed_at=?, closed_by=?, updated_at=? WHERE id=?', nowIso(), by, nowIso(), id); audit(d.store, { actorType: 'operator', actorId: by, action: 'emergency.close', subjectType: 'emergency', subjectId: id, detail: { note }, immutable: true }); return get(d, id); }
|
||||
/** 만료 자격 자동 폐기(주기) */
|
||||
export function expireCredentials(d) { return d.store.run('UPDATE emergency_sessions SET credential_revoked_at=?, updated_at=? WHERE credential_hash IS NOT NULL AND credential_revoked_at IS NULL AND credential_expires_at<?', nowIso(), nowIso(), nowIso()).changes; }
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* hal/index.mjs — HAL 프로세스 진입점과 CLI.
|
||||
*
|
||||
* node hal/index.mjs 서버 실행(기본). 기동 때 "하나의 HAL" 임대를 잡고(§34) 실행 버전을 기록한다(§32).
|
||||
* node hal/index.mjs setup-token 최초 운영자 계정을 만들 설정 토큰을 발급(1회 표시, data/setup.token). 부트스트랩 스크립트가 부른다.
|
||||
* node hal/index.mjs grant --login <id> [--hours 8] 허용 네트워크 밖 기기용 접근권 URL 발급(§25)
|
||||
* node hal/index.mjs register-release --manifest <dist/<ver>/manifest.json> [--approve] 릴리스 등록(부트스트랩·CI)
|
||||
* node hal/index.mjs link-portal 포털 연동 서비스 키 생성(service:portal / service:hal) — 포털 보관소에도 같은 값을 넣는다(출력 1회)
|
||||
* node hal/index.mjs bootstrap-release --version <v> --commit <sha> --sha256 <artifact sha> 부트스트랩 때 선택한 릴리스 정보를 기록
|
||||
*/
|
||||
import { loadConfig } from './config.mjs';
|
||||
import { createHalDeps, VERSION } from './deps.mjs';
|
||||
import { buildApp } from './app.mjs';
|
||||
import * as core from './core.mjs';
|
||||
import * as ops from './ops.mjs';
|
||||
import * as emergency from './emergency.mjs';
|
||||
import * as dr from './dr.mjs';
|
||||
import { purgeSessions } from '../lib/session.mjs';
|
||||
import { randomToken } from '../lib/crypto.mjs';
|
||||
import { writeFileSync, readFileSync, copyFileSync, mkdirSync, existsSync } from 'node:fs';
|
||||
import { join, dirname, basename } from 'node:path';
|
||||
|
||||
const log = (...a) => console.log(new Date().toISOString(), ...a);
|
||||
const cfg = loadConfig(); const args = process.argv.slice(2); const cmd = args[0] && !args[0].startsWith('--') ? args[0] : 'serve';
|
||||
const opt = (name, def = null) => { const i = args.indexOf('--' + name); return i >= 0 ? (args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : true) : def; };
|
||||
const d = createHalDeps({ config: cfg, log });
|
||||
|
||||
if (cmd === 'setup-token') {
|
||||
if (core.ensureOpsAccountExists(d)) { console.error('운영자 계정이 이미 있습니다 — 설정 토큰은 최초 1회만 쓸 수 있습니다'); process.exit(1); }
|
||||
const t = randomToken(24); writeFileSync(join(cfg.dataDir, 'setup.token'), t, { mode: 0o600 });
|
||||
console.log(`설정 토큰(1회 표시, 공유 금지): ${t}\n운영자 콘솔 ${cfg.publicUrl}/ops/ 의 최초 설정 또는 curl -X POST ${cfg.publicUrl}/ops/api/setup -d '{"setupToken":"…","login":"…","password":"…","displayName":"…"}' 로 첫 운영자 계정을 만드세요`);
|
||||
} else if (cmd === 'grant') {
|
||||
const g = ops.issueGrant(d, { accountId: opt('login'), hours: Number(opt('hours', 8)), by: null, reason: opt('reason', 'cli') }); console.log(`접근권 URL(1회, ${g.expiresAt} 까지): ${g.url}`);
|
||||
} else if (cmd === 'register-release') {
|
||||
const mf = JSON.parse(readFileSync(opt('manifest'), 'utf8')); const dir = join(core.artifactDir(d), mf.version); mkdirSync(dir, { recursive: true });
|
||||
for (const [m, a] of Object.entries(mf.artifacts)) { const src = join(dirname(opt('manifest')), a.file); if (existsSync(src)) copyFileSync(src, join(dir, basename(a.file))); }
|
||||
const r = core.registerRelease(d, { version: mf.version, mainCommit: mf.mainCommit, builtAt: mf.builtAt, artifacts: Object.fromEntries(Object.entries(mf.artifacts).map(([m, a]) => [m, { file: basename(a.file), sha256: a.sha256, size: a.size }])), checks: mf.checks || null, approve: !!opt('approve'), by: opt('by', 'cli') });
|
||||
console.log(`릴리스 ${r.version} 등록(${r.status}) — artifact ${Object.keys(r.artifacts).join(', ')}`);
|
||||
} else if (cmd === 'link-portal') {
|
||||
const p = randomToken(32), h = randomToken(32); d.vault.set('service:portal', p, { kind: 'service-key' }); d.vault.set('service:hal', h, { kind: 'service-key' });
|
||||
console.log(`포털 보관소에 넣을 값(1회 표시):\n service:hal = ${p} (포털이 HAL 을 부를 때 쓰는 키 — 포털 vault 이름 'service:hal')\n service:portal = ${h} (HAL 이 포털을 부를 때 쓰는 키 — 포털 vault 이름 'service:portal')`);
|
||||
} else if (cmd === 'bootstrap-release') {
|
||||
core.setBootstrap(d, { release_version: opt('version'), main_commit: opt('commit'), artifact_sha256: opt('sha256') }); console.log('부트스트랩 릴리스 정보를 기록했습니다');
|
||||
} else if (cmd === 'serve') {
|
||||
// "하나의 HAL" 임대(§34). 다른 인스턴스가 살아 있으면 기동 거부(--takeover 는 재해복구 절차에서만).
|
||||
try { core.claimLive(d, { instanceId: d.instanceId, version: VERSION, takeover: !!opt('takeover') }); } catch (e) { console.error(`[hal] 기동 거부: ${e.message}`); process.exit(2); }
|
||||
const app = buildApp(d); d.jobs.start(); d.jobsRunning = true;
|
||||
const every = (ms, fn) => setInterval(() => { try { const r = fn(); if (r && r.catch) r.catch((e) => log('[hal] 주기 작업 오류', e.message)); } catch (e) { log('[hal] 주기 작업 오류', e.message); } }, ms).unref();
|
||||
every(20000, () => core.renewLease(d, d.instanceId)); every(30000, () => core.markOffline(d)); every(3000, () => core.flushPortalEvents(d)); every(60000, () => emergency.expireCredentials(d));
|
||||
every(3600000, () => { core.purgeMetrics(d); purgeSessions(d.store); });
|
||||
every(60000, () => { const c = dr.getConfig(d); if (!c.snapshot_interval_minutes) return; const last = d.store.get('SELECT created_at FROM dr_snapshots WHERE verify_state=\'ok\' ORDER BY created_at DESC LIMIT 1'); if (!last || Date.now() - Date.parse(last.created_at) >= c.snapshot_interval_minutes * 60000) dr.snapshot(d, { kind: 'scheduled' }); });
|
||||
await app.listen(cfg.port, cfg.host);
|
||||
log(`[hal] Ariweb HAL v${VERSION} — http://${cfg.host}:${cfg.port} (공개 ${cfg.publicUrl}, 데이터 ${cfg.dataDir}, 부트스트랩 ${core.bootstrapComplete(d) ? '완료' : '진행 중'})`);
|
||||
process.on('SIGTERM', () => { d.stopping = true; d.jobs.stop(); setTimeout(() => process.exit(0), 500); });
|
||||
} else { console.error(`알 수 없는 명령: ${cmd}`); process.exit(1); }
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* hal/jobs.mjs — HAL 작업 종류(lib/jobs 엔진에 등록).
|
||||
*
|
||||
* 두 부류가 있다.
|
||||
* 1) 포털이 요청하는 작업(docs/HAL-JOBS.md): site.*, domain.*, backup.* — 대상 서버(또는 외부 Agent)에 Agent 작업을 배정하고 결과를 기다린 뒤 돌려준다.
|
||||
* 복합 작업(backup.create 의 원격 저장, site.relocate, backup.restore)은 여러 Agent 작업을 단계로 잇는다. 모든 단계는 ctx.step 으로 기록되어 HAL 재시작 뒤 이어서 실행된다.
|
||||
* 2) 운영자 작업(§32·§29): module.deploy / module.rollback / module.health / module.reconcile — 대상 서버마다 자식 작업을 만들고 대상별 결과를 요약한다. 일부 실패는 'partial' 이며 성공으로 보이지 않는다.
|
||||
*
|
||||
* 대상(target) 해석: 서버 id(srv_…) → 그 서버의 Agent, Agent id(ag_…) → 그 Agent(외부 Agent 포함).
|
||||
*/
|
||||
import { JobError } from '../lib/jobs.mjs';
|
||||
import { nowIso, toJson } from '../lib/db.mjs';
|
||||
import { audit } from '../lib/audit.mjs';
|
||||
import { notify } from '../lib/notify.mjs';
|
||||
import { safeError } from '../lib/redact.mjs';
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import * as core from './core.mjs';
|
||||
import { descriptor, DEPLOY_ORDER, ARTIFACT_MODULES } from './modules/index.mjs';
|
||||
|
||||
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
/** 대상 → Agent id */
|
||||
function agentFor(d, target) {
|
||||
if (!target) throw new JobError('target_required', '대상 서버 또는 Agent 가 필요합니다');
|
||||
if (String(target).startsWith('ag_')) { const a = core.getAgent(d, target); if (!a || a.status !== 'active') throw new JobError('agent_unavailable', 'Agent 가 없거나 해지되었습니다'); return a.id; }
|
||||
const s = core.getServer(d, target); if (!s) throw new JobError('server_not_found', `서버가 없습니다: ${target}`);
|
||||
if (!s.agent_id) throw new JobError('agent_missing', `서버 ${s.name} 에 Agent 가 없습니다`);
|
||||
if (s.status === 'isolated') throw new JobError('server_isolated', `서버 ${s.name} 은(는) 격리 상태입니다`);
|
||||
return s.agent_id;
|
||||
}
|
||||
/** 외부 접속 확인(§11.2 8단계). 실제 응답을 본다. */
|
||||
async function externalCheck(d, url, { expectMarker = null } = {}) {
|
||||
if (!url) return { ok: false, skipped: 'no_url' };
|
||||
const t0 = Date.now();
|
||||
try { const r = await (d.fetch || fetch)(url, { redirect: 'follow', signal: AbortSignal.timeout(15000), headers: { 'User-Agent': 'Ariweb-HAL-check/0.1' } }); const text = expectMarker ? await r.text() : ''; return { ok: r.ok && (!expectMarker || text.includes(expectMarker)), status: r.status, ms: Date.now() - t0, url, markerFound: expectMarker ? text.includes(expectMarker) : null }; }
|
||||
catch (e) { return { ok: false, error: safeError(e), ms: Date.now() - t0, url }; }
|
||||
}
|
||||
/** 단일 Agent 작업으로 끝나는 종류를 등록하는 도우미 */
|
||||
function single(engine, d, kind, { title, agentKind = kind, steps = [], build = (s) => s.payload || {}, after = null, maxAttempts = 1, target = (s) => s.target }) {
|
||||
engine.register(kind, { title, maxAttempts, steps: [{ id: 'dispatch', title: '대상 Agent 에 작업 배정' }, { id: 'await', title: '대상 서버에서 실행' }, ...steps],
|
||||
async run(job, ctx) {
|
||||
const disp = await core.ensureAgentJob(d, ctx, job, { agentId: agentFor(d, target(job.subject)), kind: agentKind, payload: build(job.subject, d) });
|
||||
const result = await ctx.step('await', () => core.awaitAgentJob(d, ctx, disp.agentJobId));
|
||||
return after ? after(job, ctx, result) : result;
|
||||
},
|
||||
retryable: (e) => !!e.retryable,
|
||||
});
|
||||
}
|
||||
|
||||
export function registerKinds(engine, d) {
|
||||
// ---------------- 사이트(§11)
|
||||
single(engine, d, 'site.deploy', { title: '사이트 배포', steps: [{ id: 'external', title: '외부 접속 확인' }], build: (s) => ({ ...s.payload, siteId: s.payload.siteId }),
|
||||
async after(job, ctx, result) {
|
||||
const ext = await ctx.step('external', () => externalCheck(d, job.subject.payload.externalCheckUrl || result?.siteUrl, { expectMarker: job.subject.payload.expectMarker }));
|
||||
return { ...result, externalCheck: ext };
|
||||
} });
|
||||
single(engine, d, 'site.rollback', { title: '이전 버전으로 되돌리기' });
|
||||
single(engine, d, 'site.remove', { title: '배포 제거' });
|
||||
single(engine, d, 'site.check', { title: '사이트 확인', async after(job, ctx, result) { const urls = job.subject.payload?.urls || []; const ext = await ctx.step('external', async () => { const out = []; for (const u of urls) out.push(await externalCheck(d, u, { expectMarker: job.subject.payload?.expectMarker })); return out; }, '외부 확인'); return { ...result, external: ext }; } });
|
||||
single(engine, d, 'site.usage', { title: '사용량 수집', maxAttempts: 3 });
|
||||
single(engine, d, 'domain.vhost', { title: '도메인 연결 설정' });
|
||||
single(engine, d, 'domain.cert', { title: 'HTTPS 인증서 발급', maxAttempts: 2 });
|
||||
single(engine, d, 'domain.probe', { title: 'DNS 확인', maxAttempts: 3 });
|
||||
|
||||
// ---------------- 백업(§19·§20). 저장 서버가 운영 서버와 다르면 HAL 전송을 거쳐 옮긴다.
|
||||
engine.register('backup.create', { title: '백업 만들기', steps: [{ id: 'create', title: '운영 서버에서 백업 생성' }, { id: 'store', title: '백업 저장소로 복사' }, { id: 'verify', title: '읽을 수 있는지 확인' }],
|
||||
async run(job, ctx) {
|
||||
const s = job.subject; const p = s.payload || {}; const siteAgent = agentFor(d, s.target); const storeServer = p.storeServerId && p.storeServerId !== s.target ? p.storeServerId : null;
|
||||
const created = await ctx.step('create', async () => { const id = core.dispatchAgentJob(d, { halJobId: job.id, agentId: siteAgent, kind: 'backup.create', payload: { ...p, uploadToHal: !!storeServer, transferId: `tr_bk_${job.id}` } }); return core.awaitAgentJob(d, ctx, id); });
|
||||
let location = `${s.target}:${created.path}`; let stored = created;
|
||||
if (storeServer) stored = await ctx.step('store', async () => { const id = core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, storeServer), kind: 'backup.store', payload: { siteId: p.siteId, backupId: created.backupId, transferId: created.transferId, sha256: created.checksum, sizeBytes: created.sizeBytes } }); const r = await core.awaitAgentJob(d, ctx, id); location = `${storeServer}:${r.path}`; return r; });
|
||||
else await ctx.step('store', () => ({ skipped: '같은 서버에 보관(저장 서버 미지정)', warning: '운영 서버와 분리된 저장소를 권장합니다(§19.2)' }));
|
||||
const verified = await ctx.step('verify', async () => { const id = core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, storeServer || s.target), kind: 'backup.verify', payload: { location: stored.path, checksum: created.checksum } }); return core.awaitAgentJob(d, ctx, id); });
|
||||
if (!verified?.ok) throw new JobError('backup_unreadable', '생성한 백업을 읽을 수 없습니다');
|
||||
return { backupId: created.backupId, location, sizeBytes: created.sizeBytes, checksum: created.checksum, verified: true, createdAt: created.createdAt, scope: created.scope };
|
||||
} });
|
||||
const splitLoc = (loc) => { const i = String(loc).indexOf(':'); return i > 0 ? { serverId: loc.slice(0, i), path: loc.slice(i + 1) } : { serverId: null, path: loc }; };
|
||||
single(engine, d, 'backup.verify', { title: '백업 검증', target: (s) => splitLoc(s.payload.location).serverId || s.target, build: (s) => ({ location: splitLoc(s.payload.location).path, checksum: s.payload.checksum }) });
|
||||
single(engine, d, 'backup.delete', { title: '백업 삭제', target: (s) => splitLoc(s.payload.location).serverId || s.target, build: (s) => ({ location: splitLoc(s.payload.location).path }) });
|
||||
engine.register('backup.restore', { title: '사이트 복원', steps: [{ id: 'fetch', title: '백업 저장소에서 가져오기' }, { id: 'protect', title: '현재 상태 보호' }, { id: 'isolated', title: '격리된 환경에서 백업 확인' }, { id: 'restore', title: '데이터 복원' }, { id: 'external', title: '외부 접속 확인' }],
|
||||
async run(job, ctx) {
|
||||
const s = job.subject; const p = s.payload || {}; const loc = splitLoc(p.location); const siteAgent = agentFor(d, s.target); const remote = loc.serverId && loc.serverId !== s.target;
|
||||
const fetched = await ctx.step('fetch', async () => { if (!remote) return { path: loc.path }; const id = core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, loc.serverId), kind: 'backup.fetch', payload: { location: loc.path, transferId: `tr_rs_${job.id}`, checksum: p.checksum } }); return core.awaitAgentJob(d, ctx, id); });
|
||||
const r = await ctx.step('restore', async () => { const id = core.dispatchAgentJob(d, { halJobId: job.id, agentId: siteAgent, kind: 'backup.restore', payload: { ...p, location: remote ? null : fetched.path, transferId: remote ? fetched.transferId : null } }); return core.awaitAgentJob(d, ctx, id); });
|
||||
await ctx.step('protect', () => ({ protectPointLocation: r.protectPointLocation })); await ctx.step('isolated', () => ({ ok: r.isolatedVerify }));
|
||||
const ext = await ctx.step('external', () => externalCheck(d, p.externalCheckUrl, { expectMarker: p.expectMarker }));
|
||||
if (p.externalCheckUrl && !ext.ok) throw new JobError('restore_external_check_failed', '복원 뒤 외부 접속 확인에 실패했습니다 — 보호 시점으로 되돌릴 수 있습니다', { detail: ext });
|
||||
return { ...r, externalCheck: ext };
|
||||
},
|
||||
async revert(job, ctx) { const s = job.subject; const r = job.steps.find((x) => x.id === 'restore')?.result; if (!r?.protectPointLocation) throw new JobError('no_protect_point', '보호 시점이 없어 되돌릴 수 없습니다'); await ctx.step('revert', async () => { const id = core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, s.target), kind: 'backup.restore', payload: { siteId: s.payload.siteId, location: r.protectPointLocation, protectPoint: false, isolatedVerify: true }, dedupeKey: `${job.id}:revert` }); return core.awaitAgentJob(d, ctx, id); }, '보호 시점으로 되돌리기'); } });
|
||||
|
||||
// ---------------- 서버 이전(§21). 두 서버에 동시에 운영 상태가 되지 않도록 verify_new 뒤에만 switch 한다.
|
||||
engine.register('site.relocate', { title: '서버 이전', steps: ['precheck', 'prepare', 'precopy', 'verify_copy', 'final_sync', 'verify_new', 'switch', 'external_check', 'cleanup'].map((id) => ({ id, title: { precheck: '조건 확인', prepare: '새 위치 준비', precopy: '데이터 사전 복사', verify_copy: '복사 결과 확인', final_sync: '최종 변경분 반영', verify_new: '새 위치 확인', switch: '연결 전환', external_check: '외부 접속 확인', cleanup: '이전 위치 정리' }[id] })),
|
||||
async run(job, ctx) {
|
||||
const p = job.subject.payload || {}; const from = agentFor(d, p.fromServerId); const to = agentFor(d, p.toServerId); const dyn = p.kind === 'dynamic';
|
||||
await ctx.step('precheck', () => { const t = core.getServer(d, p.toServerId); const f = core.getServer(d, p.fromServerId); if (!t.accepts_new || t.status !== 'ready') throw new JobError('target_not_ready', `도착 서버 ${t.name} 이(가) 신규 배치를 받을 수 없습니다`); if (f.status === 'offline') throw new JobError('source_offline', '출발 서버가 응답하지 않습니다'); const conflict = d.store.get('SELECT id FROM jobs WHERE subject_id=? AND id<>? AND kind IN (\'site.deploy\',\'backup.restore\',\'site.relocate\') AND status IN (\'queued\',\'running\',\'awaiting_external\',\'needs_user\')', p.siteId, job.id); if (conflict) throw new JobError('conflict', `같은 사이트의 다른 작업(${conflict.id})이 진행 중입니다`); return { from: f.name, to: t.name }; });
|
||||
await ctx.step('prepare', async () => core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: to, kind: 'site.prepare', payload: { siteId: p.siteId } })));
|
||||
const pre = await ctx.step('precopy', async () => core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: from, kind: 'site.export', payload: { siteId: p.siteId, transferId: `tr_rl1_${job.id}`, includeData: true } })));
|
||||
await ctx.step('verify_copy', async () => core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: to, kind: 'site.import', payload: { siteId: p.siteId, transferId: pre.transferId, checksum: pre.checksum, stage: 'pre' } })));
|
||||
const fin = await ctx.step('final_sync', async () => { if (dyn || p.writeFreeze) await core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: from, kind: 'site.freeze', payload: { siteId: p.siteId, freeze: true } })); const ex = await core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: from, kind: 'site.export', payload: { siteId: p.siteId, transferId: `tr_rl2_${job.id}`, includeData: true }, dedupeKey: `${job.id}:final` })); return core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: to, kind: 'site.import', payload: { siteId: p.siteId, transferId: ex.transferId, checksum: ex.checksum, stage: 'final' }, dedupeKey: `${job.id}:final-import` })); });
|
||||
const vn = await ctx.step('verify_new', async () => core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: to, kind: 'site.stage_check', payload: { siteId: p.siteId, expectedChecksum: fin.checksum } })));
|
||||
if (!vn?.ok) throw new JobError('verify_new_failed', '새 위치 확인에 실패해 전환하지 않습니다', { detail: vn });
|
||||
await ctx.step('switch', async () => { ctx.checkCancel(); const a = await core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: to, kind: 'site.activate', payload: { siteId: p.siteId, domains: p.domains || [], primaryDomain: p.primaryDomain, runtime: p.runtime || 'static' } })); await core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: from, kind: 'site.deactivate', payload: { siteId: p.siteId, retainDays: p.retainDays || 7 } })); return { switchedAt: nowIso(), activate: a }; });
|
||||
const ext = await ctx.step('external_check', () => externalCheck(d, p.externalCheckUrl, { expectMarker: p.expectMarker }));
|
||||
if (p.externalCheckUrl && !ext.ok) throw new JobError('relocate_external_check_failed', '전환 뒤 외부 접속 확인에 실패했습니다 — 되돌리기를 검토하세요', { detail: ext });
|
||||
const cl = await ctx.step('cleanup', () => ({ oldRetainedUntil: new Date(Date.now() + (p.retainDays || 7) * 86400000).toISOString(), note: '이전 위치 사본은 보존 기간 뒤 삭제 작업(site.remove)으로 정리' }));
|
||||
return { newServerId: p.toServerId, switchedAt: job.steps?.find((x) => x.id === 'switch')?.result?.switchedAt || nowIso(), oldRetainedUntil: cl.oldRetainedUntil, externalCheck: ext };
|
||||
},
|
||||
async revert(job, ctx) { const p = job.subject.payload || {}; const sw = (job.steps || []).find((x) => x.id === 'switch'); if (!sw || sw.state !== 'done') { await ctx.step('revert_unfreeze', async () => core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, p.fromServerId), kind: 'site.freeze', payload: { siteId: p.siteId, freeze: false }, dedupeKey: `${job.id}:unfreeze` }))); return; }
|
||||
await ctx.step('revert_switch', async () => { await core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, p.fromServerId), kind: 'site.activate', payload: { siteId: p.siteId, domains: p.domains || [], primaryDomain: p.primaryDomain, runtime: p.runtime || 'static', reactivate: true }, dedupeKey: `${job.id}:revert-act` })); await core.awaitAgentJob(d, ctx, core.dispatchAgentJob(d, { halJobId: job.id, agentId: agentFor(d, p.toServerId), kind: 'site.deactivate', payload: { siteId: p.siteId, retainDays: 1 }, dedupeKey: `${job.id}:revert-deact` })); }, '이전 위치로 연결 되돌리기'); } });
|
||||
|
||||
// ---------------- 모듈 배포(§32)
|
||||
const unitTemplate = (name) => { const f = name ? join(ROOT, 'deploy', 'units', name) : null; return f && existsSync(f) ? readFileSync(f, 'utf8') : null; };
|
||||
/** 배포 전제: 릴리스 승인, artifact 존재, HAL 먼저(다른 모듈은 HAL 이 그 릴리스로 실행 중이어야), 부트스트랩 완료 */
|
||||
function precheckModule(module, version) {
|
||||
const rel = core.getRelease(d, version); if (!rel) throw new JobError('release_not_found', `릴리스 ${version} 이(가) 없습니다`);
|
||||
if (rel.status !== 'approved') throw new JobError('release_not_approved', `릴리스 ${version} 은(는) 승인되지 않았습니다`);
|
||||
if (ARTIFACT_MODULES.includes(module) && !core.artifactPath(d, version, module)) throw new JobError('artifact_missing', `릴리스 ${version} 의 ${module} artifact 가 없습니다`);
|
||||
const health = core.selfHealth(d); if (!health.ok) throw new JobError('hal_unhealthy', 'HAL 자체 건강 확인에 실패해 배포하지 않습니다');
|
||||
if (!core.bootstrapComplete(d)) throw new JobError('bootstrap_incomplete', '부트스트랩(HAL 설치·첫 Agent 등록)이 끝나지 않았습니다');
|
||||
if (module !== 'hal' && health.bootstrap.halRunningVersion !== version) throw new JobError('hal_first', `HAL 이 아직 ${version} 으로 실행되고 있지 않습니다(현재 ${health.bootstrap.halRunningVersion}). HAL 을 먼저 배포하세요`);
|
||||
return rel;
|
||||
}
|
||||
engine.register('module.deploy.target', { title: '모듈 배포(대상)', steps: [{ id: 'dispatch', title: 'Agent 에 설치 지시' }, { id: 'await', title: '설치·재시작' }, { id: 'verify', title: '실제 버전·건강 확인' }],
|
||||
async run(job, ctx) {
|
||||
const { module, version, serverId, action = 'install' } = job.subject; const desc = descriptor(module); const srv = core.getServer(d, serverId); if (!srv) throw new JobError('server_not_found', '서버가 없습니다');
|
||||
const art = core.artifactPath(d, version, module);
|
||||
d.store.run('INSERT INTO module_targets(module, server_id, target_version, last_job_id, updated_at) VALUES(?,?,?,?,?) ON CONFLICT(module, server_id) DO UPDATE SET target_version=excluded.target_version, last_job_id=excluded.last_job_id, updated_at=excluded.updated_at', module, serverId, version, job.id, nowIso());
|
||||
const facts = srv.facts || {}; const unitName = (facts.units && facts.units[module]) || desc.unit;
|
||||
const disp = await core.ensureAgentJob(d, ctx, job, { agentId: agentFor(d, serverId), kind: action === 'rollback' ? 'module.rollback' : 'module.install', title: 'Agent 에 설치 지시', payload: { module, version, action, artifactUrl: art ? `/agent/v1/releases/${version}/artifact?module=${module}` : null, sha256: art?.sha256, size: art?.size, descriptor: { ...desc, unit: unitName }, unitTemplate: unitTemplate(desc.unitTemplate), appDir: (facts.appDirs && facts.appDirs[module]) || (desc.baseDir || `/srv/ariweb/${module}`), toVersion: job.subject.toVersion || null } });
|
||||
const r = await ctx.step('await', () => core.awaitAgentJob(d, ctx, disp.agentJobId));
|
||||
const v = await ctx.step('verify', () => {
|
||||
const expected = action === 'rollback' ? (job.subject.toVersion || r.actualVersion) : version; const ok = r.actualVersion === expected && r.health?.ok !== false;
|
||||
d.store.run('UPDATE module_targets SET actual_version=?, health=?, active=?, last_check_at=?, last_error=?, updated_at=? WHERE module=? AND server_id=?', r.actualVersion || null, r.health?.ok === false ? 'down' : 'ok', r.active ? 1 : 0, nowIso(), ok ? null : `기대 ${expected} 실제 ${r.actualVersion}`, nowIso(), module, serverId);
|
||||
if (!ok) throw new JobError('version_mismatch', `${srv.name}: 설치 뒤 실제 버전(${r.actualVersion})·건강(${r.health?.ok})이 목표와 다릅니다`, { detail: r });
|
||||
if (module === 'hal') { d.store.run('UPDATE releases SET hal_deployed_at=? WHERE version=?', nowIso(), version); core.setBootstrap(d, { hal_running_version: expected, release_version: expected }); }
|
||||
return { actualVersion: r.actualVersion, health: r.health, previousVersion: r.previousVersion };
|
||||
});
|
||||
return v;
|
||||
} });
|
||||
/** 부모: 대상 서버마다 자식 작업 → 대상별 결과 요약. 일부 실패는 실패로 끝낸다. */
|
||||
async function fanout(job, ctx, { kind, mk, serverIds }) {
|
||||
const ids = await ctx.step('fanout', () => serverIds.map((serverId) => engine.enqueue({ kind, subject: mk(serverId), parentId: job.id, groupId: null, subjectType: 'server', subjectId: serverId, idempotencyKey: `${job.id}:${serverId}`, createdBy: job.created_by, actorType: job.actor_type, label: `${job.label} — ${core.getServer(d, serverId)?.name || serverId}` }).job.id), '대상별 작업 만들기');
|
||||
const t0 = Date.now(); while (true) { if (Date.now() - t0 > (d.config.fanoutTimeoutMs || 12 * 3600 * 1000)) throw new JobError('fanout_timeout', '대상별 작업이 제한 시간 안에 끝나지 않았습니다'); const sum = engine.childrenSummary(job.id); ctx.progress(Math.round(100 * (sum.byStatus.done || 0) / Math.max(1, sum.total))); if (['done', 'partial', 'failed'].includes(sum.overall) && !sum.children.some((c) => ['queued', 'running', 'reverting'].includes(c.status))) { if (sum.overall !== 'done') throw new JobError(sum.overall === 'partial' ? 'partial_failure' : 'all_failed', `${sum.total}대 중 ${sum.children.filter((c) => c.status !== 'done').length}대 실패·대기 — 대상별 결과를 확인하세요`, { detail: sum }); return sum; } ctx.checkCancel(); await new Promise((r) => setTimeout(r, d.config.agentPollMs || 1000)); }
|
||||
}
|
||||
engine.register('module.deploy', { title: '모듈 배포', steps: [{ id: 'precheck', title: '릴리스·HAL·순서 확인' }, { id: 'fanout', title: '대상별 작업 만들기' }, { id: 'collect', title: '대상별 결과 확인' }],
|
||||
async run(job, ctx) {
|
||||
const { module, version, serverIds } = job.subject; await ctx.step('precheck', () => { precheckModule(module, version); if (!serverIds?.length) throw new JobError('no_targets', '대상 서버가 없습니다'); return { order: DEPLOY_ORDER.indexOf(module) }; });
|
||||
d.store.run('INSERT INTO modules(name, target_version, updated_at) VALUES(?,?,?) ON CONFLICT(name) DO UPDATE SET target_version=excluded.target_version, updated_at=excluded.updated_at', module, version, nowIso());
|
||||
const sum = await ctx.step('collect', () => fanout(job, ctx, { kind: 'module.deploy.target', serverIds, mk: (serverId) => ({ module, version, serverId, action: 'install' }) }));
|
||||
audit(d.store, { actorType: job.actor_type, actorId: job.created_by, action: 'deploy.module', subjectType: 'module', subjectId: module, detail: { version, serverIds, overall: sum.overall } });
|
||||
return sum;
|
||||
} });
|
||||
engine.register('module.rollback', { title: '모듈 되돌리기', steps: [{ id: 'fanout', title: '대상별 작업 만들기' }, { id: 'collect', title: '대상별 결과 확인' }],
|
||||
async run(job, ctx) { const { module, toVersion, serverIds } = job.subject; const sum = await ctx.step('collect', () => fanout(job, ctx, { kind: 'module.deploy.target', serverIds, mk: (serverId) => ({ module, version: toVersion || core.getRelease(d, d.store.get('SELECT target_version FROM module_targets WHERE module=? AND server_id=?', module, serverId)?.target_version)?.version, serverId, action: 'rollback', toVersion }) })); audit(d.store, { actorType: job.actor_type, actorId: job.created_by, action: 'deploy.module_rollback', subjectType: 'module', subjectId: module, detail: { toVersion, serverIds, overall: sum.overall } }); return sum; } });
|
||||
single(engine, d, 'module.health.target', { title: '모듈 건강 확인(대상)', agentKind: 'module.health', target: (s) => s.serverId, build: (s) => ({ module: s.module, descriptor: descriptor(s.module), unit: (core.getServer(d, s.serverId)?.facts?.units || {})[s.module] || descriptor(s.module).unit }),
|
||||
after(job, ctx, r) { d.store.run('UPDATE module_targets SET actual_version=COALESCE(?, actual_version), health=?, active=?, last_check_at=?, last_error=?, updated_at=? WHERE module=? AND server_id=?', r.actualVersion || null, r.health?.ok ? 'ok' : 'down', r.active ? 1 : 0, nowIso(), r.health?.ok ? null : (r.health?.error || '건강 확인 실패'), nowIso(), job.subject.module, job.subject.serverId); return r; } });
|
||||
engine.register('module.health', { title: '모듈 건강 확인', steps: [{ id: 'fanout', title: '대상별 작업 만들기' }, { id: 'collect', title: '대상별 결과 확인' }], async run(job, ctx) { const { module, serverIds } = job.subject; return ctx.step('collect', () => fanout(job, ctx, { kind: 'module.health.target', serverIds: serverIds || d.store.all('SELECT server_id FROM module_targets WHERE module=?', module).map((r) => r.server_id), mk: (serverId) => ({ module, serverId }) })); } });
|
||||
/** 다시 맞추기(§29·§34): 목표 버전과 실제 버전이 다른 대상만 다시 설치 */
|
||||
engine.register('module.reconcile', { title: '목표·실제 상태 맞추기', steps: [{ id: 'diff', title: '차이 확인' }, { id: 'fanout', title: '대상별 작업 만들기' }, { id: 'collect', title: '대상별 결과 확인' }],
|
||||
async run(job, ctx) { const diff = await ctx.step('diff', () => d.store.all('SELECT module, server_id, target_version, actual_version FROM module_targets WHERE target_version IS NOT NULL AND (actual_version IS NULL OR actual_version<>target_version)' + (job.subject.module ? ' AND module=?' : ''), ...(job.subject.module ? [job.subject.module] : []))); if (!diff.length) return { drift: [], overall: 'done' }; const sum = await ctx.step('collect', () => fanout(job, ctx, { kind: 'module.deploy.target', serverIds: diff.map((x) => x.server_id), mk: (serverId) => { const x = diff.find((y) => y.server_id === serverId); return { module: x.module, version: x.target_version, serverId, action: 'install' }; } })); return { drift: diff, ...sum }; } });
|
||||
|
||||
// ---------------- 포털 이벤트: 포털이 만든 작업(subject._portal)의 상태 변화를 outbox 에 넣는다
|
||||
engine.onChange((job) => { if (job.subject && job.subject._portal) core.queuePortalEvent(d, { halJobId: job.id, kind: job.kind, status: job.status, steps: job.steps, progress: job.progress, result: job.result, error: job.error, errorCode: job.error_code, userMessage: job.user_message, updatedAt: job.updated_at, subjectType: job.subject_type, subjectId: job.subject_id }); });
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/**
|
||||
* hal/modules/index.mjs — 모듈 기술서(descriptor) 모음(§32).
|
||||
*
|
||||
* HAL 이 어떤 모듈을 어떤 서버에 어떻게 설치·갱신·건강 확인하는지를 데이터로 둔다. Agent 는 이 기술서를 작업 payload 로 받아 그대로 수행하므로
|
||||
* Agent 바이너리는 모듈별 지식을 갖지 않는다(새 모듈을 더할 때 HAL 만 바꾸면 된다).
|
||||
*
|
||||
* 기술서 모양
|
||||
* name 모듈 이름(릴리스 artifact 이름과 같다: <name>-<version>.tar.gz)
|
||||
* roles 설치할 수 있는 서버 역할
|
||||
* layout 'releases'(= /srv/ariweb/<name>/releases/<ver> + current) — 도입 모듈도 같은 틀을 쓰되 데이터 경로는 기존 것을 유지한다
|
||||
* unit systemd 유닛 이름(도입 모듈은 서버에 이미 있는 이름을 쓸 수 있어 server.facts.units 로 덮어쓴다)
|
||||
* unitTemplate deploy/units/<file>.tmpl — 없으면 기존 유닛을 그대로 두고 WorkingDirectory 만 current 심링크가 가리키게 한다
|
||||
* health { url, expect } 내부 건강 확인(127.0.0.1)
|
||||
* version 실제 실행 버전을 읽는 방법: { file:'package.json' } (current/package.json 의 version) 또는 { http:'/api/health', field:'version' }
|
||||
* preStop/postStart Agent 가 실행할 추가 절차 이름(agent/jobs/module.mjs 가 아는 것만)
|
||||
* dataDirs 절대 건드리지 않는 데이터 경로(검증·백업 대상 안내용)
|
||||
*/
|
||||
export const MODULES = {
|
||||
hal: { name: 'hal', roles: ['control'], layout: 'releases', unit: 'ariweb-hal', unitTemplate: 'ariweb-hal.service.tmpl', user: 'master', port: 8900,
|
||||
health: { url: 'http://127.0.0.1:8900/api/health', expect: 'ok' }, version: { http: 'http://127.0.0.1:8900/api/health', field: 'version' }, dataDirs: ['/srv/ariweb/hal/data'], selfUpdate: true },
|
||||
portal: { name: 'portal', roles: ['control', 'web'], layout: 'releases', unit: 'ariweb-portal', unitTemplate: 'ariweb-portal.service.tmpl', user: 'master', port: 8910,
|
||||
health: { url: 'http://127.0.0.1:8910/api/health', expect: 'ok' }, version: { http: 'http://127.0.0.1:8910/api/health', field: 'version' }, dataDirs: ['/srv/ariweb/portal/data'] },
|
||||
agent: { name: 'agent', roles: ['control', 'web', 'mail', 'backup'], layout: 'releases', unit: 'ariweb-agent', unitTemplate: 'ariweb-agent.service.tmpl', user: 'master',
|
||||
health: { heartbeat: true }, version: { file: 'package.json' }, dataDirs: ['/srv/ariweb/agent/data'], selfUpdate: true },
|
||||
// 도입 모듈: Webmaker(WebHosting_AI_agent 1.7.3). 기존 서버 배치(/srv/webmaker/current, /data/webmaker, 유닛 webmaker, 포트 3080)를 존중한다 — install 은 releases 에 풀고 current 를 바꾸며,
|
||||
// 유닛은 deploy/units/webmaker.service.tmpl 로 다시 쓰되 기존 유닛의 Environment 줄(DATA_ROOT 등)을 facts 로 읽어 유지한다. 원본 deploy/remote-update.sh 의 절차(백업→전송→교체→재시작→확인)를 Agent 가 같은 순서로 수행한다.
|
||||
webmaker: { name: 'webmaker', roles: ['web'], layout: 'releases', unit: 'webmaker', unitTemplate: 'webmaker.service.tmpl', user: 'master', port: 3080, baseDir: '/srv/webmaker',
|
||||
health: { url: 'http://127.0.0.1:3080/api/auth/csrf', expect: 'any' }, version: { file: 'package.json' }, preStop: ['webmaker.backupData'], postStart: ['webmaker.unitCheck'], dataDirs: ['/data/webmaker', '/var/www/webmaker-sites'], extraUnits: ['webmaker-setup'] },
|
||||
// 도입 모듈: AriMail 1.0.11. 서버마다 유닛 이름이 다르다(arihakdang-mail / arimail / heart-mail) — 서버 facts.arimail.unit 이 정한다. 설치는 원본 deploy/deploy.sh 와 같은 절차(테스트→릴리스 복사→DB 백업·마이그레이션→재시작→health)를 Agent 가 수행한다.
|
||||
arimail: { name: 'arimail', roles: ['mail'], layout: 'releases', unit: 'arimail', unitTemplate: null, user: null, port: 8790,
|
||||
health: { url: 'http://127.0.0.1:8790/api/health', expect: 'ok' }, version: { file: 'package.json' }, preStop: ['arimail.backupDb'], postStart: ['arimail.migrate', 'arimail.haraka'], dataDirs: ['<appDir>/data'], copyPaths: ['server', 'web', 'lang', 'migrations', 'package.json', 'haraka'] },
|
||||
backup: { name: 'backup', roles: ['backup', 'web'], layout: 'agent-feature', unit: null, health: { feature: 'backup' }, version: { agent: true }, dataDirs: ['/srv/ariweb/backups'] },
|
||||
observability: { name: 'observability', roles: ['control', 'web', 'mail', 'backup'], layout: 'agent-feature', unit: null, health: { feature: 'metrics' }, version: { agent: true }, dataDirs: [] },
|
||||
};
|
||||
/** 릴리스 artifact 로 배포되는 모듈(agent-feature 는 Agent 안의 기능이라 별도 artifact 가 없다) */
|
||||
export const ARTIFACT_MODULES = Object.values(MODULES).filter((m) => m.layout === 'releases').map((m) => m.name);
|
||||
/** HAL 이 가장 먼저이고, 그다음 agent → portal → 도입 모듈 순으로 배포한다(§32.2) */
|
||||
export const DEPLOY_ORDER = ['hal', 'agent', 'portal', 'webmaker', 'arimail', 'backup', 'observability'];
|
||||
export function descriptor(name) { const d = MODULES[name]; if (!d) throw Object.assign(new Error(`알 수 없는 모듈: ${name}`), { code: 'unknown_module', status: 404 }); return d; }
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* hal/ops.mjs — 운영자 계정·로그인·다중 인증·접근권(§25, §3.2, §3.3).
|
||||
*
|
||||
* - 운영자 계정은 고객 계정과 완전히 분리된 ops_accounts 이며 세션 종류는 'operator'(lib/session). 고객 세션으로는 /ops 를 열 수 없다.
|
||||
* - 네트워크 제한: /ops/* 는 허용 CIDR(config.opsAllowCidrs) 안이거나, 유효한 접근권 쿠키(aw_ops_grant: 사용자·기기(UA 해시) 귀속, 짧은 수명)가 있어야 로그인 화면도 열린다.
|
||||
* 접근권은 (1) 서버에서 CLI `node hal/index.mjs grant --login <id> --hours 8` 로, (2) 허용 네트워크 안에서 로그인한 운영자가 API 로 만든다. 일회용 코드 URL(/ops/access?code=…)을 열면 그 기기에 쿠키가 묶인다.
|
||||
* - 반복 실패 잠금(5회 → 15분), 비밀번호 변경 시 세션 세대 증가 대신 다른 세션 종료, TOTP(선택) 등록·확인·해제, 복구 코드 1회 표시.
|
||||
* - 고위험 작업은 recentlyAuthed(10분) 를 요구한다. 보안 수준 표시: 'password' | 'mfa'.
|
||||
*/
|
||||
import { newId, randomToken, sha256, hashPassword, verifyPassword, safeEqual } from '../lib/crypto.mjs';
|
||||
import { nowIso, toJson, parseJson } from '../lib/db.mjs';
|
||||
import { audit } from '../lib/audit.mjs';
|
||||
import { JobError } from '../lib/jobs.mjs';
|
||||
import * as session from '../lib/session.mjs';
|
||||
import { cookie } from '../lib/http.mjs';
|
||||
import { ipAllowed } from './cidr.mjs';
|
||||
import { newSecret, verifyTotp, otpauthUrl } from './totp.mjs';
|
||||
|
||||
export const GRANT_COOKIE = 'aw_ops_grant';
|
||||
const MAX_FAILS = 5, LOCK_MIN = 15;
|
||||
const uaHash = (ua) => sha256(String(ua || '').slice(0, 300)).slice(0, 32);
|
||||
|
||||
export function createAccount(d, { login, password, displayName, by = null, mustChange = false }) {
|
||||
login = String(login || '').trim().toLowerCase(); if (!/^[a-z0-9][a-z0-9._-]{2,31}$/.test(login)) throw new JobError('bad_login', '아이디는 영문 소문자·숫자로 시작하는 3~32자여야 합니다');
|
||||
if (String(password || '').length < 10) throw new JobError('weak_password', '운영자 비밀번호는 10자 이상이어야 합니다');
|
||||
if (d.store.get('SELECT id FROM ops_accounts WHERE login=?', login)) throw new JobError('login_taken', '이미 있는 아이디입니다');
|
||||
const { salt, hash } = hashPassword(password); const id = newId('op_'); const now = nowIso();
|
||||
d.store.run('INSERT INTO ops_accounts(id, login, display_name, pass_hash, pass_salt, created_at, updated_at, must_change_password) VALUES(?,?,?,?,?,?,?,?)', id, login, String(displayName || login).slice(0, 80), hash, salt, now, now, mustChange ? 1 : 0);
|
||||
audit(d.store, { actorType: by ? 'operator' : 'system', actorId: by, action: 'ops.account_create', subjectType: 'ops_account', subjectId: id, detail: { login } });
|
||||
return publicAccount(d.store.get('SELECT * FROM ops_accounts WHERE id=?', id));
|
||||
}
|
||||
export function publicAccount(a) { if (!a) return null; const mfa = parseJson(a.mfa_json, {}) || {}; return { id: a.id, login: a.login, displayName: a.display_name, status: a.status, lastLoginAt: a.last_login_at, mfaEnabled: !!mfa.enabledAt, securityLevel: mfa.enabledAt ? 'mfa' : 'password', securityHint: mfa.enabledAt ? '다중 인증 사용 중' : '다중 인증을 켜는 것을 권장합니다(§3.3)', mustChangePassword: !!a.must_change_password, lockedUntil: a.lock_until && Date.parse(a.lock_until) > Date.now() ? a.lock_until : null }; }
|
||||
/** 1단계 로그인(비밀번호). MFA 가 켜져 있으면 세션을 만들지 않고 mfaRequired 와 임시 토큰을 돌려준다. */
|
||||
export function login(d, { login, password, totp, ip, ua }) {
|
||||
const a = d.store.get('SELECT * FROM ops_accounts WHERE login=?', String(login || '').trim().toLowerCase());
|
||||
const fail = () => { if (a) { const n = a.failed_logins + 1; d.store.run('UPDATE ops_accounts SET failed_logins=?, lock_until=? WHERE id=?', n, n >= MAX_FAILS ? new Date(Date.now() + LOCK_MIN * 60000).toISOString() : null, a.id); } audit(d.store, { actorType: 'operator', action: 'ops.login', outcome: 'failed', ip, detail: { login } }); throw new JobError('bad_credentials', '아이디 또는 비밀번호가 올바르지 않습니다'); };
|
||||
if (!a || a.status !== 'active') fail();
|
||||
if (a.lock_until && Date.parse(a.lock_until) > Date.now()) throw Object.assign(new JobError('locked', `로그인 시도가 반복 실패해 ${LOCK_MIN}분 동안 잠겼습니다. 해제 시각: ${a.lock_until}`), { detail: { unlockAt: a.lock_until } });
|
||||
if (!verifyPassword(password, a.pass_salt, a.pass_hash)) fail();
|
||||
const mfa = parseJson(a.mfa_json, {}) || {};
|
||||
if (mfa.enabledAt) {
|
||||
const secret = d.vault.get(`ops-totp:${a.id}`); const codes = mfa.recoveryHashes || [];
|
||||
const codeOk = totp && (verifyTotp(secret, totp) || codes.includes(sha256(String(totp).replace(/\s|-/g, ''))));
|
||||
if (!totp) return { mfaRequired: true, message: '인증 앱의 6자리 코드를 입력하세요' };
|
||||
if (!codeOk) fail();
|
||||
if (!verifyTotp(secret, totp)) { mfa.recoveryHashes = codes.filter((h) => h !== sha256(String(totp).replace(/\s|-/g, ''))); d.store.run('UPDATE ops_accounts SET mfa_json=? WHERE id=?', toJson(mfa), a.id); } // 복구 코드는 1회용
|
||||
}
|
||||
d.store.run('UPDATE ops_accounts SET failed_logins=0, lock_until=NULL, last_login_at=? WHERE id=?', nowIso(), a.id);
|
||||
const s = session.createSession(d.store, { userId: a.id, kind: 'operator', ip, ua, days: 1 });
|
||||
audit(d.store, { actorType: 'operator', actorId: a.id, action: 'ops.login', ip });
|
||||
return { session: s, account: publicAccount(a) };
|
||||
}
|
||||
export function reauth(d, accountId, password, sessionId) { const a = d.store.get('SELECT * FROM ops_accounts WHERE id=?', accountId); if (!a || !verifyPassword(password, a.pass_salt, a.pass_hash)) throw new JobError('bad_credentials', '비밀번호가 올바르지 않습니다'); session.markReauth(d.store, sessionId); audit(d.store, { actorType: 'operator', actorId: accountId, action: 'ops.reauth' }); return true; }
|
||||
export function changePassword(d, accountId, current, next, keepSessionId) { const a = d.store.get('SELECT * FROM ops_accounts WHERE id=?', accountId); if (!verifyPassword(current, a.pass_salt, a.pass_hash)) throw new JobError('bad_credentials', '현재 비밀번호가 올바르지 않습니다'); if (String(next).length < 10) throw new JobError('weak_password', '10자 이상이어야 합니다'); const { salt, hash } = hashPassword(next); d.store.run('UPDATE ops_accounts SET pass_hash=?, pass_salt=?, must_change_password=0, updated_at=? WHERE id=?', hash, salt, nowIso(), accountId); session.revokeOtherSessions(d.store, accountId, keepSessionId, 'operator'); audit(d.store, { actorType: 'operator', actorId: accountId, action: 'ops.password_change' }); return true; }
|
||||
/** TOTP 등록 시작: 비밀을 보관소에 임시로 두고 otpauth URL 을 돌려준다(확인 전까지 enabledAt 없음) */
|
||||
export function mfaBegin(d, accountId) { const a = d.store.get('SELECT * FROM ops_accounts WHERE id=?', accountId); const secret = newSecret(); d.vault.set(`ops-totp:${accountId}`, secret, { kind: 'totp' }); const mfa = { ...(parseJson(a.mfa_json, {}) || {}), pendingAt: nowIso(), enabledAt: null }; d.store.run('UPDATE ops_accounts SET mfa_json=? WHERE id=?', toJson(mfa), accountId); return { otpauth: otpauthUrl(secret, a.login), secret }; }
|
||||
/** 코드 확인 → 켜기. 복구 코드 8개를 한 번만 돌려준다. */
|
||||
export function mfaConfirm(d, accountId, code) { const secret = d.vault.get(`ops-totp:${accountId}`); if (!secret || !verifyTotp(secret, code)) throw new JobError('bad_totp', '인증 코드가 맞지 않습니다'); const codes = Array.from({ length: 8 }, () => randomToken(6).replace(/[-_]/g, 'x').slice(0, 10)); const mfa = { enabledAt: nowIso(), recoveryHashes: codes.map((c) => sha256(c)) }; d.store.run('UPDATE ops_accounts SET mfa_json=? WHERE id=?', toJson(mfa), accountId); audit(d.store, { actorType: 'operator', actorId: accountId, action: 'ops.mfa_enable' }); return { recoveryCodes: codes }; }
|
||||
export function mfaDisable(d, accountId, code) { const secret = d.vault.get(`ops-totp:${accountId}`); if (!secret || !verifyTotp(secret, code)) throw new JobError('bad_totp', '인증 코드가 맞지 않습니다'); d.vault.remove(`ops-totp:${accountId}`); d.store.run('UPDATE ops_accounts SET mfa_json=NULL WHERE id=?', accountId); audit(d.store, { actorType: 'operator', actorId: accountId, action: 'ops.mfa_disable' }); return true; }
|
||||
/** 접근권 발급: 일회용 코드(URL)로 기기에 묶는다 */
|
||||
export function issueGrant(d, { accountId, hours = 8, by = null, reason = '' }) { const a = d.store.get('SELECT id FROM ops_accounts WHERE id=? OR login=?', accountId, accountId); if (!a) throw new JobError('account_not_found', '운영자 계정이 없습니다'); const code = randomToken(24); const id = newId('gr_'); d.store.run('INSERT INTO ops_access_grants(id, account_id, code_hash, created_by, created_at, expires_at, reason) VALUES(?,?,?,?,?,?,?)', id, a.id, sha256(code), by, nowIso(), new Date(Date.now() + Math.min(72, Math.max(1, hours)) * 3600000).toISOString(), reason); audit(d.store, { actorType: by ? 'operator' : 'system', actorId: by, action: 'ops.grant_issue', subjectType: 'ops_account', subjectId: a.id, detail: { hours, reason } }); return { id, url: `${d.config.publicUrl}/ops/access?code=${code}`, expiresAt: new Date(Date.now() + hours * 3600000).toISOString() }; }
|
||||
/** 코드로 기기 묶기 → 쿠키 값(원문). 코드는 1회용. */
|
||||
export function bindGrant(d, code, { ua, ip }) { const g = d.store.get('SELECT * FROM ops_access_grants WHERE code_hash=? AND revoked_at IS NULL', sha256(String(code || ''))); if (!g || g.bound_at || Date.parse(g.expires_at) < Date.now()) throw new JobError('bad_grant', '접근 코드가 올바르지 않거나 만료·사용되었습니다'); const cookieVal = randomToken(32); d.store.run('UPDATE ops_access_grants SET bound_at=?, cookie_hash=?, ua_hash=?, ip=? WHERE id=?', nowIso(), sha256(cookieVal), uaHash(ua), ip, g.id); return { cookie: cookieVal, expiresAt: g.expires_at }; }
|
||||
export function grantValid(d, cookieVal, ua) { if (!cookieVal) return null; const g = d.store.get('SELECT * FROM ops_access_grants WHERE cookie_hash=? AND revoked_at IS NULL', sha256(cookieVal)); if (!g || Date.parse(g.expires_at) < Date.now()) return null; if (!safeEqual(g.ua_hash, uaHash(ua))) return null; return g; }
|
||||
export function revokeGrant(d, id, by) { d.store.run('UPDATE ops_access_grants SET revoked_at=? WHERE id=?', nowIso(), id); audit(d.store, { actorType: 'operator', actorId: by, action: 'ops.grant_revoke', subjectType: 'grant', subjectId: id }); }
|
||||
/** 네트워크 관문(§25): 허용 CIDR 또는 접근권 쿠키. 아니면 '접근 불가' — 로그인 화면도 주지 않는다. */
|
||||
export function networkGate(d, ctx) { if (ipAllowed(ctx.ip, d.config.opsAllowCidrs)) return { ok: true, via: 'network' }; const g = grantValid(d, ctx.cookies[GRANT_COOKIE], ctx.headers['user-agent']); return g ? { ok: true, via: 'grant', grant: g } : { ok: false, reason: 'network_not_allowed' }; }
|
||||
export const grantCookie = (val, secure) => cookie(GRANT_COOKIE, val, { maxAge: 72 * 3600, path: '/ops', secure });
|
||||
export { ipAllowed };
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* hal/schema.mjs — HAL DB 마이그레이션(hal.sqlite).
|
||||
* lib 공통 표(sessions·jobs·notifications·audit) 뒤에 HAL 고유 표를 둔다. docs/ARCHITECTURE.md §5·§10, Dev-Order §28·§32·§33·§34.
|
||||
*/
|
||||
import * as session from '../lib/session.mjs';
|
||||
import * as jobs from '../lib/jobs.mjs';
|
||||
import * as notify from '../lib/notify.mjs';
|
||||
import * as audit from '../lib/audit.mjs';
|
||||
|
||||
export const MIGRATIONS = [...session.MIGRATIONS, ...jobs.MIGRATIONS, ...notify.MIGRATIONS, ...audit.MIGRATIONS, { id: 'hal_0001', sql: `
|
||||
-- 관리 서버(§28). agent_id 는 그 서버에서 돌아가는 Node Agent. target/actual 모듈 버전은 module_targets 가 가진다.
|
||||
CREATE TABLE IF NOT EXISTS servers (
|
||||
id TEXT PRIMARY KEY, name TEXT NOT NULL, role TEXT NOT NULL, location TEXT, status TEXT NOT NULL DEFAULT 'pending', accepts_new INTEGER NOT NULL DEFAULT 0,
|
||||
approved_at TEXT, approved_by TEXT, agent_id TEXT, last_seen_at TEXT, clock_skew_ms INTEGER, agent_version TEXT, metrics_json TEXT, capacity_json TEXT, facts_json TEXT,
|
||||
maintenance_note TEXT, isolated_reason TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL
|
||||
);
|
||||
-- Node Agent(관리형) 또는 External Pull Agent(고객 외부 서버, 사이트 하나에 귀속). 비밀은 보관소('service:agent:<id>')에, 여기엔 해시만.
|
||||
CREATE TABLE IF NOT EXISTS agents (
|
||||
id TEXT PRIMARY KEY, server_id TEXT, kind TEXT NOT NULL DEFAULT 'managed', secret_hash TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'active', label TEXT,
|
||||
group_id TEXT, site_id TEXT, version TEXT, registered_at TEXT NOT NULL, last_seen_at TEXT, revoked_at TEXT, facts_json TEXT
|
||||
);
|
||||
-- 일회용 등록 토큰(10분, 1회). 외부 Agent 토큰은 group_id/site_id 가 있다.
|
||||
CREATE TABLE IF NOT EXISTS registration_tokens (
|
||||
id TEXT PRIMARY KEY, token_hash TEXT NOT NULL UNIQUE, kind TEXT NOT NULL DEFAULT 'managed', server_id TEXT, group_id TEXT, site_id TEXT, label TEXT,
|
||||
created_by TEXT, created_at TEXT NOT NULL, expires_at TEXT NOT NULL, used_at TEXT, used_by_agent TEXT
|
||||
);
|
||||
-- 릴리스 등록부(§32.1). artifacts_json = { 모듈: { file, sha256, size } }
|
||||
CREATE TABLE IF NOT EXISTS releases (
|
||||
version TEXT PRIMARY KEY, main_commit TEXT NOT NULL, built_at TEXT NOT NULL, artifacts_json TEXT NOT NULL, checks_json TEXT, status TEXT NOT NULL DEFAULT 'registered',
|
||||
approved_by TEXT, approved_at TEXT, hal_deployed_at TEXT, notes TEXT, registered_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS modules (name TEXT PRIMARY KEY, target_version TEXT, updated_at TEXT NOT NULL);
|
||||
-- 모듈 × 서버 배치 상태(목표 버전·실제 버전·건강·마지막 확인)
|
||||
CREATE TABLE IF NOT EXISTS module_targets (
|
||||
module TEXT NOT NULL, server_id TEXT NOT NULL, target_version TEXT, actual_version TEXT, health TEXT NOT NULL DEFAULT 'unknown', active INTEGER, last_check_at TEXT, last_error TEXT,
|
||||
last_job_id TEXT, updated_at TEXT NOT NULL, PRIMARY KEY(module, server_id)
|
||||
);
|
||||
-- Agent 에게 배정된 작업(HAL 작업의 자식). 상태: queued → running → done|failed|needs_user|cancelled
|
||||
CREATE TABLE IF NOT EXISTS agent_jobs (
|
||||
id TEXT PRIMARY KEY, hal_job_id TEXT, agent_id TEXT NOT NULL, kind TEXT NOT NULL, payload_json TEXT, status TEXT NOT NULL DEFAULT 'queued', progress INTEGER NOT NULL DEFAULT 0,
|
||||
steps_json TEXT, log_json TEXT, result_json TEXT, error TEXT, error_code TEXT, user_message TEXT, cancel_requested INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL, assigned_at TEXT, started_at TEXT, ended_at TEXT, heartbeat_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS agent_jobs_agent ON agent_jobs(agent_id, status, created_at);
|
||||
CREATE INDEX IF NOT EXISTS agent_jobs_hal ON agent_jobs(hal_job_id);
|
||||
-- 서버 지표(90일 이상 보존, §28.1)
|
||||
CREATE TABLE IF NOT EXISTS metrics_samples (
|
||||
server_id TEXT NOT NULL, at TEXT NOT NULL, cpu REAL, mem_used REAL, mem_total REAL, disk_used REAL, disk_total REAL, inode_used REAL, inode_total REAL, io_read REAL, io_write REAL, net_in REAL, net_out REAL, clock_skew_ms INTEGER, load1 REAL,
|
||||
PRIMARY KEY(server_id, at)
|
||||
);
|
||||
-- 사건(§31): 같은 원인은 key 로 묶는다
|
||||
CREATE TABLE IF NOT EXISTS incidents (
|
||||
id TEXT PRIMARY KEY, key TEXT NOT NULL, severity TEXT NOT NULL, title TEXT NOT NULL, component TEXT, customer_impact TEXT, started_at TEXT NOT NULL, resolved_at TEXT, verified_at TEXT,
|
||||
assignee TEXT, public INTEGER NOT NULL DEFAULT 0, public_note TEXT, notes_json TEXT, updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS incidents_key ON incidents(key, resolved_at);
|
||||
-- 운영자 계정(§25). 고객 계정과 완전히 분리. mfa_json = { totp: { secret(보관소 참조 id), enabledAt } }
|
||||
CREATE TABLE IF NOT EXISTS ops_accounts (
|
||||
id TEXT PRIMARY KEY, login TEXT NOT NULL UNIQUE, display_name TEXT NOT NULL, pass_hash TEXT NOT NULL, pass_salt TEXT NOT NULL, mfa_json TEXT, status TEXT NOT NULL DEFAULT 'active',
|
||||
failed_logins INTEGER NOT NULL DEFAULT 0, lock_until TEXT, last_login_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, must_change_password INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
-- 짧은 수명의 사용자·기기 귀속 접근권(§25: 허용 네트워크 밖에서는 이것 없이 로그인 화면도 열리지 않음)
|
||||
CREATE TABLE IF NOT EXISTS ops_access_grants (
|
||||
id TEXT PRIMARY KEY, account_id TEXT NOT NULL, code_hash TEXT UNIQUE, cookie_hash TEXT UNIQUE, ua_hash TEXT, ip TEXT, created_by TEXT, created_at TEXT NOT NULL, expires_at TEXT NOT NULL, bound_at TEXT, revoked_at TEXT, reason TEXT
|
||||
);
|
||||
-- 긴급 복구 세션(§33)
|
||||
CREATE TABLE IF NOT EXISTS emergency_sessions (
|
||||
id TEXT PRIMARY KEY, incident_key TEXT NOT NULL, reason TEXT NOT NULL, scope_json TEXT NOT NULL, status TEXT NOT NULL, declared_by TEXT NOT NULL, declared_at TEXT NOT NULL,
|
||||
approvals_json TEXT NOT NULL DEFAULT '[]', credential_hash TEXT, credential_issued_at TEXT, credential_expires_at TEXT, credential_revoked_at TEXT,
|
||||
actions_json TEXT NOT NULL DEFAULT '[]', verify_json TEXT, reconcile_json TEXT, closed_at TEXT, closed_by TEXT, hal_healthy_at_declare INTEGER, updated_at TEXT NOT NULL
|
||||
);
|
||||
-- 재해복구(§34)
|
||||
CREATE TABLE IF NOT EXISTS dr_config (id INTEGER PRIMARY KEY CHECK(id=1), rto_minutes INTEGER, rpo_minutes INTEGER, snapshot_interval_minutes INTEGER, retention_count INTEGER, external_store TEXT, updated_at TEXT, updated_by TEXT);
|
||||
CREATE TABLE IF NOT EXISTS dr_snapshots (id TEXT PRIMARY KEY, created_at TEXT NOT NULL, location TEXT NOT NULL, size_bytes INTEGER, sha256 TEXT, verify_state TEXT NOT NULL DEFAULT 'pending', verified_at TEXT, external_copied_at TEXT, external_location TEXT, kind TEXT NOT NULL DEFAULT 'scheduled', error TEXT, deleted_at TEXT);
|
||||
CREATE TABLE IF NOT EXISTS dr_restores (id TEXT PRIMARY KEY, snapshot_id TEXT NOT NULL, requested_by TEXT NOT NULL, approved_by TEXT, approved_at TEXT, old_control_stopped_confirmed_at TEXT, started_at TEXT, finished_at TEXT, status TEXT NOT NULL, checks_json TEXT, diff_json TEXT, note TEXT);
|
||||
CREATE TABLE IF NOT EXISTS dr_drills (id TEXT PRIMARY KEY, started_at TEXT NOT NULL, finished_at TEXT, snapshot_id TEXT, performed_by TEXT, steps_json TEXT NOT NULL, measured_rto_minutes REAL, measured_rpo_minutes REAL, result TEXT NOT NULL DEFAULT 'running', findings_json TEXT, next_due_at TEXT);
|
||||
-- 부트스트랩·실행 상태(§32.2). 행 하나(id=1).
|
||||
CREATE TABLE IF NOT EXISTS bootstrap_state (
|
||||
id INTEGER PRIMARY KEY CHECK(id=1), first_install INTEGER NOT NULL DEFAULT 1, release_version TEXT, main_commit TEXT, artifact_sha256 TEXT, hal_running_version TEXT, hal_started_at TEXT,
|
||||
first_agent_id TEXT, first_agent_registered_at TEXT, manual_install_locked INTEGER NOT NULL DEFAULT 0, live_instance_id TEXT, live_lease_at TEXT, completed_at TEXT, updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS transfers (id TEXT PRIMARY KEY, created_by_agent TEXT, path TEXT NOT NULL, size_bytes INTEGER, sha256 TEXT, created_at TEXT NOT NULL, expires_at TEXT NOT NULL, purpose TEXT, meta_json TEXT, deleted_at TEXT);
|
||||
CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value_json TEXT, updated_at TEXT NOT NULL, updated_by TEXT);
|
||||
CREATE TABLE IF NOT EXISTS portal_events_outbox (id TEXT PRIMARY KEY, payload_json TEXT NOT NULL, created_at TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, next_at TEXT NOT NULL, delivered_at TEXT, last_error TEXT);
|
||||
` }];
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* hal/totp.mjs — RFC 6238 TOTP(운영자 다중 인증, §3.3·§25). 외부 의존성 없이 node:crypto 만 쓴다.
|
||||
* 비밀은 base32 로 표시(인증 앱 등록용 otpauth URL)하고 저장은 보관소에만 한다.
|
||||
*/
|
||||
import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto';
|
||||
|
||||
const B32 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567';
|
||||
export function base32Encode(buf) { let bits = 0, value = 0, out = ''; for (const b of buf) { value = (value << 8) | b; bits += 8; while (bits >= 5) { out += B32[(value >>> (bits - 5)) & 31]; bits -= 5; } } if (bits > 0) out += B32[(value << (5 - bits)) & 31]; return out; }
|
||||
export function base32Decode(str) { const s = String(str).toUpperCase().replace(/=+$/, '').replace(/[^A-Z2-7]/g, ''); let bits = 0, value = 0; const out = []; for (const c of s) { value = (value << 5) | B32.indexOf(c); bits += 5; if (bits >= 8) { out.push((value >>> (bits - 8)) & 255); bits -= 8; } } return Buffer.from(out); }
|
||||
/** 새 비밀(base32 32자) */
|
||||
export const newSecret = () => base32Encode(randomBytes(20));
|
||||
/** 코드 계산. step 30초, 6자리 */
|
||||
export function totpCode(secretB32, at = Date.now(), step = 30) {
|
||||
const counter = Math.floor(at / 1000 / step); const msg = Buffer.alloc(8); msg.writeBigUInt64BE(BigInt(counter));
|
||||
const h = createHmac('sha1', base32Decode(secretB32)).update(msg).digest(); const off = h[h.length - 1] & 0xf;
|
||||
const code = ((h[off] & 0x7f) << 24 | (h[off + 1] & 0xff) << 16 | (h[off + 2] & 0xff) << 8 | (h[off + 3] & 0xff)) % 1000000;
|
||||
return String(code).padStart(6, '0');
|
||||
}
|
||||
/** 검증(앞뒤 1단계 허용, 상수 시간 비교) */
|
||||
export function verifyTotp(secretB32, code, at = Date.now()) {
|
||||
const c = String(code || '').replace(/\s/g, ''); if (!/^\d{6}$/.test(c)) return false;
|
||||
for (const d of [-1, 0, 1]) { const exp = Buffer.from(totpCode(secretB32, at + d * 30000)); const got = Buffer.from(c); if (exp.length === got.length && timingSafeEqual(exp, got)) return true; }
|
||||
return false;
|
||||
}
|
||||
/** 인증 앱 등록 URL */
|
||||
export const otpauthUrl = (secretB32, login, issuer = 'Ariweb HAL') => `otpauth://totp/${encodeURIComponent(issuer)}:${encodeURIComponent(login)}?secret=${secretB32}&issuer=${encodeURIComponent(issuer)}&digits=6&period=30`;
|
||||
Reference in New Issue
Block a user