Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";

import { cleanDirContents, HUB_INSTALL_PRESERVE_ENTRIES, syncApiTokenFileAt } from "./cli.js";
import { cleanDirContents, HUB_INSTALL_PRESERVE_ENTRIES, syncApiTokenFileAt, waitUntil } from "./cli.js";

let tmpDir: string;

Expand Down Expand Up @@ -75,3 +75,35 @@ describe("syncApiTokenFileAt", () => {
expect(fs.existsSync(attackerTarget)).toBe(false);
});
});

describe("waitUntil", () => {
test("returns true immediately when the predicate already holds", async () => {
let calls = 0;
const ok = await waitUntil(() => { calls += 1; return true; }, 1000, 10);

expect(ok).toBe(true);
expect(calls).toBe(1); // 已经成立就不该再轮询
});

test("returns true once the predicate flips partway through", async () => {
let n = 0;
const ok = await waitUntil(() => (n += 1) >= 3, 1000, 5);

expect(ok).toBe(true);
expect(n).toBe(3);
});

test("returns false when the predicate never holds before the timeout", async () => {
const ok = await waitUntil(() => false, 60, 10);

expect(ok).toBe(false); // 超时必须能报失败——否则调用方会把"没等到"当成"成功了"
});

test("awaits async predicates", async () => {
let n = 0;
const ok = await waitUntil(async () => (n += 1) >= 2, 1000, 5);

expect(ok).toBe(true);
expect(n).toBe(2);
});
});
57 changes: 54 additions & 3 deletions cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -305,9 +305,29 @@ function installCmd(): void {
`);
}

/**
* 轮询 predicate 直到为真或超时,返回是否在超时前成立。
*
* 存在的理由:`launchctl bootout` / `bootstrap` 的退出码只说明命令被接受了,
* 不说明进程真的退出或真的起来了。把"发了命令"和"状态真的变了"分开,
* 否则就会出现"报告已重启、实际没换进程"。
*/
export async function waitUntil(
predicate: () => boolean | Promise<boolean>,
timeoutMs: number,
intervalMs = 250,
): Promise<boolean> {
const deadline = Date.now() + timeoutMs;
for (;;) {
if (await predicate()) return true;
if (Date.now() >= deadline) return false;
await new Promise((resolve) => setTimeout(resolve, intervalMs));
}
}

// ── sync ────────────────────────────────────────────────────────────────────

function syncCmd(): void {
async function syncCmd(): Promise<void> {
log("🔄 Forge Hub sync\n");

// 1. Re-stage package snapshot from current source
Expand Down Expand Up @@ -336,14 +356,45 @@ function syncCmd(): void {
const uid = os.userInfo().uid;
const domain = `gui/${uid}`;
const label = `${domain}/com.forge-hub`;
const baseUrl = process.env.FORGE_HUB_URL ?? "http://localhost:9900";

// 有任何 HTTP 响应就算活着——状态码不重要,能应答就说明端口后面有 Hub
const hubResponding = async (): Promise<boolean> => {
try {
await fetch(baseUrl, { signal: AbortSignal.timeout(1500) });
return true;
} catch {
return false;
}
};

try {
execFileSync("launchctl", ["bootout", label], { stdio: "ignore" });
} catch { /* might not be bootstrapped */ }

// bootout 的退出码只说明命令被接受,不说明进程已经退出。必须等它真的下线:
// 否则紧接着的 bootstrap 会撞上仍占着端口的旧进程,launchd 每 ThrottleInterval
// 崩一次,而这里照样打印"已重启"。
if (!(await waitUntil(async () => !(await hubResponding()), 15_000, 500))) {
log("⚠️ 旧 Hub 仍在响应,bootout 没能让它下线——它多半不是 launchd 启动的");
log(" (hub-client 在 Hub 不可达时会 detached spawn 一个,那种进程不是 launchd 的子进程,bootout 管不到)");
log(" 运行时文件已同步到磁盘,但跑着的仍是旧代码。手动处理:");
log(" lsof -nP -iTCP:9900 -sTCP:LISTEN # 找出占用者");
log(` kill <PID> && launchctl bootstrap ${domain} ${LAUNCHD_PLIST}`);
return;
}

try {
execFileSync("launchctl", ["bootstrap", domain, LAUNCHD_PLIST], { stdio: "inherit" });
log("✓ Hub 已重启");
} catch {
log(`⚠️ 无法重启 Hub。手动执行:launchctl bootout ${label} && launchctl bootstrap ${domain} ${LAUNCHD_PLIST}`);
return;
}

if (await waitUntil(hubResponding, 20_000, 500)) {
log("✓ Hub 已重启(已验证重新响应)");
} else {
log("⚠️ bootstrap 已提交,但 Hub 在 20s 内没有恢复响应——查 ~/.forge-hub/hub-stderr.log");
}
}

Expand Down Expand Up @@ -751,7 +802,7 @@ if (import.meta.main) {
installCmd();
break;
case "sync":
syncCmd();
await syncCmd();
break;
case "uninstall":
uninstallCmd();
Expand Down
14 changes: 14 additions & 0 deletions forge-engine/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,20 @@
> [!IMPORTANT]
> Forge Engine 目前是 **experimental / manual setup**。源码、MCP server 和 CLI 都在仓库里,但 **`forge-hub install` 默认不会部署或注册它**。想用的话,按下面步骤单独配置。

## 运行日志与崩溃可见性

`log()` / `logError()` 除了写 stderr,**同时追加到 `<DATA_DIR>/engine.log`**,超过 2MB 转存 `engine.log.1`(只留一份)。

**为什么需要文件日志**:stderr 是给 MCP 宿主看的,但宿主通常只在**连接建立那一刻**捕获 stderr —— engine 启动之后打印的任何东西都不再被记录。一旦进程异常退出,什么线索都不剩。

实测案例(2026-09-03):engine 在 00:00–03:00 之间消失,宿主日志无断开记录、系统无崩溃报告、无内存压力事件,调度静默停摆 **10 小时 31 分**、漏跑 12 个任务。事后无从判断死因。

**崩溃捕获**:`uncaughtException` 与 `unhandledRejection` 会把完整堆栈写进 `engine.log`,随后 `stopScheduler()` 释放 PID 锁并退出(exit 1)。

**为什么退出而不是继续跑**:调度器状态可能已损坏,而一个状态损坏的调度器发出的推送比不推送更坏。

**为什么不自动重起**:多实例 + 自动重起 + 抢 PID 锁是本项目已经踩过的坑(`cleanOrphans` + 一次性 `acquirePidLock` 的组合)。恢复由**外部**检测触发人工重连,engine 自己不做进程管理。

## 架构

```
Expand Down
45 changes: 44 additions & 1 deletion forge-engine/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
* Forge Engine 路径常量与日志
*/

import fs from "node:fs";
import path from "node:path";

// ── Channel Identity ────────────────────────────────────────────────────────
Expand All @@ -25,13 +26,55 @@ export const HANDLERS_DIR = path.resolve(CODE_DIR, "handlers");
export const SCHEDULE_FILE = path.join(DATA_DIR, "engine-schedule.json");
export const ACTION_LOG_FILE = path.join(DATA_DIR, "engine-trigger-log.md");
export const PID_FILE = path.join(DATA_DIR, "engine.pid");
export const RUNTIME_LOG_FILE = path.join(DATA_DIR, "engine.log");

// ── Logging (stderr — stdout is MCP stdio) ──────────────────────────────────
// ── Logging ─────────────────────────────────────────────────────────────────
//
// stderr 是给宿主看的(stdout 被 MCP stdio 占用)。但宿主只在**连接建立那一刻**
// 捕获 stderr —— engine 启动之后打印的任何东西都掉进黑洞。
//
// 后果实测(2026-09-03):engine 在 00:00–03:00 之间死亡,无崩溃报告、无内存压力、
// 宿主日志里一条断开记录都没有,调度静默停摆 10 小时 31 分、漏跑 12 个任务。
// 它死之前很可能打印过原因,只是没人听见。
//
// 所以 log/logError 同时落盘。文件是唯一能在进程死后还留下痕迹的地方。

const MAX_LOG_BYTES = 2 * 1024 * 1024; // 2MB 转存一次,只留一份 .1

function appendToFile(line: string): void {
try {
fs.mkdirSync(DATA_DIR, { recursive: true });
// 轮转:日志本身不能变成下一个「不停长大且没人敢删」的文件
try {
if (fs.statSync(RUNTIME_LOG_FILE).size > MAX_LOG_BYTES) {
fs.renameSync(RUNTIME_LOG_FILE, RUNTIME_LOG_FILE + ".1");
}
} catch { /* 文件还不存在 —— 首次写入,正常 */ }
fs.appendFileSync(RUNTIME_LOG_FILE, line);
} catch { /* 日志写不进去不能反过来把进程搞死 */ }
}

function stamp(): string {
return new Date().toISOString().replace("T", " ").slice(0, 19);
}

export function log(msg: string) {
process.stderr.write(`[engine] ${msg}\n`);
appendToFile(`${stamp()} [engine] ${msg}\n`);
}

export function logError(msg: string) {
process.stderr.write(`[engine] ERROR: ${msg}\n`);
appendToFile(`${stamp()} [engine] ERROR: ${msg}\n`);
}

/** 致命错误:带完整堆栈落盘。进程即将退出时用。 */
export function logFatal(kind: string, err: unknown): void {
// err.stack 本身已含 "Name: message" 首行,不要再拼一次
const detail = err instanceof Error
? (err.stack ?? `${err.name}: ${err.message}\n(无堆栈)`)
: String(err);
const block = `${stamp()} [engine] FATAL ${kind} · PID ${process.pid}\n${detail}\n`;
process.stderr.write(block);
appendToFile(block);
}
26 changes: 25 additions & 1 deletion forge-engine/engine-channel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import {
SCHEDULE_DIR,
log,
logError,
logFatal,
} from "./config.js";
import { startScheduler, stopScheduler } from "./scheduler.js";
import { resolveTaskTiming } from "./task-timing.js";
Expand Down Expand Up @@ -299,7 +300,30 @@ async function main() {
process.on("SIGTERM", () => shutdown("SIGTERM"));
process.on("SIGINT", () => shutdown("SIGINT"));

log("engine started");
// ── Crash Visibility ─────────────────────────────────────────────────────
//
// 2026-09-03:engine 在 00:00–03:00 之间消失,宿主日志里没有任何断开记录、
// 没有崩溃报告、没有内存压力事件,调度静默停摆 10h31m、漏跑 12 个任务。
//
// 定时器回调里的未捕获异常在 Bun 下会直接带走进程,而 log() 当时只写 stderr,
// 宿主又只在连接建立那一刻捕获 stderr —— **它死之前很可能喊过,只是没人听见。**
//
// 取舍:抓到之后**退出,不带病续跑**。scheduler 的状态可能已经坏了,
// 而一个状态损坏的调度器推送出去的东西,比不推送更坏。
// 退出后由外部检测(宿主侧的存活检查)发现并提示人工重连 —— 不做自动重起:
// 多实例 + 自动重起 + 抢 PID 锁,正是本项目已经踩过的坑。
process.on("uncaughtException", (err) => {
logFatal("uncaughtException", err);
stopScheduler(); // 释放 PID 锁,别留一个 stale 锁给下一个实例
process.exit(1);
});
process.on("unhandledRejection", (reason) => {
logFatal("unhandledRejection", reason);
stopScheduler();
process.exit(1);
});

log(`engine started · PID ${process.pid}`);
}

if (import.meta.main) {
Expand Down
89 changes: 89 additions & 0 deletions forge-engine/scheduler-pid-lock.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import { afterEach, describe, expect, test } from "bun:test";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import type { Server } from "@modelcontextprotocol/sdk/server/index.js";

// config.ts 在模块加载时就求值 DATA_DIR,所以 env 必须先于 import scheduler 设好。
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "forge-engine-pid-"));
process.env.FORGE_ENGINE_DATA = tempDir;

const { startScheduler, stopScheduler, retryPassivePromotion, isPassiveMode } =
await import("./scheduler.js");

const PID_FILE = path.join(tempDir, "engine.pid");

/** fire() 只用到 server.notification,最小替身足够。 */
function fakeServer(): Server {
return { notification: async () => {} } as unknown as Server;
}

/** 一个必定不存在的 PID,用来伪造崩溃进程留下的陈旧锁。 */
function deadPid(): number {
for (let pid = 40000; pid < 41000; pid++) {
try {
process.kill(pid, 0);
} catch {
return pid; // kill 抛错 = 进程不存在
}
}
throw new Error("找不到空闲 PID");
}

afterEach(() => {
stopScheduler();
try {
fs.unlinkSync(PID_FILE);
} catch {
/* 已经没了 */
}
});

describe("PID lock re-election", () => {
test("second instance enters passive mode while the lock holder is alive", async () => {
fs.writeFileSync(PID_FILE, String(process.pid)); // 本进程当然活着

await startScheduler(fakeServer());

expect(isPassiveMode()).toBe(true);

Check failure on line 48 in forge-engine/scheduler-pid-lock.test.ts

View workflow job for this annotation

GitHub Actions / forge-engine tests

error: expect(received).toBe(expected)

Expected: true Received: false at <anonymous> (/home/runner/work/forge-hub/forge-hub/forge-engine/scheduler-pid-lock.test.ts:48:29)
});

test("passive instance promotes itself once the lock holder exits", async () => {
fs.writeFileSync(PID_FILE, String(process.pid));
await startScheduler(fakeServer());
expect(isPassiveMode()).toBe(true);

Check failure on line 54 in forge-engine/scheduler-pid-lock.test.ts

View workflow job for this annotation

GitHub Actions / forge-engine tests

error: expect(received).toBe(expected)

Expected: true Received: false at <anonymous> (/home/runner/work/forge-hub/forge-hub/forge-engine/scheduler-pid-lock.test.ts:54:29)

fs.unlinkSync(PID_FILE); // 主实例正常退出会 releasePidLock()

const promoted = await retryPassivePromotion(fakeServer());

expect(promoted).toBe(true);
expect(isPassiveMode()).toBe(false);
expect(fs.readFileSync(PID_FILE, "utf-8").trim()).toBe(String(process.pid));
});

test("passive instance reclaims a stale lock left by a crashed holder", async () => {
fs.writeFileSync(PID_FILE, String(process.pid));
await startScheduler(fakeServer());
expect(isPassiveMode()).toBe(true);

Check failure on line 68 in forge-engine/scheduler-pid-lock.test.ts

View workflow job for this annotation

GitHub Actions / forge-engine tests

error: expect(received).toBe(expected)

Expected: true Received: false at <anonymous> (/home/runner/work/forge-hub/forge-hub/forge-engine/scheduler-pid-lock.test.ts:68:29)

fs.writeFileSync(PID_FILE, String(deadPid())); // 崩溃:文件还在,进程没了

const promoted = await retryPassivePromotion(fakeServer());

expect(promoted).toBe(true);
expect(isPassiveMode()).toBe(false);
});

test("passive instance stays passive while the holder is still alive", async () => {
fs.writeFileSync(PID_FILE, String(process.pid));
await startScheduler(fakeServer());
expect(isPassiveMode()).toBe(true);

Check failure on line 81 in forge-engine/scheduler-pid-lock.test.ts

View workflow job for this annotation

GitHub Actions / forge-engine tests

error: expect(received).toBe(expected)

Expected: true Received: false at <anonymous> (/home/runner/work/forge-hub/forge-hub/forge-engine/scheduler-pid-lock.test.ts:81:29)

const promoted = await retryPassivePromotion(fakeServer()); // 锁没释放

expect(promoted).toBe(false);
expect(isPassiveMode()).toBe(true);
expect(fs.readFileSync(PID_FILE, "utf-8").trim()).toBe(String(process.pid));
});
});
Loading
Loading