Skip to content
68 changes: 49 additions & 19 deletions apps/server/src/assets/AssetAccess.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import { vi } from "vite-plus/test";

import * as AssetSigner from "./AssetSigner.ts";
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
import { toolOutputImageAttachmentId } from "../attachmentStore.ts";
import * as ServerConfig from "../config.ts";
import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts";
Expand Down Expand Up @@ -102,16 +103,32 @@ const oversizedScreenshotItem = {
},
};

const storedScreenshotAttachmentId = toolOutputImageAttachmentId(screenshotPng, ".png");
const foreignAttachmentId = "thread-2-00000000-0000-4000-8000-000000000041-png";
const storedScreenshotItem = {
...screenshotItem,
id: TurnItemId.make("tool-screenshot-stored"),
output: {
content: [{ type: "image", mimeType: "image/png", attachmentId: storedScreenshotAttachmentId }],
},
};
// Rows persisted before the server stored images can carry any id a provider emitted.
const foreignScreenshotItem = {
...screenshotItem,
id: TurnItemId.make("tool-screenshot-foreign"),
output: {
content: [{ type: "image", mimeType: "image/png", attachmentId: foreignAttachmentId }],
},
};

const layerTest = Layer.mergeAll(
NodeHttpPlatform.layer,
Layer.mock(Orchestrator.OrchestratorV2)({
getTurnItem: ({ itemId }) =>
Effect.succeed(
itemId === screenshotItem.id
? screenshotItem
: itemId === oversizedScreenshotItem.id
? oversizedScreenshotItem
: null,
[screenshotItem, oversizedScreenshotItem, storedScreenshotItem, foreignScreenshotItem].find(
(item) => item.id === itemId,
) ?? null,
),
}),
layerConfig,
Expand Down Expand Up @@ -186,31 +203,44 @@ describe("AssetAccess", () => {
);
});

it.effect("serves an image a tool returned inline from the stored item", () =>
it.effect("serves an image a tool returned from the stored item or its attachment file", () =>
Effect.gen(function* () {
const config = yield* ServerConfig.ServerConfig;
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
yield* fs.makeDirectory(config.attachmentsDir, { recursive: true });
for (const attachmentId of [storedScreenshotAttachmentId, foreignAttachmentId]) {
yield* fs.writeFile(path.join(config.attachmentsDir, `${attachmentId}.png`), screenshotPng);
}
const resource = {
_tag: "tool-output-image" as const,
threadId: screenshotItem.threadId,
itemId: screenshotItem.id,
index: 0,
};
const result = yield* issueAssetUrl({ resource });
expect(result.imageDimensions).toEqual({ width: 390, height: 844 });
const suffix = result.relativeUrl.slice(`${ASSET_ROUTE_PREFIX}/`.length);
const separator = suffix.indexOf("/");
expect(yield* resolveAsset(suffix.slice(0, separator), suffix.slice(separator + 1))).toEqual({
kind: "bytes",
mimeType: "image/png",
bytes: Buffer.from(screenshotPng),
});
for (const itemId of [screenshotItem.id, storedScreenshotItem.id]) {
const result = yield* issueAssetUrl({ resource: { ...resource, itemId } });
expect(result.imageDimensions).toEqual({ width: 390, height: 844 });
const suffix = result.relativeUrl.slice(`${ASSET_ROUTE_PREFIX}/`.length);
const separator = suffix.indexOf("/");
expect(
yield* resolveAsset(suffix.slice(0, separator), suffix.slice(separator + 1)),
).toEqual({
kind: "bytes",
mimeType: "image/png",
bytes: Buffer.from(screenshotPng),
});
}
const missing = yield* issueAssetUrl({ resource: { ...resource, index: 1 } }).pipe(
Effect.flip,
);
expect(missing._tag).toBe("AssetWorkspaceAssetNotFoundError");
const oversized = yield* issueAssetUrl({
resource: { ...resource, itemId: oversizedScreenshotItem.id },
}).pipe(Effect.flip);
expect(oversized._tag).toBe("AssetWorkspaceAssetNotFoundError");
for (const itemId of [oversizedScreenshotItem.id, foreignScreenshotItem.id]) {
const unserved = yield* issueAssetUrl({ resource: { ...resource, itemId } }).pipe(
Effect.flip,
);
expect(unserved._tag).toBe("AssetWorkspaceAssetNotFoundError");
}
}).pipe(Effect.provide(layerTest)),
);

Expand Down
21 changes: 20 additions & 1 deletion apps/server/src/assets/AssetAccess.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,12 @@ import * as Stream from "effect/Stream";
import { base64UrlDecodeUtf8, base64UrlEncode } from "../auth/utils.ts";
import * as AssetSigner from "./AssetSigner.ts";
import { makeIsServerInternalPath } from "../auth/serverInternalPaths.ts";
import { parseAttachmentFileExtension, resolveAttachmentPathById } from "../attachmentStore.ts";
import {
parseAttachmentFileExtension,
resolveAttachmentPathById,
toolOutputImageAttachmentId,
} from "../attachmentStore.ts";
import { inferImageExtension } from "../imageMime.ts";
import * as ServerConfig from "../config.ts";
import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion";
Expand Down Expand Up @@ -243,6 +248,20 @@ const readToolOutputImage = Effect.fn("AssetAccess.readToolOutputImage")(functio
const item = yield* orchestrator.getTurnItem(input);
const image =
item?.type === "dynamic_tool" ? toolOutputImages(item.output)[input.index] : undefined;
if (image?.attachmentId !== undefined) {
const config = yield* ServerConfig.ServerConfig;
const filePath = resolveAttachmentPathById({
attachmentsDir: config.attachmentsDir,
attachmentId: image.attachmentId,
});
if (filePath === null) return null;
const fileSystem = yield* FileSystem.FileSystem;
const bytes = yield* fileSystem.readFile(filePath);
const extension = inferImageExtension({ mimeType: image.mimeType });
return toolOutputImageAttachmentId(bytes, extension) === image.attachmentId
? { mimeType: image.mimeType, bytes }
: null;
}
return image?.data === undefined || image.data.length > MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH
? null
: { mimeType: image.mimeType, bytes: Buffer.from(image.data, "base64") };
Expand Down
7 changes: 7 additions & 0 deletions apps/server/src/attachmentStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,13 @@ export function chatAttachmentStableKey(
return `${messageId}:${index}:${digest}`;
}

/** Content-addressed: the asset reader re-derives it from the file's bytes, so an id that
* provider output carried in a stored row can never resolve to a file it did not hash to. */
export function toolOutputImageAttachmentId(bytes: Uint8Array, extension: string): string {
const digest = NodeCrypto.createHash("sha256").update(bytes).digest("hex");
return `${createDeterministicAttachmentId("tool_output", digest)}${attachmentIdExtensionSuffix(extension)}`;
}

export function parseThreadSegmentFromAttachmentId(attachmentId: string): string | null {
const normalizedId = normalizeAttachmentRelativePath(attachmentId);
if (!normalizedId || normalizedId.includes("/") || normalizedId.includes(".")) {
Expand Down
21 changes: 13 additions & 8 deletions apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ const layerTestEventSink = EventSink.layer.pipe(
);

const layerGit = GitVcsDriver.layer.pipe(
Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "ingestor-git-" })),
Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "ingestor-git-" })),
Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
);

Expand Down Expand Up @@ -1547,7 +1547,7 @@ layer("ProviderEventIngestorV2", (it) => {
}),
);

it.effect("stores tool image bytes only where a tool-output-image asset serves them", () =>
it.effect("keeps tool image bytes out of both SQLite tables, served ones included", () =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const eventSink = yield* EventSink.EventSinkV2;
Expand Down Expand Up @@ -1624,17 +1624,22 @@ layer("ProviderEventIngestorV2", (it) => {
});

assert.equal(storedJson.includes(readBase64), false);
assert.equal(storedJson.includes(screenshotBase64), true);
assert.equal(storedJson.includes(screenshotBase64), false);
assert.deepEqual(projectedRead?.type === "dynamic_tool" ? projectedRead.output : null, {
type: "image",
file: { type: "image/png", originalSize: 30_000, sizeBytes: 30_000 },
});
assert.deepEqual(
toolOutputImages(
projectedScreenshot?.type === "dynamic_tool" ? projectedScreenshot.output : null,
),
[{ mimeType: "image/png", data: screenshotBase64 }],
assert.equal(JSON.stringify(projectedScreenshot).includes(screenshotBase64), false);
const [stored] = toolOutputImages(
projectedScreenshot?.type === "dynamic_tool" ? projectedScreenshot.output : null,
);
assert.equal(stored?.mimeType, "image/png");
const config = yield* ServerConfig.ServerConfig;
const fs = yield* FileSystem.FileSystem;
const storedBytes = yield* fs.readFile(
`${config.attachmentsDir}/${stored?.attachmentId ?? "missing"}.png`,
);
assert.equal(Buffer.from(storedBytes).toString("base64"), screenshotBase64);
}),
);

Expand Down
20 changes: 18 additions & 2 deletions apps/server/src/orchestration-v2/ProviderEventIngestor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ import {
import * as Context from "effect/Context";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Path from "effect/Path";
import * as Schema from "effect/Schema";
import * as SqlError from "effect/sql/SqlError";

Expand All @@ -30,6 +32,7 @@ import {
ProviderTurnAnalytics,
type ProviderTurnAnalyticsContext,
} from "../telemetry/ProviderUsageAnalytics.ts";
import * as ServerConfig from "../config.ts";
import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
import * as EventSink from "./EventSink.ts";
Expand All @@ -38,7 +41,10 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
import { makeProviderFailureTurnItem } from "@t3tools/provider-core/server/failure";
import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
import { stripUnservedToolOutputImageBytes } from "./toolOutputImageBytes.ts";
import {
storeServedToolOutputImages,
stripUnservedToolOutputImageBytes,
} from "./toolOutputImageBytes.ts";
import { boundFileChangeForStorage } from "./WireProjection.ts";

export class ProviderEventNormalizeError extends Schema.TaggedError<ProviderEventNormalizeError>()(
Expand Down Expand Up @@ -243,6 +249,9 @@ export const layer: Layer.Layer<
| ProjectionStore.ProjectionStoreV2
| ThreadCommandExecutor.ThreadCommandExecutor
| ProviderInstanceRegistry.ProviderInstanceRegistry
| ServerConfig.ServerConfig
| FileSystem.FileSystem
| Path.Path
> = Layer.effect(
ProviderEventIngestorV2,
Effect.gen(function* () {
Expand All @@ -254,6 +263,9 @@ export const layer: Layer.Layer<
const analytics = yield* ProviderTurnAnalytics;
const agentScope = yield* AgentScope;
const instances = yield* ProviderInstanceRegistry.ProviderInstanceRegistry;
const imageStoreContext = yield* Effect.context<
ServerConfig.ServerConfig | FileSystem.FileSystem | Path.Path
>();
const scope = yield* Effect.scope;
const completedTurnAnalytics = new Set<string>();

Expand Down Expand Up @@ -574,7 +586,11 @@ export const layer: Layer.Layer<
type: "turn-item.updated",
threadId: input.event.turnItem.threadId,
payload: boundFileChangeForStorage(
stripUnservedToolOutputImageBytes(input.event.turnItem),
stripUnservedToolOutputImageBytes(
yield* storeServedToolOutputImages(input.event.turnItem).pipe(
Effect.provide(imageStoreContext),
),
),
),
runId: input.event.turnItem.runId,
nodeId: input.event.turnItem.nodeId,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import { TestClock } from "effect/testing";
import { HttpServer } from "effect/http";

import { ProviderWorkspaceMissingError } from "../provider/Errors.ts";
import * as ServerConfig from "../config.ts";
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
import * as ProjectService from "../project/ProjectService.ts";
import * as McpProviderSession from "@t3tools/provider-core/server/mcpSession";
Expand Down Expand Up @@ -717,6 +718,7 @@ function layerTest(input: {
Layer.mock(GitVcsDriver.GitVcsDriver)({
resolveBranchName: () => Effect.succeed(null),
}),
ServerConfig.layerTest(process.cwd(), { prefix: "provider-session-manager-" }),
),
),
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -368,8 +368,8 @@ export function layerWithRegistry<Error>(
IdAllocator.layer,
ThreadCommandExecutor.layer,
GitVcsDriver.layer.pipe(
Layer.provide(layerServerConfig),
Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
Layer.provideMerge(layerServerConfig),
Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
),
),
),
Expand Down
83 changes: 82 additions & 1 deletion apps/server/src/orchestration-v2/toolOutputImageBytes.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,20 @@ import {
MAX_TOOL_OUTPUT_IMAGES,
toolOutputImages,
} from "@t3tools/shared/toolOutput";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { it as effectIt } from "@effect/vitest";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as PlatformError from "effect/PlatformError";
import { describe, expect, it } from "vite-plus/test";

import { stripUnservedToolOutputImageBytes } from "./toolOutputImageBytes.ts";
import * as ServerConfig from "../config.ts";
import {
storeServedToolOutputImages,
stripUnservedToolOutputImageBytes,
} from "./toolOutputImageBytes.ts";

const PNG_BYTES = 3_000;
const PNG_BASE64 = Buffer.alloc(PNG_BYTES, 7).toString("base64");
Expand Down Expand Up @@ -213,3 +223,74 @@ describe("stripUnservedToolOutputImageBytes", () => {
expect(stripped.result[0]).toBe(untouched);
});
});

describe("storeServedToolOutputImages", () => {
effectIt.effect("drops attachment ids the provider put in its own image blocks", () =>
Effect.gen(function* () {
const attachmentId = "other-thread-00000000-0000-4000-8000-000000000001-png";
for (const data of [undefined, "A".repeat(MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH + 4)]) {
const item = toolItem({
content: [{ type: "image", mimeType: "image/png", data, attachmentId }],
});
const stored = stripUnservedToolOutputImageBytes(yield* storeServedToolOutputImages(item));
expect(
toolOutputImages(stored.type === "dynamic_tool" ? stored.output : null)[0]?.attachmentId,
).toBeUndefined();
}
}).pipe(
Effect.provide(
ServerConfig.layerTest(process.cwd(), { prefix: "tool-output-images-" }).pipe(
Layer.provideMerge(NodeServices.layer),
),
),
),
);

effectIt.effect("keeps an image inline until a retried write stores all of its bytes", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
let failNextWrite = true;
const faultyFs = {
...fs,
writeFile: (filePath: string, content: Uint8Array) =>
failNextWrite
? fs.writeFile(filePath, content.subarray(0, 10)).pipe(
Effect.andThen(
Effect.fail(
PlatformError.systemError({
_tag: "Unknown",
module: "FileSystem",
method: "writeFile",
description: "ENOSPC",
}),
),
),
)
: fs.writeFile(filePath, content),
} satisfies FileSystem.FileSystem;
const item = toolItem({
content: [
{ type: "image", source: { type: "base64", media_type: "image/png", data: PNG_BASE64 } },
],
});
const store = storeServedToolOutputImages(item).pipe(
Effect.provideService(FileSystem.FileSystem, faultyFs),
);

expect(yield* store).toBe(item);
failNextWrite = false;
const stored = yield* store;
const [image] = toolOutputImages(stored.type === "dynamic_tool" ? stored.output : null);
const config = yield* ServerConfig.ServerConfig;
const bytes = yield* fs.readFile(`${config.attachmentsDir}/${image?.attachmentId}.png`);
expect(Buffer.from(bytes).toString("base64")).toBe(PNG_BASE64);
expect((yield* fs.readDirectory(config.attachmentsDir)).length).toBe(1);
}).pipe(
Effect.provide(
ServerConfig.layerTest(process.cwd(), { prefix: "tool-output-images-" }).pipe(
Layer.provideMerge(NodeServices.layer),
),
),
),
);
});
Loading
Loading