From d921d4b3575785e79edd3376f16114c669a73e0d Mon Sep 17 00:00:00 2001 From: Magolor Date: Wed, 2 Sep 2026 12:37:15 +0800 Subject: [PATCH] fix(storage-json): reject cross-version legacy bootstrap (#3431) * fix(storage-json): reject cross-version legacy bootstrap * test(webworker): sync projection cache fixture version * docs(storage): record legacy bootstrap version ownership --- ...ojection-cache-per-session-files.i18n.yaml | 4 +- ...8-19-projection-cache-per-session-files.md | 6 ++- ...9-projection-cache-per-session-files.zh.md | 6 ++- .../home/storages/session_projcache.json | 2 +- .../tests/vfs-example-fixture.spec.ts | 2 +- .../tests/vfs-example-fixture.ts | 2 +- .../session-projection-cache/src/spec.ts | 2 +- .../tests/cache.spec.ts | 2 +- .../storage/storage-json/README.i18n.yaml | 4 +- packages/storage/storage-json/README.md | 2 +- packages/storage/storage-json/README.zh.md | 2 +- .../storage-json/src/per-record-unit.ts | 23 ++++++----- .../storage-json/tests/json-backend.spec.ts | 40 ++++++++++++++++--- 13 files changed, 66 insertions(+), 31 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.i18n.yaml b/.agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.i18n.yaml index 7409f997a1..1bcec5b231 100644 --- a/.agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.md -2026-08-19-projection-cache-per-session-files.md: 9e102e786a6c06d82d1a0f45cc2f96a50c8abcd8 -2026-08-19-projection-cache-per-session-files.zh.md: d875c3f57800936f66fbf65233637df9bf300e2d +2026-08-19-projection-cache-per-session-files.md: 0fd171649c1d8c7c3be7a8089287d43782b84714 +2026-08-19-projection-cache-per-session-files.zh.md: 47decb598cc70233c287feab2dddff46bd7bcbc9 diff --git a/.agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.md b/.agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.md index 9e102e786a..0fd171649c 100644 --- a/.agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.md +++ b/.agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.md @@ -20,8 +20,9 @@ Reads and writes share ONE coherent state: every read (`cachedSnapshot`) is a sy - Listing is a synchronous in-memory read; a session without a record document simply lacks the projection column. - ACP, headless, SDK, and Web sessions publish cache rows for later consumers. The log-leading durability barrier may flush a covered prefix at the cache cadence and split otherwise coalesced physical JSONL runs; recorded profile snapshots re-pack the logical event stream so cache timing does not define fixture layout. - The per-record contract scopes failure: a malformed or stale-version document reads as an absent record at open, so one bad file never bricks the cache, and a checkpoint schema bump discards stale sessions per record instead of rejecting the whole domain. -- The json backend bootstraps the per-record tree from the legacy whole-unit cache only when enumeration finds no new-layout document path. Any new document path, including an unreadable or stale file, suppresses the bootstrap for the whole unit; missing session rows refold from the log. The legacy file remains untouched. -- The cache record is bound to the same log lifecycle as before: the stored `{createdAt, cwd}` identity guards against a recreated id. +- The json backend bootstraps the per-record tree from the legacy whole-unit cache only when enumeration finds no new-layout document path and the legacy unit name and version match the requested descriptor. A different version remains untouched and the new domain opens empty; storage never relabels its values as the current version. Any new document path, including an unreadable or stale file, suppresses the bootstrap for the whole unit; missing session rows refold from the log. +- The `session_projcache` domain uses version 6. Every version-5 record reads as absent, including a healthy one, so a poisoned version-5 record cannot fail domain validation. Session headers and event logs remain in session persistence; an exact read refolds them and writes a version-6 cache record, while zero-I/O listings lack that projection until the cache returns. +- The cache record is bound to the same log lifecycle as before: the stored `{createdAt, cwd, isSeeded, inheritedEventCount}` identity guards against a recreated id or a mismatched inherited prefix. ## Alternatives considered @@ -29,3 +30,4 @@ Reads and writes share ONE coherent state: every read (`cachedSnapshot`) is a sy - **Cache-owned per-session files** (`//projection_cache.json`, the first revision of this change). Tried and reverted in review: the cache hand-rolled the medium — paths, per-path write chains, in-flight tracking, owner-only file modes, and a sqlite no-path special case — and its listing read hit the disk directly on every call while writes were throttled, so reads and writes were never consistent. - **Resolve the path through `sessionPersistence.locate(meta)`** (the file beside the session log). Rejected: the cache would have to guess "beside the log" from a log artifact path (`dirname` + fixed filename), coupling the cache to the persistence service and to a backend's layout. - **Make `per-record` a mode of the existing unit instead of a separate unit class.** Rejected: the two layouts have genuinely different state models — `single` is memory-authoritative with whole-file publish, `per-record` is stateless (the directory is the state; `loadAll` re-reads the tree) — so they are separate small classes behind one backend, with record keys validated path-safe instead of encoded. +- **Copy legacy values across unit versions.** Rejected: the json backend does not know a domain's record schema and cannot derive session-lineage fields. Copying raw values under the requested version relabels data without migrating it. A domain that requires compatibility owns an explicit migration; the projection cache instead discards old records and rebuilds them from session logs. diff --git a/.agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.zh.md b/.agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.zh.md index d875c3f578..47decb598c 100644 --- a/.agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.zh.md +++ b/.agents/notes/implemented/architecture/2026-08-19-projection-cache-per-session-files.zh.md @@ -20,8 +20,9 @@ Status: implemented - 列表读取是同步内存读；没有记录文档的会话只是缺少投影列。 - ACP、headless、SDK 与 Web 会话都会发布缓存行，供后续消费方使用。确保日志领先的持久性屏障可能按缓存节奏 flush 已覆盖的前缀，并拆分原本会合并的物理 JSONL 行；各 profile 的录制快照会重新 pack 逻辑事件流，因此缓存时序不会决定 fixture 布局。 - per-record 契约把故障范围缩小到单记录：畸形或过期版本的文档在打开时读作"无此记录"，单个坏文件不会拖垮整个缓存；检查点 schema 升级按会话丢弃过期行，而不是拒绝整个域。 -- json 后端仅在枚举时没有发现任何新布局文档路径，才从旧整单元缓存引导 per-record 目录树。只要存在任意新文档路径，即使文件不可读或版本陈旧，也会对整个单元禁用引导；缺失的会话行从日志重折叠。旧文件保持不变。 -- 缓存记录仍绑定同一日志生命周期：存储的 `{createdAt, cwd}` 身份防止被重建的 id 误导。 +- json 后端仅在枚举时没有发现任何新布局文档路径，且旧单元名称和版本与请求的 descriptor 相同时，才从旧整单元缓存引导 per-record 目录树。版本不同时，旧文件保持不变，新域为空；存储不会把旧值改标为当前版本。只要存在任意新文档路径，即使文件不可读或版本陈旧，也会对整个单元禁用引导；缺失的会话行从日志重折叠。 +- `session_projcache` 域使用版本 6。所有版本 5 记录都读作缺失，包括健康记录，因此被污染的版本 5 记录不能再使域校验失败。会话 header 和事件日志仍保存在会话持久化中；精确读取会重折叠这些数据并写入版本 6 缓存，而零 I/O 列表在缓存恢复前缺少对应投影。 +- 缓存记录仍绑定同一日志生命周期：存储的 `{createdAt, cwd, isSeeded, inheritedEventCount}` 身份防止被重建的 id 或不匹配的继承前缀误导。 ## Alternatives considered @@ -29,3 +30,4 @@ Status: implemented - **缓存自持的每会话文件**（`//projection_cache.json`，本改动的第一版）。试过并在评审中回退：缓存手搓了介质——路径、按路径的写链、在途跟踪、仅属主文件权限，以及 sqlite 无路径特判——而且它的列表读每次调用都直读磁盘、写却在节流，读写永不一致。 - **经 `sessionPersistence.locate(meta)` 解析路径**（文件放在会话日志旁）。未采用：缓存得从日志 artifact 路径"猜"日志旁边（`dirname` + 固定文件名），把缓存耦合到持久化服务与后端的布局。 - **把 `per-record` 做成既有单元的一种模式而非独立单元类。** 未采用：两种布局的状态模型本质不同——`single` 内存权威、整文件发布；`per-record` 无状态（目录即状态，`loadAll` 重扫目录树）——所以它们是同一后端下的两个小型独立类，记录键做路径安全校验而非编码。 +- **跨单元版本复制旧值。** 未采用：json 后端不知道域的记录 schema，也无法推导会话谱系字段。按请求版本复制原始值只会修改数据标签，不会迁移数据。需要兼容性的域负责显式迁移；投影缓存改为丢弃旧记录，并从会话日志重建。 diff --git a/packages/experimental/webworker-runtime/tests/fixtures/vfs-example/home/storages/session_projcache.json b/packages/experimental/webworker-runtime/tests/fixtures/vfs-example/home/storages/session_projcache.json index e612124830..a4a3725927 100644 --- a/packages/experimental/webworker-runtime/tests/fixtures/vfs-example/home/storages/session_projcache.json +++ b/packages/experimental/webworker-runtime/tests/fixtures/vfs-example/home/storages/session_projcache.json @@ -1,7 +1,7 @@ { "unit": { "name": "session_projcache", - "version": 5 + "version": 6 }, "global": null, "tables": { diff --git a/packages/experimental/webworker-runtime/tests/vfs-example-fixture.spec.ts b/packages/experimental/webworker-runtime/tests/vfs-example-fixture.spec.ts index bd7c0f6838..cfb102f0fb 100644 --- a/packages/experimental/webworker-runtime/tests/vfs-example-fixture.spec.ts +++ b/packages/experimental/webworker-runtime/tests/vfs-example-fixture.spec.ts @@ -64,7 +64,7 @@ describe('WebWorker preview VFS example', () => { }> } } - expect(cache.unit).toEqual({ name: 'session_projcache', version: 5 }) + expect(cache.unit).toEqual({ name: 'session_projcache', version: 6 }) expect(cache.tables.sessions[VFS_EXAMPLE_SESSION_IDS.main]).toMatchObject({ identity: { createdAt: 1_787_472_000_000, diff --git a/packages/experimental/webworker-runtime/tests/vfs-example-fixture.ts b/packages/experimental/webworker-runtime/tests/vfs-example-fixture.ts index 20ac1e3838..1ee62bc7cf 100644 --- a/packages/experimental/webworker-runtime/tests/vfs-example-fixture.ts +++ b/packages/experimental/webworker-runtime/tests/vfs-example-fixture.ts @@ -412,7 +412,7 @@ export function buildVfsExampleFiles(): ReadonlyMap { const project = projectKey(WORKSPACE) const sessionPath = (id: string): string => `home/sessions/${project}/${id}/session.jsonl` const projectionCache = `${JSON.stringify({ - unit: { name: 'session_projcache', version: 5 }, + unit: { name: 'session_projcache', version: 6 }, global: null, tables: { sessions: { diff --git a/packages/session/session-projection-cache/src/spec.ts b/packages/session/session-projection-cache/src/spec.ts index 35351917a5..ce79ea56be 100644 --- a/packages/session/session-projection-cache/src/spec.ts +++ b/packages/session/session-projection-cache/src/spec.ts @@ -72,7 +72,7 @@ export type CheckpointRecord = z.infer */ export const projectionCacheDomainSpec = defineDomain({ name: 'session_projcache', - version: 5, + version: 6, layout: 'per-record', tables: { sessions: domainTable(checkpointRecord) }, }) diff --git a/packages/session/session-projection-cache/tests/cache.spec.ts b/packages/session/session-projection-cache/tests/cache.spec.ts index bfdca18567..197ed8e8cc 100644 --- a/packages/session/session-projection-cache/tests/cache.spec.ts +++ b/packages/session/session-projection-cache/tests/cache.spec.ts @@ -383,7 +383,7 @@ describe('SessionProjectionCache listing read', () => { const path = recordPath(root, SessionId('all-stale')) await mkdir(dirname(path), { recursive: true }) await writeFile(path, JSON.stringify({ - version: projectionCacheDomainSpec.version + 1, + version: projectionCacheDomainSpec.version - 1, record: { identity: { createdAt: 0, isSeeded: false, inheritedEventCount: 0 }, rows: { 'cache-test/marks': { ver: 1, seq: 4, val: { marks: ['old'] } } }, diff --git a/packages/storage/storage-json/README.i18n.yaml b/packages/storage/storage-json/README.i18n.yaml index ed3d4b14a3..4ecb05fb60 100644 --- a/packages/storage/storage-json/README.i18n.yaml +++ b/packages/storage/storage-json/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/storage/storage-json/README.md -README.md: 383452b039f6190a523b8dcf969693e5e08a9456 -README.zh.md: 06cf35804abfea4c221fe966dc4d395d0ad5a707 +README.md: 2972a617248c73e6105e8f19635e2325337405b6 +README.zh.md: 5c9f52d2a12981f2bff36e4df198270d09f2a8e5 diff --git a/packages/storage/storage-json/README.md b/packages/storage/storage-json/README.md index 383452b039..2972a61724 100644 --- a/packages/storage/storage-json/README.md +++ b/packages/storage/storage-json/README.md @@ -55,7 +55,7 @@ The generated [configuration catalog](../../../docs/config-catalog.md#deepseek-a A missing `single` file or `per-record` directory opens as an empty unit and materializes on the first write. In `single`, malformed content rejects with `malformed-medium`, and a different stored version rejects with `version-mismatch`. In `per-record`, each malformed, unreadable, or differently versioned document reads as an absent record, so one bad document does not reject the unit. Record keys must match `[a-zA-Z0-9_-]+`; an unsafe key rejects before any file operation. Every resolved write is durable, and operations after close reject with `closed`. -An empty `per-record` tree can initialize its declared tables from a valid `/.json` whole-unit document. The backend leaves that source file unchanged. Any document path in a declared table, or a declared `global.json`, suppresses this initialization for the complete unit, even if that document is unreadable or stale. +An empty `per-record` tree can initialize its declared tables from a valid `/.json` whole-unit document only when the source unit name and version match the current descriptor. The backend leaves that source file unchanged. A different source version leaves the new tree empty. Any document path in a declared table, or a declared `global.json`, suppresses this initialization for the complete unit, even if that document is unreadable or stale. ----- diff --git a/packages/storage/storage-json/README.zh.md b/packages/storage/storage-json/README.zh.md index 06cf35804a..5c9f52d2a1 100644 --- a/packages/storage/storage-json/README.zh.md +++ b/packages/storage/storage-json/README.zh.md @@ -55,7 +55,7 @@ kind: "package-reference" 缺失的 `single` 文件或 `per-record` 目录会作为空单元打开，并在第一次写入时物化。在 `single` 中，畸形内容以 `malformed-medium` 拒绝，不同的已存版本以 `version-mismatch` 拒绝。在 `per-record` 中，每份畸形、不可读或版本不同的文档都读作记录不存在，因此单个坏文档不会使单元被拒绝。记录键必须匹配 `[a-zA-Z0-9_-]+`；不安全的键在任何文件操作前被拒绝。每次已完成的写入都已持久化，关闭后的操作以 `closed` 拒绝。 -空的 `per-record` 目录树可以从有效的 `/.json` 整单元文档初始化其已声明表。后端保持该源文件不变。已声明表中只要存在任意文档路径，或存在已声明的 `global.json`，就会对整个单元禁止该初始化，即使该文档不可读或版本陈旧。 +只有当源单元名称和版本与当前描述符相同时，空的 `per-record` 目录树才可以从有效的 `/.json` 整单元文档初始化其已声明表。后端保持该源文件不变。源版本不同会使新目录树保持为空。已声明表中只要存在任意文档路径，或存在已声明的 `global.json`，就会对整个单元禁止该初始化，即使该文档不可读或版本陈旧。 ----- diff --git a/packages/storage/storage-json/src/per-record-unit.ts b/packages/storage/storage-json/src/per-record-unit.ts index b75f9f9452..de78d1fd75 100644 --- a/packages/storage/storage-json/src/per-record-unit.ts +++ b/packages/storage/storage-json/src/per-record-unit.ts @@ -17,8 +17,9 @@ * * Legacy bootstrap: when the new tree has no document path, a legacy * whole-unit file `/.json` (the pre-per-record layout) seeds - * per-record documents. Any new document path, including one whose contents - * are unreadable or stale, suppresses the bootstrap for the whole unit. The + * per-record documents only when its unit name and version match the current + * descriptor. Any new document path, including one whose contents are + * unreadable or stale, suppresses the bootstrap for the whole unit. The * legacy file is never changed or deleted. * @module @deepseek-ai/dsh-storage-json/src/per-record-unit */ @@ -100,9 +101,10 @@ async function loadPerRecordState(descriptor: KvUnitDescriptor, dir: string): Pr /** * Bootstrap an empty per-record tree from a legacy whole-unit file * (`/.json`, the pre-per-record layout). Every declared-table - * record is copied into a current-version document, while the legacy file is - * retained unchanged. A missing, foreign (another unit's name), malformed, - * or non-unit legacy file is left alone; other read failures propagate. + * record is copied into a same-version document, while the legacy file is + * retained unchanged. A missing, foreign (another unit name or version), + * malformed, or non-unit legacy file is left alone; other read failures + * propagate. * @param descriptor - Static identity and shape of the unit. * @param dir - The per-record unit directory (`/`). * @param state - The empty tree state; bootstrapped records are added. @@ -116,16 +118,15 @@ async function bootstrapLegacyUnit(descriptor: KvUnitDescriptor, dir: string, st if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error return } - // The legacy document is runtime data: only `unit.name` and the tables map - // shape are checked here — the record values are migrated as-is and the - // domain layer's schemas judge them. - let document: { unit?: { name?: unknown }; tables?: unknown } + // The legacy document is runtime data: its unit identity and the tables map + // shape are checked here; the domain layer's schemas judge record values. + let document: { unit?: { name?: unknown; version?: unknown }; tables?: unknown } try { - document = JSON.parse(text) as { unit?: { name?: unknown }; tables?: unknown } + document = JSON.parse(text) as { unit?: { name?: unknown; version?: unknown }; tables?: unknown } } catch { return // Malformed legacy file: not ours to interpret or delete. } - if (document.unit?.name !== descriptor.name) return + if (document.unit?.name !== descriptor.name || document.unit.version !== descriptor.version) return const tables = document.tables if (typeof tables !== 'object' || tables === null) return const recordsByTable = tables as Record> diff --git a/packages/storage/storage-json/tests/json-backend.spec.ts b/packages/storage/storage-json/tests/json-backend.spec.ts index bafbf92ef4..4d6d4cebcf 100644 --- a/packages/storage/storage-json/tests/json-backend.spec.ts +++ b/packages/storage/storage-json/tests/json-backend.spec.ts @@ -329,10 +329,10 @@ describe('per-record layout', () => { it('bootstraps an empty per-record tree from a legacy whole-unit file and preserves it', async () => { const root = await freshRoot() - // A legacy single-layout file for the same unit (any older version); - // the extra table is not declared and must be skipped. + // A legacy single-layout file for the same unit and version; the extra + // table is not declared and must be skipped. const legacy = JSON.stringify({ - unit: { name: 'recs', version: 3 }, + unit: { name: 'recs', version: descriptor.version }, global: null, tables: { t: { old1: { v: 1 }, old2: { v: 2 } }, undeclared: { k: { v: 0 } } }, }) @@ -347,10 +347,26 @@ describe('per-record layout', () => { await backend.close() }) + it('leaves an older-version legacy whole-unit file unconverted', async () => { + const root = await freshRoot() + const legacy = JSON.stringify({ + unit: { name: 'recs', version: descriptor.version - 1 }, + global: null, + tables: { t: { old: { v: 1 } } }, + }) + await writeFile(join(root, 'recs.json'), legacy, 'utf8') + const backend = new JsonStorageBackend(root) + const unit = await backend.kv.open(descriptor) + expect(await unit.loadAll()).toEqual({ tables: { t: {} }, global: null }) + await expect(readFile(recordPath(root, 'old'), 'utf8')).rejects.toMatchObject({ code: 'ENOENT' }) + await expect(readFile(join(root, 'recs.json'), 'utf8')).resolves.toBe(legacy) + await backend.close() + }) + it('ignores the legacy whole-unit file when any new document path exists', async () => { const root = await freshRoot() const legacy = JSON.stringify({ - unit: { name: 'recs', version: 1 }, + unit: { name: 'recs', version: descriptor.version }, global: null, tables: { t: { old: { v: 1 } } }, }) @@ -400,11 +416,25 @@ describe('per-record layout', () => { await backend4.close() const root5 = await freshRoot() - await writeFile(join(root5, 'recs.json'), JSON.stringify({ unit: { name: 'recs' }, tables: 'not an object' }), 'utf8') + await writeFile( + join(root5, 'recs.json'), + JSON.stringify({ unit: { name: 'recs', version: descriptor.version }, tables: 'not an object' }), + 'utf8', + ) const backend5 = new JsonStorageBackend(root5) const unit5 = await backend5.kv.open(descriptor) expect(await unit5.loadAll()).toEqual({ tables: { t: {} }, global: null }) await expect(readFile(join(root5, 'recs.json'), 'utf8')).resolves.toContain('not an object') await backend5.close() + + await writeFile( + join(root5, 'recs.json'), + JSON.stringify({ unit: { name: 'recs', version: descriptor.version }, tables: null }), + 'utf8', + ) + const backend6 = new JsonStorageBackend(root5) + const unit6 = await backend6.kv.open(descriptor) + expect(await unit6.loadAll()).toEqual({ tables: { t: {} }, global: null }) + await backend6.close() }) })