From 83a526eea1342f3be36c54554b43f8b98ca6c87d Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 10:57:39 +0800 Subject: [PATCH] feat(attachment-local): store a deterministic canonical image encoding Admission now validates a wide source envelope (32MiB, 100MP, 16384px per side) and persists a canonical encoding instead of refusing large sources: EXIF orientation baked in, metadata stripped, long edge downscaled to the configured canonical target (default 2048px), PNG palette for alpha/PNG/GIF sources and a fixed JPEG quality ladder (85/75/60/45) until the canonical byte target holds (default 1MiB). In-budget PNG/JPEG/WebP passes through byte-identically so equal sources keep deduplicating to the same content address; GIF always re-encodes to the PNG of its first frame, pinning the first-frame meaning providers apply. Encoder parameters are fixed by design; only the canonical budget is deployment configuration. --- .../attachment-local/src/canonical.ts | 103 +++++++++++++ .../attachment/attachment-local/src/index.ts | 45 ++++-- .../attachment/attachment-local/src/store.ts | 21 ++- .../attachment-local/tests/canonical.spec.ts | 139 ++++++++++++++++++ .../attachment-local/tests/index.spec.ts | 10 +- .../attachment-local/tests/store.spec.ts | 64 +++++--- 6 files changed, 340 insertions(+), 42 deletions(-) create mode 100644 packages/attachment/attachment-local/src/canonical.ts create mode 100644 packages/attachment/attachment-local/tests/canonical.spec.ts diff --git a/packages/attachment/attachment-local/src/canonical.ts b/packages/attachment/attachment-local/src/canonical.ts new file mode 100644 index 0000000000..ada2164566 --- /dev/null +++ b/packages/attachment/attachment-local/src/canonical.ts @@ -0,0 +1,103 @@ +/** + * Deterministic canonical image encoding. Admission stores this encoding, so + * the same source bytes always publish the same content address on one + * runtime: encoder parameters are fixed here, never configurable, because a + * parameter change would silently split the content-addressed space. The + * deployment chooses only the canonical budget (long edge and byte target). + */ + +import sharp, { type Sharp } from 'sharp' +import { AttachmentError } from '@deepseek-ai/dsh-attachment' +import type { ImageMediaType } from '@deepseek-ai/dsh-attachment' +import type { DetectedImage } from './image.ts' + +/** Deployment-resolved canonical encoding budget. */ +export interface CanonicalImagePolicy { + /** Long-edge target in pixels; a larger source is downscaled proportionally. */ + maxDimension: number + /** Encoded-byte target; a larger encoding falls down the fixed quality ladder. */ + maxBytes: number +} + +/** Canonical bytes beside the facts a durable reference records about them. */ +export interface CanonicalImage { + data: Uint8Array + mediaType: ImageMediaType + width: number + height: number +} + +/** JPEG quality ladder tried in order once the preferred encoding exceeds the byte target. */ +const JPEG_QUALITIES = [85, 75, 60, 45] as const + +/** Encode one prepared pipeline and report the exact output facts. */ +async function encode(pipeline: Sharp, mediaType: 'image/png' | 'image/jpeg'): Promise { + const { data, info } = await pipeline.toBuffer({ resolveWithObject: true }) + return { data: new Uint8Array(data), mediaType, width: info.width, height: info.height } +} + +/** + * Whether stored bytes may be the submitted bytes unchanged. Byte-identical + * passthrough is preferred whenever the source already fits the budget: it + * keeps re-submissions of the same original deduplicating to the same object + * and never re-encodes what no policy requires changing. GIF is excluded — + * only its first frame is model-visible, so admission pins that meaning into + * the stored object instead of letting each provider drop frames differently. + * @param detected - verified source format and dimensions. + * @param bytes - submitted encoded byte length. + * @param policy - resolved canonical budget. + * @returns whether the submitted encoding already is canonical. + */ +export function isCanonical(detected: DetectedImage, bytes: number, policy: CanonicalImagePolicy): boolean { + return detected.mediaType !== 'image/gif' + && bytes <= policy.maxBytes + && Math.max(detected.width, detected.height) <= policy.maxDimension +} + +/** + * Produce the canonical encoding of one fully validated source raster. + * Passthrough returns the submitted array; every re-encode bakes EXIF + * orientation into pixels, strips metadata, downscales to the policy's long + * edge, and encodes with fixed parameters: PNG (palette) for sources that + * carry alpha or were PNG/GIF, JPEG for photographic sources, falling down + * one fixed JPEG quality ladder until the byte target holds. + * @param data - submitted encoded bytes, already fully decoded by admission. + * @param detected - verified source format and dimensions. + * @param policy - resolved canonical budget. + * @returns canonical bytes and their reference facts. + * @throws AttachmentError `IMAGE_TOO_LARGE` when the smallest ladder step still exceeds the byte target. + */ +export async function canonicalizeImage( + data: Uint8Array, + detected: DetectedImage, + policy: CanonicalImagePolicy, +): Promise { + if (isCanonical(detected, data.byteLength, policy)) { + return { data, mediaType: detected.mediaType, width: detected.width, height: detected.height } + } + try { + const source = sharp(data, { failOn: 'error', limitInputPixels: false }) + const { hasAlpha } = await source.metadata() + const prepared = source.rotate().resize({ + width: policy.maxDimension, + height: policy.maxDimension, + fit: 'inside', + withoutEnlargement: true, + }) + const preferPng = hasAlpha || detected.mediaType === 'image/png' || detected.mediaType === 'image/gif' + if (preferPng) { + const png = await encode(prepared.clone().png({ compressionLevel: 9, palette: true }), 'image/png') + if (png.data.byteLength <= policy.maxBytes) return png + } + for (const quality of JPEG_QUALITIES) { + const jpeg = await encode( + prepared.clone().flatten({ background: '#ffffff' }).jpeg({ quality }), + 'image/jpeg', + ) + if (jpeg.data.byteLength <= policy.maxBytes) return jpeg + } + } catch (error) { + throw new AttachmentError('Unable to canonicalize image attachment.', 'ATTACHMENT_WRITE_FAILED', { cause: error }) + } + throw new AttachmentError('Image cannot be encoded within the configured canonical byte target.', 'IMAGE_TOO_LARGE') +} diff --git a/packages/attachment/attachment-local/src/index.ts b/packages/attachment/attachment-local/src/index.ts index b529270c31..cbd702c2bf 100644 --- a/packages/attachment/attachment-local/src/index.ts +++ b/packages/attachment/attachment-local/src/index.ts @@ -6,41 +6,50 @@ import z from '@deepseek-ai/schemastery' import { AttachmentStore } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, SavedImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' import { resolveDshHome } from '@deepseek-ai/dsh-home-paths' +import type { CanonicalImagePolicy } from './canonical.ts' import { readImageFile, saveImageFile, validateImageFile } from './store.ts' +export { canonicalizeImage, isCanonical } from './canonical.ts' +export type { CanonicalImage, CanonicalImagePolicy } from './canonical.ts' export { readImageFile, saveImageFile, validateImageFile } from './store.ts' -/** Default maximum encoded bytes for one image. */ -export const DEFAULT_MAX_IMAGE_BYTES = 3.5 * 1024 * 1024 +/** Default maximum encoded bytes for one submitted image; oversized sources are refused, not shrunk. */ +export const DEFAULT_MAX_IMAGE_BYTES = 32 * 1024 * 1024 /** Default maximum images in one prompt. */ export const DEFAULT_MAX_IMAGES_PER_MESSAGE = 20 /** Default maximum aggregate image bytes in one prompt. */ export const DEFAULT_MAX_MESSAGE_IMAGE_BYTES = 100 * 1024 * 1024 -/** Default maximum intrinsic pixels for one image. */ -export const DEFAULT_MAX_IMAGE_PIXELS = 40_000_000 +/** Default maximum intrinsic pixels for one submitted image. */ +export const DEFAULT_MAX_IMAGE_PIXELS = 100_000_000 +/** Default per-side pixel cap for one submitted image. */ +export const DEFAULT_MAX_IMAGE_DIMENSION = 16384 /** - * Default maximum intrinsic width and height for one image. Deployed model - * routes reject any request whose history carries an image with a side above - * 2000px once the request holds many images, and an admitted image rides - * every later request of its session, so admission refuses at the same line - * to keep the durable history streamable. + * Default long-edge target of the stored canonical encoding. A larger source + * is admitted and downscaled to this edge, so admission bounds what rides + * every later model request without refusing ordinary large sources. */ -export const DEFAULT_MAX_IMAGE_DIMENSION = 2000 +export const DEFAULT_CANONICAL_MAX_DIMENSION = 2048 +/** Default byte target of the stored canonical encoding. */ +export const DEFAULT_CANONICAL_MAX_BYTES = 1024 * 1024 /** Local attachment backend configuration. */ export interface Config { /** Explicit harness home; omitted follows `DSH_HOME`, then `~/.dsh`. */ dshHome?: string - /** Maximum encoded bytes accepted for one image. */ + /** Maximum encoded bytes accepted for one submitted image. */ maxImageBytes?: number /** Maximum image count accepted in one submitted message. */ maxImagesPerMessage?: number /** Maximum aggregate encoded image bytes accepted in one submitted message. */ maxMessageImageBytes?: number - /** Maximum intrinsic width multiplied by height accepted for one image. */ + /** Maximum intrinsic width multiplied by height accepted for one submitted image. */ maxImagePixels?: number - /** Maximum intrinsic width and maximum intrinsic height accepted for one image. */ + /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. */ maxImageDimension?: number + /** Long-edge pixel target of the stored canonical encoding. */ + canonicalMaxDimension?: number + /** Encoded-byte target of the stored canonical encoding. */ + canonicalMaxBytes?: number } /** Persistent content-addressed local attachment store. */ @@ -52,11 +61,15 @@ export class LocalAttachmentStore extends AttachmentStore { maxMessageImageBytes: z.number().step(1).min(1).default(DEFAULT_MAX_MESSAGE_IMAGE_BYTES), maxImagePixels: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGE_PIXELS), maxImageDimension: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGE_DIMENSION), + canonicalMaxDimension: z.number().step(1).min(1).default(DEFAULT_CANONICAL_MAX_DIMENSION), + canonicalMaxBytes: z.number().step(1).min(1).default(DEFAULT_CANONICAL_MAX_BYTES), }) /** Absolute versioned storage root. */ readonly root: string readonly imageLimits: ImageAttachmentLimits + /** Resolved canonical encoding budget applied by every save. */ + readonly canonicalPolicy: Readonly constructor(ctx: Context, config: Config) { super(ctx) @@ -69,6 +82,10 @@ export class LocalAttachmentStore extends AttachmentStore { maxImageDimension: config.maxImageDimension ?? DEFAULT_MAX_IMAGE_DIMENSION, mediaTypes: Object.freeze(['image/png', 'image/jpeg', 'image/webp', 'image/gif'] as const), }) + this.canonicalPolicy = Object.freeze({ + maxDimension: config.canonicalMaxDimension ?? DEFAULT_CANONICAL_MAX_DIMENSION, + maxBytes: config.canonicalMaxBytes ?? DEFAULT_CANONICAL_MAX_BYTES, + }) } async validateImage(input: SaveImageAttachment): Promise { @@ -76,7 +93,7 @@ export class LocalAttachmentStore extends AttachmentStore { } async saveImage(input: SaveImageAttachment): Promise { - return saveImageFile(this.root, input, this.imageLimits) + return saveImageFile(this.root, input, this.imageLimits, this.canonicalPolicy) } async readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise { diff --git a/packages/attachment/attachment-local/src/store.ts b/packages/attachment/attachment-local/src/store.ts index f98dbf0765..9da83e30a0 100644 --- a/packages/attachment/attachment-local/src/store.ts +++ b/packages/attachment/attachment-local/src/store.ts @@ -15,6 +15,8 @@ import type { SavedImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' +import { canonicalizeImage } from './canonical.ts' +import type { CanonicalImagePolicy } from './canonical.ts' import { detectImage, probeImage } from './image.ts' const ID_PATTERN = /^sha256:([a-f0-9]{64})$/ @@ -128,20 +130,26 @@ async function ensureDurableHome(path: string): Promise { } /** - * Save and verify immutable image bytes below a versioned attachment root. + * Save and verify one image below a versioned attachment root. Admission + * validates the submitted source, then stores its deterministic canonical + * encoding; the returned reference describes the stored canonical bytes while + * `source` preserves the submitted raster's facts. * @param root - absolute `DSH_HOME/attachments/v1` root. * @param input - encoded bytes and declared metadata. - * @param limits - resolved storage policy. + * @param limits - resolved source admission policy. + * @param policy - resolved canonical encoding budget. * @returns durable content-addressed reference beside the submitted source facts. */ export async function saveImageFile( root: string, input: SaveImageAttachment, limits: ImageAttachmentLimits, + policy: CanonicalImagePolicy, ): Promise { if (input.data.byteLength > limits.maxImageBytes) throw new AttachmentError('Image exceeds the configured byte limit.', 'IMAGE_TOO_LARGE') const metadata = await inspectMetadata(input.data, input.mediaType, limits) - const sha256 = digest(input.data) + const canonical = await canonicalizeImage(input.data, metadata, policy) + const sha256 = digest(canonical.data) const bucket = join(root, 'objects', sha256.slice(0, 2)) const staging = join(root, 'tmp') // Establish DSH_HOME itself against the filesystem root once per process. @@ -155,7 +163,7 @@ export async function saveImageFile( let handle try { handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600) - await handle.writeFile(input.data) + await handle.writeFile(canonical.data) await handle.sync() await handle.close() handle = undefined @@ -194,7 +202,10 @@ export async function saveImageFile( return { ref: { attachmentId: AttachmentId(`sha256:${sha256}`), - ...metadata, + mediaType: canonical.mediaType, + bytes: canonical.data.byteLength, + width: canonical.width, + height: canonical.height, ...(name !== undefined ? { name } : {}), }, source: metadata, diff --git a/packages/attachment/attachment-local/tests/canonical.spec.ts b/packages/attachment/attachment-local/tests/canonical.spec.ts new file mode 100644 index 0000000000..0441fbc462 --- /dev/null +++ b/packages/attachment/attachment-local/tests/canonical.spec.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from 'vitest' +import sharp from 'sharp' +import { canonicalizeImage, isCanonical } from '../src/canonical.ts' +import type { CanonicalImagePolicy } from '../src/canonical.ts' +import { detectImage } from '../src/image.ts' + +const POLICY: CanonicalImagePolicy = { maxDimension: 2048, maxBytes: 1024 * 1024 } + +/** Deterministic pseudo-random RGB noise; PNG cannot compress it below raw size. */ +function noisePixels(width: number, height: number): Uint8Array { + const pixels = new Uint8Array(width * height * 3) + let state = 0x2545f491 + for (let index = 0; index < pixels.length; index += 1) { + state = (state * 1103515245 + 12345) & 0x7fffffff + pixels[index] = state & 0xff + } + return pixels +} + +async function noiseImage(width: number, height: number, format: 'png' | 'jpeg' | 'webp' | 'gif'): Promise { + const image = sharp(noisePixels(width, height), { raw: { width, height, channels: 3 } }) + return new Uint8Array(await image.toFormat(format).toBuffer()) +} + +async function flatImage(width: number, height: number, format: 'png' | 'jpeg' | 'webp' | 'gif', alpha = false): Promise { + const image = sharp({ + create: { width, height, channels: alpha ? 4 : 3, background: { r: 12, g: 200, b: 64, alpha: alpha ? 0.5 : 1 } }, + }) + return new Uint8Array(await image.toFormat(format, format === 'webp' && alpha ? { lossless: true } : {}).toBuffer()) +} + +describe('isCanonical', () => { + it('accepts an in-budget PNG/JPEG/WebP and refuses GIF, oversized edges, and oversized bytes', () => { + expect(isCanonical({ mediaType: 'image/png', width: 2048, height: 4 }, 100, POLICY)).toBe(true) + expect(isCanonical({ mediaType: 'image/gif', width: 4, height: 4 }, 100, POLICY)).toBe(false) + expect(isCanonical({ mediaType: 'image/jpeg', width: 2049, height: 4 }, 100, POLICY)).toBe(false) + expect(isCanonical({ mediaType: 'image/webp', width: 4, height: 4 }, POLICY.maxBytes + 1, POLICY)).toBe(false) + }) +}) + +describe('canonicalizeImage', () => { + it('passes an already-canonical source through byte-identically', async () => { + const data = await flatImage(6, 4, 'webp') + const detected = await detectImage(data) + + const canonical = await canonicalizeImage(data, detected, POLICY) + + expect(canonical.data).toBe(data) + expect(canonical).toMatchObject({ mediaType: 'image/webp', width: 6, height: 4 }) + }) + + it('downscales an oversized PNG to the long-edge target and stays PNG', async () => { + const data = await flatImage(10, 6, 'png') + const detected = await detectImage(data) + + const canonical = await canonicalizeImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) + + expect(canonical).toMatchObject({ mediaType: 'image/png', width: 5, height: 3 }) + await expect(detectImage(canonical.data)).resolves.toEqual({ mediaType: 'image/png', width: 5, height: 3 }) + const again = await canonicalizeImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) + expect(again.data).toEqual(canonical.data) + }) + + it('re-encodes the canonical output of a resize into itself (idempotence)', async () => { + const data = await flatImage(10, 6, 'png') + const first = await canonicalizeImage(data, await detectImage(data), { maxDimension: 5, maxBytes: POLICY.maxBytes }) + + const second = await canonicalizeImage(first.data, await detectImage(first.data), { maxDimension: 5, maxBytes: POLICY.maxBytes }) + + expect(second.data).toBe(first.data) + }) + + it('always re-encodes GIF to the PNG of its first frame', async () => { + const data = await flatImage(6, 4, 'gif') + const detected = await detectImage(data) + + const canonical = await canonicalizeImage(data, detected, POLICY) + + expect(canonical.mediaType).toBe('image/png') + await expect(detectImage(canonical.data)).resolves.toEqual({ mediaType: 'image/png', width: 6, height: 4 }) + }) + + it('keeps alpha sources on PNG when the budget holds', async () => { + const data = await flatImage(9, 5, 'webp', true) + const detected = await detectImage(data) + + const canonical = await canonicalizeImage(data, detected, { maxDimension: 4, maxBytes: POLICY.maxBytes }) + + expect(canonical).toMatchObject({ mediaType: 'image/png', width: 4, height: 2 }) + }) + + it('re-encodes an oversized photographic JPEG as JPEG', async () => { + const data = await noiseImage(64, 32, 'jpeg') + const detected = await detectImage(data) + + const canonical = await canonicalizeImage(data, detected, { maxDimension: 32, maxBytes: POLICY.maxBytes }) + + expect(canonical).toMatchObject({ mediaType: 'image/jpeg', width: 32, height: 16 }) + }) + + it('falls from PNG to the JPEG ladder when palette PNG exceeds the byte target', async () => { + // A smooth gradient: palette quantization dithers it into a sizable PNG + // while JPEG at quality 85 stays far smaller, so the budget between the + // two forces exactly one ladder hop. + const side = 256 + const pixels = new Uint8Array(side * side * 3) + for (let y = 0; y < side; y += 1) { + for (let x = 0; x < side; x += 1) { + const index = (y * side + x) * 3 + pixels[index] = x & 0xff + pixels[index + 1] = y & 0xff + pixels[index + 2] = (x + y) >> 1 & 0xff + } + } + const data = new Uint8Array(await sharp(pixels, { raw: { width: side, height: side, channels: 3 } }).png().toBuffer()) + const detected = await detectImage(data) + const paletteSize = (await sharp(data).png({ compressionLevel: 9, palette: true }).toBuffer()).byteLength + const jpegSize = (await sharp(data).flatten({ background: '#ffffff' }).jpeg({ quality: 85 }).toBuffer()).byteLength + expect(jpegSize).toBeLessThan(paletteSize) + const budget = { maxDimension: 2048, maxBytes: paletteSize - 1 } + + const canonical = await canonicalizeImage(data, detected, budget) + + expect(canonical.mediaType).toBe('image/jpeg') + expect(canonical.data.byteLength).toBeLessThanOrEqual(budget.maxBytes) + }) + + it('refuses a source that no ladder step fits into the byte target', async () => { + const data = await noiseImage(64, 64, 'png') + + await expect(canonicalizeImage(data, await detectImage(data), { maxDimension: 2048, maxBytes: 10 })) + .rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' }) + }) + + it('maps an encoder fault on undecodable bytes to a storage failure', async () => { + await expect(canonicalizeImage(Uint8Array.of(1, 2, 3), { mediaType: 'image/png', width: 5000, height: 5000 }, POLICY)) + .rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED' }) + }) +}) diff --git a/packages/attachment/attachment-local/tests/index.spec.ts b/packages/attachment/attachment-local/tests/index.spec.ts index 92bbe3c0aa..0e86957f82 100644 --- a/packages/attachment/attachment-local/tests/index.spec.ts +++ b/packages/attachment/attachment-local/tests/index.spec.ts @@ -5,6 +5,8 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it } from 'vitest' import LocalAttachmentStore, { + DEFAULT_CANONICAL_MAX_BYTES, + DEFAULT_CANONICAL_MAX_DIMENSION, DEFAULT_MAX_IMAGE_BYTES, DEFAULT_MAX_IMAGE_DIMENSION, DEFAULT_MAX_IMAGE_PIXELS, @@ -15,7 +17,7 @@ import LocalAttachmentStore, { describe('local attachment service', () => { it('resolves every omitted admission limit explicitly', () => { const service = new LocalAttachmentStore(new Context(), {}) - expect(DEFAULT_MAX_IMAGE_BYTES).toBe(3.5 * 1024 * 1024) + expect(DEFAULT_MAX_IMAGE_BYTES).toBe(32 * 1024 * 1024) expect(service.imageLimits).toEqual({ maxImageBytes: DEFAULT_MAX_IMAGE_BYTES, maxImagesPerMessage: DEFAULT_MAX_IMAGES_PER_MESSAGE, @@ -24,6 +26,10 @@ describe('local attachment service', () => { maxImageDimension: DEFAULT_MAX_IMAGE_DIMENSION, mediaTypes: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'], }) + expect(service.canonicalPolicy).toEqual({ + maxDimension: DEFAULT_CANONICAL_MAX_DIMENSION, + maxBytes: DEFAULT_CANONICAL_MAX_BYTES, + }) }) it('saves and reads through the service boundary', async () => { @@ -34,7 +40,7 @@ describe('local attachment service', () => { 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=', 'base64', )) - const ref = await service.saveImage({ data, mediaType: 'image/png' }) + const { ref } = await service.saveImage({ data, mediaType: 'image/png' }) await expect(service.readImage(ref)).resolves.toEqual({ ref, data }) } finally { await rm(dshHome, { recursive: true, force: true }) diff --git a/packages/attachment/attachment-local/tests/store.spec.ts b/packages/attachment/attachment-local/tests/store.spec.ts index a5b831e933..8fdd076f6e 100644 --- a/packages/attachment/attachment-local/tests/store.spec.ts +++ b/packages/attachment/attachment-local/tests/store.spec.ts @@ -7,6 +7,7 @@ import { mkdtemp, rm } from 'node:fs/promises' import { afterEach, describe, expect, it, vi } from 'vitest' import sharp from 'sharp' import type { ImageAttachmentLimits } from '@deepseek-ai/dsh-attachment' +import type { CanonicalImagePolicy } from '../src/canonical.ts' import { readImageFile, saveImageFile } from '../src/store.ts' const fsControl = vi.hoisted(() => ({ @@ -38,6 +39,8 @@ const PNG = Uint8Array.from(Buffer.from( 'base64', )) +const POLICY: CanonicalImagePolicy = { maxDimension: 2048, maxBytes: 1024 * 1024 } + const LIMITS: ImageAttachmentLimits = { maxImageBytes: 1024, maxImagesPerMessage: 2, @@ -79,7 +82,7 @@ describe('local attachment store', () => { const bucket = join(objects, sha256.slice(0, 2)) fsControl.syncedDirectories.length = 0 - await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) // Each process first proves DSH_HOME durable all the way to the filesystem // root; existence alone cannot vouch for a concurrent creator's fsync. @@ -104,7 +107,7 @@ describe('local attachment store', () => { it('creates and persists a missing nested home directory against the filesystem root', async () => { const storageRoot = join(await root(), 'home', 'attachments', 'v1') - const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) await expect(readImageFile(storageRoot, ref)).resolves.toEqual({ ref, data: PNG }) }) @@ -113,12 +116,12 @@ describe('local attachment store', () => { const storageRoot = await root() const first = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png', name: '/private/tmp/pixel.png', - }, LIMITS) - const second = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + }, LIMITS, POLICY) + const second = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) const sha256 = createHash('sha256').update(PNG).digest('hex') const object = join(storageRoot, 'objects', sha256.slice(0, 2), sha256) - expect(first).toEqual({ + expect(first.ref).toEqual({ attachmentId: `sha256:${sha256}`, mediaType: 'image/png', bytes: PNG.byteLength, @@ -126,25 +129,44 @@ describe('local attachment store', () => { height: 1, name: 'pixel.png', }) - expect(second.attachmentId).toBe(first.attachmentId) + expect(first.source).toEqual({ mediaType: 'image/png', bytes: PNG.byteLength, width: 1, height: 1 }) + expect(second.ref.attachmentId).toBe(first.ref.attachmentId) expect(new Uint8Array(await readFile(object))).toEqual(PNG) if (process.platform !== 'win32') { expect((await stat(object)).mode & 0o777).toBe(0o600) expect((await stat(join(storageRoot, 'objects', sha256.slice(0, 2)))).mode & 0o777).toBe(0o700) } - await expect(readImageFile(storageRoot, first)).resolves.toEqual({ ref: first, data: PNG }) + await expect(readImageFile(storageRoot, first.ref)).resolves.toEqual({ ref: first.ref, data: PNG }) + }) + + it('stores the canonical encoding of an oversized source and reads it back verified', async () => { + const storageRoot = await root() + const oversized = new Uint8Array(await sharp({ + create: { width: 4, height: 4, channels: 3, background: { r: 9, g: 9, b: 9 } }, + }).png().toBuffer()) + + const saved = await saveImageFile(storageRoot, { + data: oversized, mediaType: 'image/png', name: 'big.png', + }, { ...LIMITS, maxImagePixels: 64 }, { maxDimension: 2, maxBytes: 1024 * 1024 }) + + expect(saved.source).toEqual({ mediaType: 'image/png', bytes: oversized.byteLength, width: 4, height: 4 }) + expect(saved.ref).toMatchObject({ mediaType: 'image/png', width: 2, height: 2, name: 'big.png' }) + expect(saved.ref.bytes).not.toBe(oversized.byteLength) + const read = await readImageFile(storageRoot, saved.ref) + expect(read.data.byteLength).toBe(saved.ref.bytes) + expect(String(saved.ref.attachmentId)).toBe(`sha256:${createHash('sha256').update(read.data).digest('hex')}`) }) it('keeps admitted history readable after deployment limits become stricter', async () => { const storageRoot = await root() - const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) await expect(readImageFile(storageRoot, ref)).resolves.toEqual({ ref, data: PNG }) }) it('forwards read cancellation to the filesystem and preserves its reason', async () => { const storageRoot = await root() - const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) const controller = new AbortController() fsControl.readSignals.length = 0 @@ -160,35 +182,35 @@ describe('local attachment store', () => { const storageRoot = await root() await expect(saveImageFile(storageRoot, { data: new Uint8Array(0), mediaType: 'image/png', - }, LIMITS)).rejects.toMatchObject({ code: 'INVALID_IMAGE' }) + }, LIMITS, POLICY)).rejects.toMatchObject({ code: 'INVALID_IMAGE' }) await expect(saveImageFile(storageRoot, { data: Uint8Array.of(1, 2, 3), mediaType: 'image/png', - }, LIMITS)).rejects.toMatchObject({ code: 'INVALID_IMAGE' }) + }, LIMITS, POLICY)).rejects.toMatchObject({ code: 'INVALID_IMAGE' }) await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/jpeg', - }, LIMITS)).rejects.toMatchObject({ code: 'IMAGE_TYPE_MISMATCH' }) + }, LIMITS, POLICY)).rejects.toMatchObject({ code: 'IMAGE_TYPE_MISMATCH' }) await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png', - }, { ...LIMITS, maxImageBytes: 1 })).rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' }) + }, { ...LIMITS, maxImageBytes: 1 }, POLICY)).rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' }) const wide = new Uint8Array(await sharp({ create: { width: 5, height: 5, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 1 } }, }).png().toBuffer()) await expect(saveImageFile(storageRoot, { data: wide, mediaType: 'image/png', - }, LIMITS)).rejects.toMatchObject({ code: 'IMAGE_TOO_MANY_PIXELS' }) + }, LIMITS, POLICY)).rejects.toMatchObject({ code: 'IMAGE_TOO_MANY_PIXELS' }) await expect(saveImageFile(storageRoot, { data: wide, mediaType: 'image/png', - }, { ...LIMITS, maxImagePixels: 25, maxImageDimension: 4 })).rejects.toMatchObject({ code: 'IMAGE_DIMENSION_TOO_LARGE' }) + }, { ...LIMITS, maxImagePixels: 25, maxImageDimension: 4 }, POLICY)).rejects.toMatchObject({ code: 'IMAGE_DIMENSION_TOO_LARGE' }) const unnamed = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png', name: '\u0000', - }, LIMITS) - expect(unnamed).not.toHaveProperty('name') + }, LIMITS, POLICY) + expect(unnamed.ref).not.toHaveProperty('name') }) it('fails closed when an object is missing, corrupted, or addressed by an invalid reference', async () => { const storageRoot = await root() - const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) const sha256 = String(ref.attachmentId).slice('sha256:'.length) const object = join(storageRoot, 'objects', sha256.slice(0, 2), sha256) await chmod(object, 0o600) @@ -216,11 +238,11 @@ describe('local attachment store', () => { const target = join(storageRoot, 'objects', sha256.slice(0, 2), sha256) await mkdir(join(storageRoot, 'objects', sha256.slice(0, 2)), { recursive: true }) await writeFile(target, Uint8Array.of(1, 2, 3)) - await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)) + await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY)) .rejects.toMatchObject({ code: 'ATTACHMENT_CORRUPT' }) await writeFile(target, PNG) - const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) await expect(readImageFile(storageRoot, { ...ref, width: ref.width + 1 })) .rejects.toMatchObject({ code: 'ATTACHMENT_CORRUPT' }) }) @@ -231,7 +253,7 @@ describe('local attachment store', () => { const target = join(storageRoot, 'objects', sha256.slice(0, 2), sha256) await mkdir(target, { recursive: true }) - await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)) + await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY)) .rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED' }) }) })