feat(attachment-local): store a deterministic canonical image encoding
Admission now validates a wide source envelope (32MiB, 100MP, 16384px per side) and persists a canonical encoding instead of refusing large sources: EXIF orientation baked in, metadata stripped, long edge downscaled to the configured canonical target (default 2048px), PNG palette for alpha/PNG/GIF sources and a fixed JPEG quality ladder (85/75/60/45) until the canonical byte target holds (default 1MiB). In-budget PNG/JPEG/WebP passes through byte-identically so equal sources keep deduplicating to the same content address; GIF always re-encodes to the PNG of its first frame, pinning the first-frame meaning providers apply. Encoder parameters are fixed by design; only the canonical budget is deployment configuration.
This commit is contained in:
creatixchu
parent
8f83853b60
commit
83a526eea1
6 changed files with 340 additions and 42 deletions
103
packages/attachment/attachment-local/src/canonical.ts
Normal file
103
packages/attachment/attachment-local/src/canonical.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
/**
|
||||
* Deterministic canonical image encoding. Admission stores this encoding, so
|
||||
* the same source bytes always publish the same content address on one
|
||||
* runtime: encoder parameters are fixed here, never configurable, because a
|
||||
* parameter change would silently split the content-addressed space. The
|
||||
* deployment chooses only the canonical budget (long edge and byte target).
|
||||
*/
|
||||
|
||||
import sharp, { type Sharp } from 'sharp'
|
||||
import { AttachmentError } from '@deepseek-ai/dsh-attachment'
|
||||
import type { ImageMediaType } from '@deepseek-ai/dsh-attachment'
|
||||
import type { DetectedImage } from './image.ts'
|
||||
|
||||
/** Deployment-resolved canonical encoding budget. */
|
||||
export interface CanonicalImagePolicy {
|
||||
/** Long-edge target in pixels; a larger source is downscaled proportionally. */
|
||||
maxDimension: number
|
||||
/** Encoded-byte target; a larger encoding falls down the fixed quality ladder. */
|
||||
maxBytes: number
|
||||
}
|
||||
|
||||
/** Canonical bytes beside the facts a durable reference records about them. */
|
||||
export interface CanonicalImage {
|
||||
data: Uint8Array
|
||||
mediaType: ImageMediaType
|
||||
width: number
|
||||
height: number
|
||||
}
|
||||
|
||||
/** JPEG quality ladder tried in order once the preferred encoding exceeds the byte target. */
|
||||
const JPEG_QUALITIES = [85, 75, 60, 45] as const
|
||||
|
||||
/** Encode one prepared pipeline and report the exact output facts. */
|
||||
async function encode(pipeline: Sharp, mediaType: 'image/png' | 'image/jpeg'): Promise<CanonicalImage> {
|
||||
const { data, info } = await pipeline.toBuffer({ resolveWithObject: true })
|
||||
return { data: new Uint8Array(data), mediaType, width: info.width, height: info.height }
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether stored bytes may be the submitted bytes unchanged. Byte-identical
|
||||
* passthrough is preferred whenever the source already fits the budget: it
|
||||
* keeps re-submissions of the same original deduplicating to the same object
|
||||
* and never re-encodes what no policy requires changing. GIF is excluded —
|
||||
* only its first frame is model-visible, so admission pins that meaning into
|
||||
* the stored object instead of letting each provider drop frames differently.
|
||||
* @param detected - verified source format and dimensions.
|
||||
* @param bytes - submitted encoded byte length.
|
||||
* @param policy - resolved canonical budget.
|
||||
* @returns whether the submitted encoding already is canonical.
|
||||
*/
|
||||
export function isCanonical(detected: DetectedImage, bytes: number, policy: CanonicalImagePolicy): boolean {
|
||||
return detected.mediaType !== 'image/gif'
|
||||
&& bytes <= policy.maxBytes
|
||||
&& Math.max(detected.width, detected.height) <= policy.maxDimension
|
||||
}
|
||||
|
||||
/**
|
||||
* Produce the canonical encoding of one fully validated source raster.
|
||||
* Passthrough returns the submitted array; every re-encode bakes EXIF
|
||||
* orientation into pixels, strips metadata, downscales to the policy's long
|
||||
* edge, and encodes with fixed parameters: PNG (palette) for sources that
|
||||
* carry alpha or were PNG/GIF, JPEG for photographic sources, falling down
|
||||
* one fixed JPEG quality ladder until the byte target holds.
|
||||
* @param data - submitted encoded bytes, already fully decoded by admission.
|
||||
* @param detected - verified source format and dimensions.
|
||||
* @param policy - resolved canonical budget.
|
||||
* @returns canonical bytes and their reference facts.
|
||||
* @throws AttachmentError `IMAGE_TOO_LARGE` when the smallest ladder step still exceeds the byte target.
|
||||
*/
|
||||
export async function canonicalizeImage(
|
||||
data: Uint8Array,
|
||||
detected: DetectedImage,
|
||||
policy: CanonicalImagePolicy,
|
||||
): Promise<CanonicalImage> {
|
||||
if (isCanonical(detected, data.byteLength, policy)) {
|
||||
return { data, mediaType: detected.mediaType, width: detected.width, height: detected.height }
|
||||
}
|
||||
try {
|
||||
const source = sharp(data, { failOn: 'error', limitInputPixels: false })
|
||||
const { hasAlpha } = await source.metadata()
|
||||
const prepared = source.rotate().resize({
|
||||
width: policy.maxDimension,
|
||||
height: policy.maxDimension,
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
const preferPng = hasAlpha || detected.mediaType === 'image/png' || detected.mediaType === 'image/gif'
|
||||
if (preferPng) {
|
||||
const png = await encode(prepared.clone().png({ compressionLevel: 9, palette: true }), 'image/png')
|
||||
if (png.data.byteLength <= policy.maxBytes) return png
|
||||
}
|
||||
for (const quality of JPEG_QUALITIES) {
|
||||
const jpeg = await encode(
|
||||
prepared.clone().flatten({ background: '#ffffff' }).jpeg({ quality }),
|
||||
'image/jpeg',
|
||||
)
|
||||
if (jpeg.data.byteLength <= policy.maxBytes) return jpeg
|
||||
}
|
||||
} catch (error) {
|
||||
throw new AttachmentError('Unable to canonicalize image attachment.', 'ATTACHMENT_WRITE_FAILED', { cause: error })
|
||||
}
|
||||
throw new AttachmentError('Image cannot be encoded within the configured canonical byte target.', 'IMAGE_TOO_LARGE')
|
||||
}
|
||||
|
|
@ -6,41 +6,50 @@ import z from '@deepseek-ai/schemastery'
|
|||
import { AttachmentStore } from '@deepseek-ai/dsh-attachment'
|
||||
import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, SavedImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment'
|
||||
import { resolveDshHome } from '@deepseek-ai/dsh-home-paths'
|
||||
import type { CanonicalImagePolicy } from './canonical.ts'
|
||||
import { readImageFile, saveImageFile, validateImageFile } from './store.ts'
|
||||
|
||||
export { canonicalizeImage, isCanonical } from './canonical.ts'
|
||||
export type { CanonicalImage, CanonicalImagePolicy } from './canonical.ts'
|
||||
export { readImageFile, saveImageFile, validateImageFile } from './store.ts'
|
||||
|
||||
/** Default maximum encoded bytes for one image. */
|
||||
export const DEFAULT_MAX_IMAGE_BYTES = 3.5 * 1024 * 1024
|
||||
/** Default maximum encoded bytes for one submitted image; oversized sources are refused, not shrunk. */
|
||||
export const DEFAULT_MAX_IMAGE_BYTES = 32 * 1024 * 1024
|
||||
/** Default maximum images in one prompt. */
|
||||
export const DEFAULT_MAX_IMAGES_PER_MESSAGE = 20
|
||||
/** Default maximum aggregate image bytes in one prompt. */
|
||||
export const DEFAULT_MAX_MESSAGE_IMAGE_BYTES = 100 * 1024 * 1024
|
||||
/** Default maximum intrinsic pixels for one image. */
|
||||
export const DEFAULT_MAX_IMAGE_PIXELS = 40_000_000
|
||||
/** Default maximum intrinsic pixels for one submitted image. */
|
||||
export const DEFAULT_MAX_IMAGE_PIXELS = 100_000_000
|
||||
/** Default per-side pixel cap for one submitted image. */
|
||||
export const DEFAULT_MAX_IMAGE_DIMENSION = 16384
|
||||
/**
|
||||
* Default maximum intrinsic width and height for one image. Deployed model
|
||||
* routes reject any request whose history carries an image with a side above
|
||||
* 2000px once the request holds many images, and an admitted image rides
|
||||
* every later request of its session, so admission refuses at the same line
|
||||
* to keep the durable history streamable.
|
||||
* Default long-edge target of the stored canonical encoding. A larger source
|
||||
* is admitted and downscaled to this edge, so admission bounds what rides
|
||||
* every later model request without refusing ordinary large sources.
|
||||
*/
|
||||
export const DEFAULT_MAX_IMAGE_DIMENSION = 2000
|
||||
export const DEFAULT_CANONICAL_MAX_DIMENSION = 2048
|
||||
/** Default byte target of the stored canonical encoding. */
|
||||
export const DEFAULT_CANONICAL_MAX_BYTES = 1024 * 1024
|
||||
|
||||
/** Local attachment backend configuration. */
|
||||
export interface Config {
|
||||
/** Explicit harness home; omitted follows `DSH_HOME`, then `~/.dsh`. */
|
||||
dshHome?: string
|
||||
/** Maximum encoded bytes accepted for one image. */
|
||||
/** Maximum encoded bytes accepted for one submitted image. */
|
||||
maxImageBytes?: number
|
||||
/** Maximum image count accepted in one submitted message. */
|
||||
maxImagesPerMessage?: number
|
||||
/** Maximum aggregate encoded image bytes accepted in one submitted message. */
|
||||
maxMessageImageBytes?: number
|
||||
/** Maximum intrinsic width multiplied by height accepted for one image. */
|
||||
/** Maximum intrinsic width multiplied by height accepted for one submitted image. */
|
||||
maxImagePixels?: number
|
||||
/** Maximum intrinsic width and maximum intrinsic height accepted for one image. */
|
||||
/** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. */
|
||||
maxImageDimension?: number
|
||||
/** Long-edge pixel target of the stored canonical encoding. */
|
||||
canonicalMaxDimension?: number
|
||||
/** Encoded-byte target of the stored canonical encoding. */
|
||||
canonicalMaxBytes?: number
|
||||
}
|
||||
|
||||
/** Persistent content-addressed local attachment store. */
|
||||
|
|
@ -52,11 +61,15 @@ export class LocalAttachmentStore extends AttachmentStore {
|
|||
maxMessageImageBytes: z.number().step(1).min(1).default(DEFAULT_MAX_MESSAGE_IMAGE_BYTES),
|
||||
maxImagePixels: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGE_PIXELS),
|
||||
maxImageDimension: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGE_DIMENSION),
|
||||
canonicalMaxDimension: z.number().step(1).min(1).default(DEFAULT_CANONICAL_MAX_DIMENSION),
|
||||
canonicalMaxBytes: z.number().step(1).min(1).default(DEFAULT_CANONICAL_MAX_BYTES),
|
||||
})
|
||||
|
||||
/** Absolute versioned storage root. */
|
||||
readonly root: string
|
||||
readonly imageLimits: ImageAttachmentLimits
|
||||
/** Resolved canonical encoding budget applied by every save. */
|
||||
readonly canonicalPolicy: Readonly<CanonicalImagePolicy>
|
||||
|
||||
constructor(ctx: Context, config: Config) {
|
||||
super(ctx)
|
||||
|
|
@ -69,6 +82,10 @@ export class LocalAttachmentStore extends AttachmentStore {
|
|||
maxImageDimension: config.maxImageDimension ?? DEFAULT_MAX_IMAGE_DIMENSION,
|
||||
mediaTypes: Object.freeze(['image/png', 'image/jpeg', 'image/webp', 'image/gif'] as const),
|
||||
})
|
||||
this.canonicalPolicy = Object.freeze({
|
||||
maxDimension: config.canonicalMaxDimension ?? DEFAULT_CANONICAL_MAX_DIMENSION,
|
||||
maxBytes: config.canonicalMaxBytes ?? DEFAULT_CANONICAL_MAX_BYTES,
|
||||
})
|
||||
}
|
||||
|
||||
async validateImage(input: SaveImageAttachment): Promise<void> {
|
||||
|
|
@ -76,7 +93,7 @@ export class LocalAttachmentStore extends AttachmentStore {
|
|||
}
|
||||
|
||||
async saveImage(input: SaveImageAttachment): Promise<SavedImageAttachment> {
|
||||
return saveImageFile(this.root, input, this.imageLimits)
|
||||
return saveImageFile(this.root, input, this.imageLimits, this.canonicalPolicy)
|
||||
}
|
||||
|
||||
async readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise<StoredImageAttachment> {
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ import type {
|
|||
SavedImageAttachment,
|
||||
StoredImageAttachment,
|
||||
} from '@deepseek-ai/dsh-attachment'
|
||||
import { canonicalizeImage } from './canonical.ts'
|
||||
import type { CanonicalImagePolicy } from './canonical.ts'
|
||||
import { detectImage, probeImage } from './image.ts'
|
||||
|
||||
const ID_PATTERN = /^sha256:([a-f0-9]{64})$/
|
||||
|
|
@ -128,20 +130,26 @@ async function ensureDurableHome(path: string): Promise<string> {
|
|||
}
|
||||
|
||||
/**
|
||||
* Save and verify immutable image bytes below a versioned attachment root.
|
||||
* Save and verify one image below a versioned attachment root. Admission
|
||||
* validates the submitted source, then stores its deterministic canonical
|
||||
* encoding; the returned reference describes the stored canonical bytes while
|
||||
* `source` preserves the submitted raster's facts.
|
||||
* @param root - absolute `DSH_HOME/attachments/v1` root.
|
||||
* @param input - encoded bytes and declared metadata.
|
||||
* @param limits - resolved storage policy.
|
||||
* @param limits - resolved source admission policy.
|
||||
* @param policy - resolved canonical encoding budget.
|
||||
* @returns durable content-addressed reference beside the submitted source facts.
|
||||
*/
|
||||
export async function saveImageFile(
|
||||
root: string,
|
||||
input: SaveImageAttachment,
|
||||
limits: ImageAttachmentLimits,
|
||||
policy: CanonicalImagePolicy,
|
||||
): Promise<SavedImageAttachment> {
|
||||
if (input.data.byteLength > limits.maxImageBytes) throw new AttachmentError('Image exceeds the configured byte limit.', 'IMAGE_TOO_LARGE')
|
||||
const metadata = await inspectMetadata(input.data, input.mediaType, limits)
|
||||
const sha256 = digest(input.data)
|
||||
const canonical = await canonicalizeImage(input.data, metadata, policy)
|
||||
const sha256 = digest(canonical.data)
|
||||
const bucket = join(root, 'objects', sha256.slice(0, 2))
|
||||
const staging = join(root, 'tmp')
|
||||
// Establish DSH_HOME itself against the filesystem root once per process.
|
||||
|
|
@ -155,7 +163,7 @@ export async function saveImageFile(
|
|||
let handle
|
||||
try {
|
||||
handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600)
|
||||
await handle.writeFile(input.data)
|
||||
await handle.writeFile(canonical.data)
|
||||
await handle.sync()
|
||||
await handle.close()
|
||||
handle = undefined
|
||||
|
|
@ -194,7 +202,10 @@ export async function saveImageFile(
|
|||
return {
|
||||
ref: {
|
||||
attachmentId: AttachmentId(`sha256:${sha256}`),
|
||||
...metadata,
|
||||
mediaType: canonical.mediaType,
|
||||
bytes: canonical.data.byteLength,
|
||||
width: canonical.width,
|
||||
height: canonical.height,
|
||||
...(name !== undefined ? { name } : {}),
|
||||
},
|
||||
source: metadata,
|
||||
|
|
|
|||
139
packages/attachment/attachment-local/tests/canonical.spec.ts
Normal file
139
packages/attachment/attachment-local/tests/canonical.spec.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import sharp from 'sharp'
|
||||
import { canonicalizeImage, isCanonical } from '../src/canonical.ts'
|
||||
import type { CanonicalImagePolicy } from '../src/canonical.ts'
|
||||
import { detectImage } from '../src/image.ts'
|
||||
|
||||
const POLICY: CanonicalImagePolicy = { maxDimension: 2048, maxBytes: 1024 * 1024 }
|
||||
|
||||
/** Deterministic pseudo-random RGB noise; PNG cannot compress it below raw size. */
|
||||
function noisePixels(width: number, height: number): Uint8Array {
|
||||
const pixels = new Uint8Array(width * height * 3)
|
||||
let state = 0x2545f491
|
||||
for (let index = 0; index < pixels.length; index += 1) {
|
||||
state = (state * 1103515245 + 12345) & 0x7fffffff
|
||||
pixels[index] = state & 0xff
|
||||
}
|
||||
return pixels
|
||||
}
|
||||
|
||||
async function noiseImage(width: number, height: number, format: 'png' | 'jpeg' | 'webp' | 'gif'): Promise<Uint8Array> {
|
||||
const image = sharp(noisePixels(width, height), { raw: { width, height, channels: 3 } })
|
||||
return new Uint8Array(await image.toFormat(format).toBuffer())
|
||||
}
|
||||
|
||||
async function flatImage(width: number, height: number, format: 'png' | 'jpeg' | 'webp' | 'gif', alpha = false): Promise<Uint8Array> {
|
||||
const image = sharp({
|
||||
create: { width, height, channels: alpha ? 4 : 3, background: { r: 12, g: 200, b: 64, alpha: alpha ? 0.5 : 1 } },
|
||||
})
|
||||
return new Uint8Array(await image.toFormat(format, format === 'webp' && alpha ? { lossless: true } : {}).toBuffer())
|
||||
}
|
||||
|
||||
describe('isCanonical', () => {
|
||||
it('accepts an in-budget PNG/JPEG/WebP and refuses GIF, oversized edges, and oversized bytes', () => {
|
||||
expect(isCanonical({ mediaType: 'image/png', width: 2048, height: 4 }, 100, POLICY)).toBe(true)
|
||||
expect(isCanonical({ mediaType: 'image/gif', width: 4, height: 4 }, 100, POLICY)).toBe(false)
|
||||
expect(isCanonical({ mediaType: 'image/jpeg', width: 2049, height: 4 }, 100, POLICY)).toBe(false)
|
||||
expect(isCanonical({ mediaType: 'image/webp', width: 4, height: 4 }, POLICY.maxBytes + 1, POLICY)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('canonicalizeImage', () => {
|
||||
it('passes an already-canonical source through byte-identically', async () => {
|
||||
const data = await flatImage(6, 4, 'webp')
|
||||
const detected = await detectImage(data)
|
||||
|
||||
const canonical = await canonicalizeImage(data, detected, POLICY)
|
||||
|
||||
expect(canonical.data).toBe(data)
|
||||
expect(canonical).toMatchObject({ mediaType: 'image/webp', width: 6, height: 4 })
|
||||
})
|
||||
|
||||
it('downscales an oversized PNG to the long-edge target and stays PNG', async () => {
|
||||
const data = await flatImage(10, 6, 'png')
|
||||
const detected = await detectImage(data)
|
||||
|
||||
const canonical = await canonicalizeImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes })
|
||||
|
||||
expect(canonical).toMatchObject({ mediaType: 'image/png', width: 5, height: 3 })
|
||||
await expect(detectImage(canonical.data)).resolves.toEqual({ mediaType: 'image/png', width: 5, height: 3 })
|
||||
const again = await canonicalizeImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes })
|
||||
expect(again.data).toEqual(canonical.data)
|
||||
})
|
||||
|
||||
it('re-encodes the canonical output of a resize into itself (idempotence)', async () => {
|
||||
const data = await flatImage(10, 6, 'png')
|
||||
const first = await canonicalizeImage(data, await detectImage(data), { maxDimension: 5, maxBytes: POLICY.maxBytes })
|
||||
|
||||
const second = await canonicalizeImage(first.data, await detectImage(first.data), { maxDimension: 5, maxBytes: POLICY.maxBytes })
|
||||
|
||||
expect(second.data).toBe(first.data)
|
||||
})
|
||||
|
||||
it('always re-encodes GIF to the PNG of its first frame', async () => {
|
||||
const data = await flatImage(6, 4, 'gif')
|
||||
const detected = await detectImage(data)
|
||||
|
||||
const canonical = await canonicalizeImage(data, detected, POLICY)
|
||||
|
||||
expect(canonical.mediaType).toBe('image/png')
|
||||
await expect(detectImage(canonical.data)).resolves.toEqual({ mediaType: 'image/png', width: 6, height: 4 })
|
||||
})
|
||||
|
||||
it('keeps alpha sources on PNG when the budget holds', async () => {
|
||||
const data = await flatImage(9, 5, 'webp', true)
|
||||
const detected = await detectImage(data)
|
||||
|
||||
const canonical = await canonicalizeImage(data, detected, { maxDimension: 4, maxBytes: POLICY.maxBytes })
|
||||
|
||||
expect(canonical).toMatchObject({ mediaType: 'image/png', width: 4, height: 2 })
|
||||
})
|
||||
|
||||
it('re-encodes an oversized photographic JPEG as JPEG', async () => {
|
||||
const data = await noiseImage(64, 32, 'jpeg')
|
||||
const detected = await detectImage(data)
|
||||
|
||||
const canonical = await canonicalizeImage(data, detected, { maxDimension: 32, maxBytes: POLICY.maxBytes })
|
||||
|
||||
expect(canonical).toMatchObject({ mediaType: 'image/jpeg', width: 32, height: 16 })
|
||||
})
|
||||
|
||||
it('falls from PNG to the JPEG ladder when palette PNG exceeds the byte target', async () => {
|
||||
// A smooth gradient: palette quantization dithers it into a sizable PNG
|
||||
// while JPEG at quality 85 stays far smaller, so the budget between the
|
||||
// two forces exactly one ladder hop.
|
||||
const side = 256
|
||||
const pixels = new Uint8Array(side * side * 3)
|
||||
for (let y = 0; y < side; y += 1) {
|
||||
for (let x = 0; x < side; x += 1) {
|
||||
const index = (y * side + x) * 3
|
||||
pixels[index] = x & 0xff
|
||||
pixels[index + 1] = y & 0xff
|
||||
pixels[index + 2] = (x + y) >> 1 & 0xff
|
||||
}
|
||||
}
|
||||
const data = new Uint8Array(await sharp(pixels, { raw: { width: side, height: side, channels: 3 } }).png().toBuffer())
|
||||
const detected = await detectImage(data)
|
||||
const paletteSize = (await sharp(data).png({ compressionLevel: 9, palette: true }).toBuffer()).byteLength
|
||||
const jpegSize = (await sharp(data).flatten({ background: '#ffffff' }).jpeg({ quality: 85 }).toBuffer()).byteLength
|
||||
expect(jpegSize).toBeLessThan(paletteSize)
|
||||
const budget = { maxDimension: 2048, maxBytes: paletteSize - 1 }
|
||||
|
||||
const canonical = await canonicalizeImage(data, detected, budget)
|
||||
|
||||
expect(canonical.mediaType).toBe('image/jpeg')
|
||||
expect(canonical.data.byteLength).toBeLessThanOrEqual(budget.maxBytes)
|
||||
})
|
||||
|
||||
it('refuses a source that no ladder step fits into the byte target', async () => {
|
||||
const data = await noiseImage(64, 64, 'png')
|
||||
|
||||
await expect(canonicalizeImage(data, await detectImage(data), { maxDimension: 2048, maxBytes: 10 }))
|
||||
.rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' })
|
||||
})
|
||||
|
||||
it('maps an encoder fault on undecodable bytes to a storage failure', async () => {
|
||||
await expect(canonicalizeImage(Uint8Array.of(1, 2, 3), { mediaType: 'image/png', width: 5000, height: 5000 }, POLICY))
|
||||
.rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED' })
|
||||
})
|
||||
})
|
||||
|
|
@ -5,6 +5,8 @@ import { tmpdir } from 'node:os'
|
|||
import { join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import LocalAttachmentStore, {
|
||||
DEFAULT_CANONICAL_MAX_BYTES,
|
||||
DEFAULT_CANONICAL_MAX_DIMENSION,
|
||||
DEFAULT_MAX_IMAGE_BYTES,
|
||||
DEFAULT_MAX_IMAGE_DIMENSION,
|
||||
DEFAULT_MAX_IMAGE_PIXELS,
|
||||
|
|
@ -15,7 +17,7 @@ import LocalAttachmentStore, {
|
|||
describe('local attachment service', () => {
|
||||
it('resolves every omitted admission limit explicitly', () => {
|
||||
const service = new LocalAttachmentStore(new Context(), {})
|
||||
expect(DEFAULT_MAX_IMAGE_BYTES).toBe(3.5 * 1024 * 1024)
|
||||
expect(DEFAULT_MAX_IMAGE_BYTES).toBe(32 * 1024 * 1024)
|
||||
expect(service.imageLimits).toEqual({
|
||||
maxImageBytes: DEFAULT_MAX_IMAGE_BYTES,
|
||||
maxImagesPerMessage: DEFAULT_MAX_IMAGES_PER_MESSAGE,
|
||||
|
|
@ -24,6 +26,10 @@ describe('local attachment service', () => {
|
|||
maxImageDimension: DEFAULT_MAX_IMAGE_DIMENSION,
|
||||
mediaTypes: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'],
|
||||
})
|
||||
expect(service.canonicalPolicy).toEqual({
|
||||
maxDimension: DEFAULT_CANONICAL_MAX_DIMENSION,
|
||||
maxBytes: DEFAULT_CANONICAL_MAX_BYTES,
|
||||
})
|
||||
})
|
||||
|
||||
it('saves and reads through the service boundary', async () => {
|
||||
|
|
@ -34,7 +40,7 @@ describe('local attachment service', () => {
|
|||
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=',
|
||||
'base64',
|
||||
))
|
||||
const ref = await service.saveImage({ data, mediaType: 'image/png' })
|
||||
const { ref } = await service.saveImage({ data, mediaType: 'image/png' })
|
||||
await expect(service.readImage(ref)).resolves.toEqual({ ref, data })
|
||||
} finally {
|
||||
await rm(dshHome, { recursive: true, force: true })
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import { mkdtemp, rm } from 'node:fs/promises'
|
|||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import sharp from 'sharp'
|
||||
import type { ImageAttachmentLimits } from '@deepseek-ai/dsh-attachment'
|
||||
import type { CanonicalImagePolicy } from '../src/canonical.ts'
|
||||
import { readImageFile, saveImageFile } from '../src/store.ts'
|
||||
|
||||
const fsControl = vi.hoisted(() => ({
|
||||
|
|
@ -38,6 +39,8 @@ const PNG = Uint8Array.from(Buffer.from(
|
|||
'base64',
|
||||
))
|
||||
|
||||
const POLICY: CanonicalImagePolicy = { maxDimension: 2048, maxBytes: 1024 * 1024 }
|
||||
|
||||
const LIMITS: ImageAttachmentLimits = {
|
||||
maxImageBytes: 1024,
|
||||
maxImagesPerMessage: 2,
|
||||
|
|
@ -79,7 +82,7 @@ describe('local attachment store', () => {
|
|||
const bucket = join(objects, sha256.slice(0, 2))
|
||||
fsControl.syncedDirectories.length = 0
|
||||
|
||||
await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)
|
||||
await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY)
|
||||
|
||||
// Each process first proves DSH_HOME durable all the way to the filesystem
|
||||
// root; existence alone cannot vouch for a concurrent creator's fsync.
|
||||
|
|
@ -104,7 +107,7 @@ describe('local attachment store', () => {
|
|||
it('creates and persists a missing nested home directory against the filesystem root', async () => {
|
||||
const storageRoot = join(await root(), 'home', 'attachments', 'v1')
|
||||
|
||||
const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)
|
||||
const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY)
|
||||
|
||||
await expect(readImageFile(storageRoot, ref)).resolves.toEqual({ ref, data: PNG })
|
||||
})
|
||||
|
|
@ -113,12 +116,12 @@ describe('local attachment store', () => {
|
|||
const storageRoot = await root()
|
||||
const first = await saveImageFile(storageRoot, {
|
||||
data: PNG, mediaType: 'image/png', name: '/private/tmp/pixel.png',
|
||||
}, LIMITS)
|
||||
const second = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)
|
||||
}, LIMITS, POLICY)
|
||||
const second = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY)
|
||||
const sha256 = createHash('sha256').update(PNG).digest('hex')
|
||||
const object = join(storageRoot, 'objects', sha256.slice(0, 2), sha256)
|
||||
|
||||
expect(first).toEqual({
|
||||
expect(first.ref).toEqual({
|
||||
attachmentId: `sha256:${sha256}`,
|
||||
mediaType: 'image/png',
|
||||
bytes: PNG.byteLength,
|
||||
|
|
@ -126,25 +129,44 @@ describe('local attachment store', () => {
|
|||
height: 1,
|
||||
name: 'pixel.png',
|
||||
})
|
||||
expect(second.attachmentId).toBe(first.attachmentId)
|
||||
expect(first.source).toEqual({ mediaType: 'image/png', bytes: PNG.byteLength, width: 1, height: 1 })
|
||||
expect(second.ref.attachmentId).toBe(first.ref.attachmentId)
|
||||
expect(new Uint8Array(await readFile(object))).toEqual(PNG)
|
||||
if (process.platform !== 'win32') {
|
||||
expect((await stat(object)).mode & 0o777).toBe(0o600)
|
||||
expect((await stat(join(storageRoot, 'objects', sha256.slice(0, 2)))).mode & 0o777).toBe(0o700)
|
||||
}
|
||||
await expect(readImageFile(storageRoot, first)).resolves.toEqual({ ref: first, data: PNG })
|
||||
await expect(readImageFile(storageRoot, first.ref)).resolves.toEqual({ ref: first.ref, data: PNG })
|
||||
})
|
||||
|
||||
it('stores the canonical encoding of an oversized source and reads it back verified', async () => {
|
||||
const storageRoot = await root()
|
||||
const oversized = new Uint8Array(await sharp({
|
||||
create: { width: 4, height: 4, channels: 3, background: { r: 9, g: 9, b: 9 } },
|
||||
}).png().toBuffer())
|
||||
|
||||
const saved = await saveImageFile(storageRoot, {
|
||||
data: oversized, mediaType: 'image/png', name: 'big.png',
|
||||
}, { ...LIMITS, maxImagePixels: 64 }, { maxDimension: 2, maxBytes: 1024 * 1024 })
|
||||
|
||||
expect(saved.source).toEqual({ mediaType: 'image/png', bytes: oversized.byteLength, width: 4, height: 4 })
|
||||
expect(saved.ref).toMatchObject({ mediaType: 'image/png', width: 2, height: 2, name: 'big.png' })
|
||||
expect(saved.ref.bytes).not.toBe(oversized.byteLength)
|
||||
const read = await readImageFile(storageRoot, saved.ref)
|
||||
expect(read.data.byteLength).toBe(saved.ref.bytes)
|
||||
expect(String(saved.ref.attachmentId)).toBe(`sha256:${createHash('sha256').update(read.data).digest('hex')}`)
|
||||
})
|
||||
|
||||
it('keeps admitted history readable after deployment limits become stricter', async () => {
|
||||
const storageRoot = await root()
|
||||
const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)
|
||||
const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY)
|
||||
|
||||
await expect(readImageFile(storageRoot, ref)).resolves.toEqual({ ref, data: PNG })
|
||||
})
|
||||
|
||||
it('forwards read cancellation to the filesystem and preserves its reason', async () => {
|
||||
const storageRoot = await root()
|
||||
const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)
|
||||
const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY)
|
||||
const controller = new AbortController()
|
||||
fsControl.readSignals.length = 0
|
||||
|
||||
|
|
@ -160,35 +182,35 @@ describe('local attachment store', () => {
|
|||
const storageRoot = await root()
|
||||
await expect(saveImageFile(storageRoot, {
|
||||
data: new Uint8Array(0), mediaType: 'image/png',
|
||||
}, LIMITS)).rejects.toMatchObject({ code: 'INVALID_IMAGE' })
|
||||
}, LIMITS, POLICY)).rejects.toMatchObject({ code: 'INVALID_IMAGE' })
|
||||
await expect(saveImageFile(storageRoot, {
|
||||
data: Uint8Array.of(1, 2, 3), mediaType: 'image/png',
|
||||
}, LIMITS)).rejects.toMatchObject({ code: 'INVALID_IMAGE' })
|
||||
}, LIMITS, POLICY)).rejects.toMatchObject({ code: 'INVALID_IMAGE' })
|
||||
await expect(saveImageFile(storageRoot, {
|
||||
data: PNG, mediaType: 'image/jpeg',
|
||||
}, LIMITS)).rejects.toMatchObject({ code: 'IMAGE_TYPE_MISMATCH' })
|
||||
}, LIMITS, POLICY)).rejects.toMatchObject({ code: 'IMAGE_TYPE_MISMATCH' })
|
||||
await expect(saveImageFile(storageRoot, {
|
||||
data: PNG, mediaType: 'image/png',
|
||||
}, { ...LIMITS, maxImageBytes: 1 })).rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' })
|
||||
}, { ...LIMITS, maxImageBytes: 1 }, POLICY)).rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' })
|
||||
|
||||
const wide = new Uint8Array(await sharp({
|
||||
create: { width: 5, height: 5, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 1 } },
|
||||
}).png().toBuffer())
|
||||
await expect(saveImageFile(storageRoot, {
|
||||
data: wide, mediaType: 'image/png',
|
||||
}, LIMITS)).rejects.toMatchObject({ code: 'IMAGE_TOO_MANY_PIXELS' })
|
||||
}, LIMITS, POLICY)).rejects.toMatchObject({ code: 'IMAGE_TOO_MANY_PIXELS' })
|
||||
await expect(saveImageFile(storageRoot, {
|
||||
data: wide, mediaType: 'image/png',
|
||||
}, { ...LIMITS, maxImagePixels: 25, maxImageDimension: 4 })).rejects.toMatchObject({ code: 'IMAGE_DIMENSION_TOO_LARGE' })
|
||||
}, { ...LIMITS, maxImagePixels: 25, maxImageDimension: 4 }, POLICY)).rejects.toMatchObject({ code: 'IMAGE_DIMENSION_TOO_LARGE' })
|
||||
const unnamed = await saveImageFile(storageRoot, {
|
||||
data: PNG, mediaType: 'image/png', name: '\u0000',
|
||||
}, LIMITS)
|
||||
expect(unnamed).not.toHaveProperty('name')
|
||||
}, LIMITS, POLICY)
|
||||
expect(unnamed.ref).not.toHaveProperty('name')
|
||||
})
|
||||
|
||||
it('fails closed when an object is missing, corrupted, or addressed by an invalid reference', async () => {
|
||||
const storageRoot = await root()
|
||||
const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)
|
||||
const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY)
|
||||
const sha256 = String(ref.attachmentId).slice('sha256:'.length)
|
||||
const object = join(storageRoot, 'objects', sha256.slice(0, 2), sha256)
|
||||
await chmod(object, 0o600)
|
||||
|
|
@ -216,11 +238,11 @@ describe('local attachment store', () => {
|
|||
const target = join(storageRoot, 'objects', sha256.slice(0, 2), sha256)
|
||||
await mkdir(join(storageRoot, 'objects', sha256.slice(0, 2)), { recursive: true })
|
||||
await writeFile(target, Uint8Array.of(1, 2, 3))
|
||||
await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS))
|
||||
await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY))
|
||||
.rejects.toMatchObject({ code: 'ATTACHMENT_CORRUPT' })
|
||||
|
||||
await writeFile(target, PNG)
|
||||
const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)
|
||||
const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY)
|
||||
await expect(readImageFile(storageRoot, { ...ref, width: ref.width + 1 }))
|
||||
.rejects.toMatchObject({ code: 'ATTACHMENT_CORRUPT' })
|
||||
})
|
||||
|
|
@ -231,7 +253,7 @@ describe('local attachment store', () => {
|
|||
const target = join(storageRoot, 'objects', sha256.slice(0, 2), sha256)
|
||||
await mkdir(target, { recursive: true })
|
||||
|
||||
await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS))
|
||||
await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY))
|
||||
.rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED' })
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue