diff --git a/.changeset/r2-local-public-fixes.md b/.changeset/r2-local-public-fixes.md new file mode 100644 index 0000000000..ae49144bbc --- /dev/null +++ b/.changeset/r2-local-public-fixes.md @@ -0,0 +1,5 @@ +--- +"miniflare": patch +--- + +Fix edge cases on the local R2 public bucket endpoint (`/cdn-cgi/local/r2/public`) to match r2.dev: write methods are rejected with 401, malformed/multiple/inverted ranges with 400 and unsatisfiable ranges (including `bytes=-0`) with 416, `Range` is honored on HEAD requests with a bodyless 206, `Content-Range` is correct for suffix ranges, and object keys are percent-decoded exactly once (keys containing a literal `%` no longer fail). Unread object bodies are also cancelled (on HEAD and unsatisfiable-range responses) instead of leaking a read stream until garbage collection. diff --git a/.changeset/r2-local-s3-endpoint.md b/.changeset/r2-local-s3-endpoint.md new file mode 100644 index 0000000000..0bf012e24e --- /dev/null +++ b/.changeset/r2-local-s3-endpoint.md @@ -0,0 +1,7 @@ +--- +"miniflare": minor +--- + +Add a local S3-compatible API for R2 buckets at `/cdn-cgi/local/r2/s3/` + +Buckets configured with `s3Credentials: { accessKeyId, secretAccessKey }` in `r2Buckets` are served over an S3-compatible HTTP API, authenticated with AWS Signature Version 4 (both `Authorization` header and presigned URL query authentication). Supported operations: GetObject, HeadObject, PutObject, CopyObject, DeleteObject, DeleteObjects, ListObjects, ListObjectsV2, HeadBucket, ListBuckets, CreateMultipartUpload, UploadPart, UploadPartCopy, CompleteMultipartUpload, and AbortMultipartUpload. Status codes, error responses, and unsupported-header screening mirror R2's S3 endpoint, including its static responses for bucket-configuration reads and its named errors for unimplemented operations. diff --git a/.changeset/r2-local-s3-wrangler.md b/.changeset/r2-local-s3-wrangler.md new file mode 100644 index 0000000000..57f64a04ff --- /dev/null +++ b/.changeset/r2-local-s3-wrangler.md @@ -0,0 +1,22 @@ +--- +"wrangler": minor +--- + +Add experimental `experimental_local_s3_credentials` to `r2_buckets` config + +When set, the R2 bucket is served over a local S3-compatible API at `/cdn-cgi/local/r2/s3/` during local development, authenticated with the configured AWS SigV4 credentials: + +```jsonc +{ + "r2_buckets": [ + { + "binding": "BUCKET", + "bucket_name": "my-bucket", + "experimental_local_s3_credentials": { + "accessKeyId": "local-access-key-id", + "secretAccessKey": "local-secret-access-key", + }, + }, + ], +} +``` diff --git a/packages/miniflare/package.json b/packages/miniflare/package.json index 0825350499..8e1da9221f 100644 --- a/packages/miniflare/package.json +++ b/packages/miniflare/package.json @@ -57,6 +57,9 @@ "youch": "4.1.0-beta.10" }, "devDependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-sdk/client-s3": "^3.721.0", + "@aws-sdk/s3-request-presigner": "3.721.0", "@cloudflare/cli-shared-helpers": "workspace:*", "@cloudflare/containers-shared": "workspace:*", "@cloudflare/kv-asset-handler": "workspace:*", @@ -68,6 +71,7 @@ "@hey-api/openapi-ts": "catalog:default", "@microsoft/api-extractor": "^7.52.8", "@puppeteer/browsers": "^2.10.6", + "@smithy/signature-v4": "^4.2.4", "@types/debug": "^4.1.7", "@types/estree": "^1.0.0", "@types/glob-to-regexp": "^0.4.1", @@ -89,6 +93,7 @@ "devtools-protocol": "^0.0.1182435", "esbuild": "catalog:default", "expect-type": "^0.15.0", + "fast-xml-parser": "^4.4.1", "get-port": "^7.1.0", "glob-to-regexp": "0.4.1", "hono": "^4.12.5", diff --git a/packages/miniflare/src/plugins/core/index.ts b/packages/miniflare/src/plugins/core/index.ts index 56ec132245..5a3e2cd86c 100644 --- a/packages/miniflare/src/plugins/core/index.ts +++ b/packages/miniflare/src/plugins/core/index.ts @@ -27,7 +27,12 @@ import { RPC_PROXY_SERVICE_NAME } from "../assets/constants"; import { getCacheServiceName } from "../cache"; import { DURABLE_OBJECTS_STORAGE_SERVICE_NAME } from "../do"; import { IMAGES_PLUGIN_NAME } from "../images"; -import { getR2PublicService, R2_PUBLIC_SERVICE_NAME } from "../r2"; +import { + getR2PublicService, + getR2S3Service, + R2_PUBLIC_SERVICE_NAME, + R2_S3_SERVICE_NAME, +} from "../r2"; import { getUserBindingServiceName, kUnsafeEphemeralUniqueKey, @@ -1105,6 +1110,13 @@ export function getGlobalServices({ service: { name: R2_PUBLIC_SERVICE_NAME }, }); } + const r2S3Service = getR2S3Service(allWorkerOpts ?? []); + if (r2S3Service !== undefined) { + serviceEntryBindings.push({ + name: CoreBindings.SERVICE_R2_S3, + service: { name: R2_S3_SERVICE_NAME }, + }); + } const imagesBinding = allWorkerOpts ?.map((worker) => worker.images?.images) .find( @@ -1191,6 +1203,9 @@ export function getGlobalServices({ if (r2PublicService !== undefined) { services.push(r2PublicService); } + if (r2S3Service !== undefined) { + services.push(r2S3Service); + } if (sharedOptions.unsafeLocalExplorer) { const localExplorerUiPath = resolveLocalExplorerUi(tmpPath); diff --git a/packages/miniflare/src/plugins/r2/index.ts b/packages/miniflare/src/plugins/r2/index.ts index bb1f2d5ddb..63b373abb4 100644 --- a/packages/miniflare/src/plugins/r2/index.ts +++ b/packages/miniflare/src/plugins/r2/index.ts @@ -1,8 +1,11 @@ import fs from "node:fs/promises"; import SCRIPT_R2_BUCKET_OBJECT from "worker:r2/bucket"; import SCRIPT_R2_PUBLIC from "worker:r2/public"; +import SCRIPT_R2_S3 from "worker:r2/s3/index"; import { z } from "zod"; +import { MiniflareCoreError } from "../../shared"; import { SharedBindings } from "../../workers"; +import { R2S3Bindings } from "../../workers/r2/constants"; import { getMiniflareObjectBindings, getPersistPath, @@ -21,8 +24,16 @@ import type { Worker_Binding, Worker_Binding_DurableObjectNamespaceDesignator, } from "../../runtime"; +import type { S3Credentials } from "../../workers/r2/constants"; import type { Plugin, RemoteProxyConnectionString } from "../shared"; +export const R2S3CredentialsSchema = z.object({ + accessKeyId: z.string(), + secretAccessKey: z.string(), +}) satisfies z.ZodType; + +export type R2S3Credentials = z.infer; + export const R2OptionsSchema = z.object({ r2Buckets: z .union([ @@ -34,6 +45,7 @@ export const R2OptionsSchema = z.object({ remoteProxyConnectionString: z .custom() .optional(), + s3Credentials: R2S3CredentialsSchema.optional(), }), ]) ), @@ -49,18 +61,27 @@ export const R2_PLUGIN_NAME = "r2"; const R2_STORAGE_SERVICE_NAME = `${R2_PLUGIN_NAME}:storage`; const R2_BUCKET_SERVICE_PREFIX = `${R2_PLUGIN_NAME}:bucket`; export const R2_PUBLIC_SERVICE_NAME = `${R2_PLUGIN_NAME}:public`; +export const R2_S3_SERVICE_NAME = `${R2_PLUGIN_NAME}:s3`; const R2_BUCKET_OBJECT_CLASS_NAME = "R2BucketObject"; const R2_BUCKET_OBJECT: Worker_Binding_DurableObjectNamespaceDesignator = { serviceName: R2_BUCKET_SERVICE_PREFIX, className: R2_BUCKET_OBJECT_CLASS_NAME, }; +interface R2BucketEntry { + id: string; + remoteProxyConnectionString?: RemoteProxyConnectionString; + s3Credentials?: R2S3Credentials; +} + export function getR2PublicService( allWorkerOpts: { r2?: z.infer }[] ): Service | undefined { const publicBucketIds = new Set(); for (const worker of allWorkerOpts) { - for (const [, bucket] of namespaceEntries(worker.r2?.r2Buckets)) { + for (const [, bucket] of namespaceEntries( + worker.r2?.r2Buckets + )) { if (bucket.remoteProxyConnectionString !== undefined) { continue; } @@ -86,6 +107,67 @@ export function getR2PublicService( }; } +export function getR2S3Service( + allWorkerOpts: { r2?: z.infer }[] +): Service | undefined { + const credentialsById: Record< + string, + z.infer + > = {}; + for (const worker of allWorkerOpts) { + for (const [, bucket] of namespaceEntries( + worker.r2?.r2Buckets + )) { + if ( + bucket.remoteProxyConnectionString !== undefined || + bucket.s3Credentials === undefined + ) { + continue; + } + + const existing = credentialsById[bucket.id]; + if ( + existing !== undefined && + (existing.accessKeyId !== bucket.s3Credentials.accessKeyId || + existing.secretAccessKey !== bucket.s3Credentials.secretAccessKey) + ) { + throw new MiniflareCoreError( + "ERR_DIFFERENT_S3_CREDENTIALS", + `Bucket "${bucket.id}" is bound by multiple Workers with different S3 credentials` + ); + } + + credentialsById[bucket.id] = bucket.s3Credentials; + } + } + + const bucketIds = Object.keys(credentialsById); + if (bucketIds.length === 0) { + return undefined; + } + + const bindings = bucketIds.map((id) => ({ + name: `${R2S3Bindings.BUCKET_PREFIX}${id}`, + r2Bucket: { + name: getUserBindingServiceName(R2_BUCKET_SERVICE_PREFIX, id), + }, + })); + bindings.push({ + name: R2S3Bindings.JSON_CREDENTIALS, + json: JSON.stringify(credentialsById), + }); + + return { + name: R2_S3_SERVICE_NAME, + worker: { + compatibilityDate: "2026-01-01", + compatibilityFlags: ["nodejs_compat"], + modules: [{ name: "s3.worker.js", esModule: SCRIPT_R2_S3() }], + bindings, + }, + }; +} + export const R2_PLUGIN: Plugin< typeof R2OptionsSchema, typeof R2SharedOptionsSchema @@ -93,7 +175,7 @@ export const R2_PLUGIN: Plugin< options: R2OptionsSchema, sharedOptions: R2SharedOptionsSchema, getBindings(options) { - const buckets = namespaceEntries(options.r2Buckets); + const buckets = namespaceEntries(options.r2Buckets); return buckets.map(([name, bucket]) => ({ name, r2Bucket: { @@ -120,7 +202,7 @@ export const R2_PLUGIN: Plugin< unsafeStickyBlobs, }) { const persist = sharedOptions.r2Persist; - const buckets = namespaceEntries(options.r2Buckets); + const buckets = namespaceEntries(options.r2Buckets); const services = buckets.map( ([name, { id, remoteProxyConnectionString }]) => ({ name: getUserBindingServiceName( diff --git a/packages/miniflare/src/plugins/shared/index.ts b/packages/miniflare/src/plugins/shared/index.ts index 0ba71c3b62..0e8ef4e7ba 100644 --- a/packages/miniflare/src/plugins/shared/index.ts +++ b/packages/miniflare/src/plugins/shared/index.ts @@ -181,35 +181,25 @@ export type RemoteProxyConnectionString = URL & { __brand: "RemoteProxyConnectionString"; }; -export function namespaceEntries( - namespaces?: - | Record< - string, - | string - | { - id: string; - remoteProxyConnectionString?: RemoteProxyConnectionString; - } - > - | string[] -): [ - bindingName: string, - { id: string; remoteProxyConnectionString?: RemoteProxyConnectionString }, -][] { +export function namespaceEntries< + Entry extends { + id: string; + remoteProxyConnectionString?: RemoteProxyConnectionString; + } = { id: string; remoteProxyConnectionString?: RemoteProxyConnectionString }, +>( + namespaces?: Record | string[] +): [bindingName: string, entry: Entry][] { if (Array.isArray(namespaces)) { - return namespaces.map((bindingName) => [bindingName, { id: bindingName }]); + return namespaces.map((bindingName) => [ + bindingName, + { id: bindingName } as Entry, + ]); } else if (namespaces !== undefined) { return Object.entries(namespaces).map(([key, value]) => { if (typeof value === "string") { - return [key, { id: value }]; + return [key, { id: value } as Entry]; } - return [ - key, - { - id: value.id, - remoteProxyConnectionString: value.remoteProxyConnectionString, - }, - ]; + return [key, value]; }); } else { return []; diff --git a/packages/miniflare/src/shared/error.ts b/packages/miniflare/src/shared/error.ts index add9fc4fcf..4c6be44524 100644 --- a/packages/miniflare/src/shared/error.ts +++ b/packages/miniflare/src/shared/error.ts @@ -13,6 +13,7 @@ export const USER_ERROR_CODES = new Set([ "ERR_DIFFERENT_STORAGE_BACKEND", // Multiple Durable Object bindings declared for same class with different storage backends "ERR_DIFFERENT_UNIQUE_KEYS", // Multiple Durable Object bindings declared for same class with different unsafe unique keys "ERR_DIFFERENT_PREVENT_EVICTION", // Multiple Durable Object bindings declared for same class with different unsafe prevent eviction values + "ERR_DIFFERENT_S3_CREDENTIALS", // Multiple R2 bucket bindings declared for same bucket with different S3 credentials "ERR_MULTIPLE_OUTBOUNDS", // Both `outboundService` and `fetchMock` specified "ERR_INVALID_WRAPPED", // Worker not allowed to be used as wrapped binding "ERR_MISSING_INSPECTOR_PROXY_PORT", // An inspector proxy has been requested but no inspector port to use has been specified diff --git a/packages/miniflare/src/workers/core/constants.ts b/packages/miniflare/src/workers/core/constants.ts index de18f19e75..579f7a3fe5 100644 --- a/packages/miniflare/src/workers/core/constants.ts +++ b/packages/miniflare/src/workers/core/constants.ts @@ -23,6 +23,8 @@ export const CorePaths = { IMAGE_DELIVERY: "/cdn-cgi/mf/imagedelivery", /** Public R2 bucket object serving endpoint */ R2_PUBLIC: "/cdn-cgi/local/r2/public", + /** S3-compatible API endpoint for local R2 buckets */ + R2_S3: "/cdn-cgi/local/r2/s3", } as const; export const CoreHeaders = { @@ -84,6 +86,7 @@ export const CoreBindings = { SERVICE_STREAM: "MINIFLARE_STREAM", SERVICE_IMAGES_DELIVERY: "MINIFLARE_IMAGES_DELIVERY", SERVICE_R2_PUBLIC: "MINIFLARE_R2_PUBLIC", + SERVICE_R2_S3: "MINIFLARE_R2_S3", } as const; export const ProxyOps = { diff --git a/packages/miniflare/src/workers/core/entry.worker.ts b/packages/miniflare/src/workers/core/entry.worker.ts index 013de035eb..a4a778450a 100644 --- a/packages/miniflare/src/workers/core/entry.worker.ts +++ b/packages/miniflare/src/workers/core/entry.worker.ts @@ -16,6 +16,7 @@ type Env = { [CoreBindings.SERVICE_STREAM]?: Fetcher; [CoreBindings.SERVICE_IMAGES_DELIVERY]?: Fetcher; [CoreBindings.SERVICE_R2_PUBLIC]?: Fetcher; + [CoreBindings.SERVICE_R2_S3]?: Fetcher; [CoreBindings.TEXT_CUSTOM_SERVICE]: string; [CoreBindings.TEXT_UPSTREAM_URL]?: string; [CoreBindings.JSON_CF_BLOB]: IncomingRequestCfProperties; @@ -615,6 +616,28 @@ export default >{ return await r2PublicService.fetch(request); } + const r2S3Service = env[CoreBindings.SERVICE_R2_S3]; + if ( + (url.pathname === CorePaths.R2_S3 || + url.pathname.startsWith(`${CorePaths.R2_S3}/`)) && + r2S3Service + ) { + // SigV4 verification compares against the host the client + // signed, so undo the `upstream` URL/Host rewrite from + // `getUserRequest()` + let s3Request = request; + const originalHostname = request.headers.get( + CoreHeaders.ORIGINAL_HOSTNAME + ); + if (originalHostname !== null) { + const s3Url = new URL(url); + s3Url.host = originalHostname; + s3Request = new Request(s3Url, request); + s3Request.headers.set("Host", originalHostname); + } + return await r2S3Service.fetch(s3Request); + } + let response = await service.fetch(request); if (!disablePrettyErrorPage) { response = await maybePrettifyError(request, response, env); diff --git a/packages/miniflare/src/workers/r2/constants.ts b/packages/miniflare/src/workers/r2/constants.ts index 57f07ca04c..151cbeab6f 100644 --- a/packages/miniflare/src/workers/r2/constants.ts +++ b/packages/miniflare/src/workers/r2/constants.ts @@ -8,6 +8,19 @@ export const R2Limits = { MIN_MULTIPART_PART_SIZE_TEST: 50, } as const; +/** AWS SigV4 credentials guarding a bucket on the local S3 endpoint */ +export interface S3Credentials { + accessKeyId: string; + secretAccessKey: string; +} + +export const R2S3Bindings = { + /** JSON map of bucket id to `{ accessKeyId, secretAccessKey }` */ + JSON_CREDENTIALS: "MINIFLARE_R2_S3_CREDENTIALS", + /** Prefix for per-bucket `R2Bucket` bindings (followed by the bucket id) */ + BUCKET_PREFIX: "MINIFLARE_R2_S3_BUCKET_", +} as const; + export const R2Headers = { ERROR: "cf-r2-error", REQUEST: "cf-r2-request", diff --git a/packages/miniflare/src/workers/r2/public.worker.ts b/packages/miniflare/src/workers/r2/public.worker.ts index 39d50c6bda..48dfad93c7 100644 --- a/packages/miniflare/src/workers/r2/public.worker.ts +++ b/packages/miniflare/src/workers/r2/public.worker.ts @@ -1,18 +1,10 @@ import { cors } from "hono/cors"; import { Hono } from "hono/tiny"; import { CorePaths } from "../core/constants"; +import { parseRangeHeader, serveR2Object } from "./serve.worker"; type Env = Record; -function objectHeaders(object: R2Object): Headers { - const headers = new Headers(); - object.writeHttpMetadata(headers); - headers.set("ETag", object.httpEtag); - headers.set("Last-Modified", object.uploaded.toUTCString()); - headers.set("Accept-Ranges", "bytes"); - return headers; -} - const app = new Hono<{ Bindings: Env }>().basePath(CorePaths.R2_PUBLIC); app.use( @@ -20,91 +12,36 @@ app.use( ); app.on(["GET", "HEAD"], "/:bucketId/:key{.+}", async (c) => { - const bucketId = decodeURIComponent(c.req.param("bucketId")); - const key = decodeURIComponent(c.req.param("key")); + const bucketId = c.req.param("bucketId"); + const key = c.req.param("key"); const bucket = c.env[bucketId]; if (bucket === undefined) { return c.notFound(); } - const hasRange = c.req.header("Range") !== undefined; - // `bucket.head()` cannot evaluate conditional headers (the R2 head - // operation only carries the key), so HEAD also uses `bucket.get()` and - // discards the body. - const object = await bucket.get(key, { - onlyIf: c.req.raw.headers, - range: hasRange && c.req.method === "GET" ? c.req.raw.headers : undefined, - }); - - if (object === null) { - return c.notFound(); - } - - const headers = objectHeaders(object); - - if (!("body" in object)) { - // Some conditional header failed, but `bucket.get()` reports the - // failure without naming the header. We need to determine which header - // failed to determine the status code to return. - // - // https://datatracker.ietf.org/doc/html/rfc7232#section-6 gives the - // order for checking headers. We know at least one header failed. - // We must first check for a precondition header failure. - // - // The logic in `_testR2Conditional` ensures we can simultaneously - // check both "If-Match" and "If-Unmodified-Since" (since a failure in - // "If-Unmodified-Since" can be suppressed by success for a present - // "If-Match"). These both yield status 412s upon failure. - let preconditions: Headers | undefined; - for (const name of ["If-Match", "If-Unmodified-Since"]) { - const value = c.req.raw.headers.get(name); - if (value !== null) { - preconditions ??= new Headers(); - preconditions.set(name, value); - } - } - if (preconditions !== undefined) { - const recheck = await bucket.get(key, { onlyIf: preconditions }); - if (recheck === null) { - return c.notFound(); - } - if (!("body" in recheck)) { - return c.body(null, { status: 412, headers: objectHeaders(recheck) }); - } - } - - // Otherwise, the preconditions hold, so the failure came from a cache validator. - return c.body(null, { status: 304, headers }); - } - - if (c.req.method === "HEAD") { - headers.set("Content-Length", `${object.size}`); - return c.body(null, { headers }); + // Reject malformed, multiple, and inverted ranges with 400 rather than + // ignoring them, and zero-length suffix ranges (`bytes=-0`) with 416 + const rangeHeader = c.req.header("Range"); + const parsedRange = + rangeHeader === undefined ? undefined : parseRangeHeader(rangeHeader); + if (parsedRange !== undefined && "error" in parsedRange) { + return c.body(null, parsedRange.error === "unsatisfiable" ? 416 : 400); } - const range = object.range; - if ( - hasRange && - range !== undefined && - "offset" in range && - "length" in range - ) { - const { offset = 0, length = object.size - offset } = range; - headers.set( - "Content-Range", - `bytes ${offset}-${offset + length - 1}/${object.size}` - ); - headers.set("Content-Length", `${length}`); - return c.body(object.body, { status: 206, headers }); - } - - headers.set("Content-Length", `${object.size}`); - return c.body(object.body, { headers }); + return serveR2Object( + c.req.raw, + bucket, + key, + { + notFound: () => c.notFound(), + preconditionFailed: () => c.body(null, 412), + invalidRange: () => c.body(null, 416), + }, + parsedRange + ); }); -app.all("/:bucketId/:key{.+}", (c) => - c.text("Method Not Allowed", 405, { Allow: "GET, HEAD, OPTIONS" }) -); +app.all("/:bucketId/:key{.+}", (c) => c.body(null, 401)); export default app; diff --git a/packages/miniflare/src/workers/r2/s3/account.worker.ts b/packages/miniflare/src/workers/r2/s3/account.worker.ts new file mode 100644 index 0000000000..336010e8cf --- /dev/null +++ b/packages/miniflare/src/workers/r2/s3/account.worker.ts @@ -0,0 +1,87 @@ +// The S3 API's one account-level operation: ListBuckets. There is no bucket +// in the path to resolve credentials from, so authentication works +// differently from the per-bucket operations in `dispatch.worker.ts`. +import assert from "node:assert"; +import { R2S3Bindings } from "../constants"; +import { credentialsEqual, verifyRequest } from "./auth.worker"; +import { stripBodyForHead, xmlResponse } from "./common.worker"; +import { isScreenedParam } from "./detect.worker"; +import { notImplemented, routeNotFound } from "./errors.worker"; +import type { S3Credentials } from "../constants"; +import type { S3Context } from "./common.worker"; + +/** + * Verifies the request against every configured credential set, returning + * the matching credentials, or the auth error if none verify. The request + * can only have been signed with one set, so the first success is the match. + */ +async function verifyAgainstSome( + c: S3Context +): Promise<{ matched: S3Credentials } | { error: Response }> { + let error: Response | undefined; + const seenPairs = new Set(); + for (const credentials of Object.values( + c.env[R2S3Bindings.JSON_CREDENTIALS] + )) { + // Buckets often share a credential pair; verify each pair once + const pair = `${credentials.accessKeyId}\0${credentials.secretAccessKey}`; + if (seenPairs.has(pair)) { + continue; + } + seenPairs.add(pair); + + const result = await verifyRequest(c.req.raw, credentials); + if (result === undefined) { + return { matched: credentials }; + } + + // A 401 just means this set's access key id didn't match; an error + // from a set whose key id did match (e.g. SignatureDoesNotMatch) is + // the precise one + if (error === undefined || error.status === 401) { + error = result; + } + } + + // The service only exists when at least one credential set is configured + assert(error !== undefined); + return { error }; +} + +/** + * ListBuckets is account-level: there is no bucket to resolve credentials + * from, so try each configured credential set; the request can only have + * been signed with one of them. Lists the buckets the matching credentials + * grant access to (locally, all buckets sharing that credential pair). + */ +export async function listBuckets(c: S3Context): Promise { + return stripBodyForHead(c, await listBucketsInner(c)); +} + +async function listBucketsInner(c: S3Context): Promise { + if (c.req.method !== "GET") { + return routeNotFound(); + } + + const credentialsById = c.env[R2S3Bindings.JSON_CREDENTIALS]; + const verified = await verifyAgainstSome(c); + if ("error" in verified) { + return verified.error; + } + const matched = verified.matched; + + for (const name of new URL(c.req.url).searchParams.keys()) { + if (!isScreenedParam(name)) { + return notImplemented( + `ListBuckets search parameter ${name} not implemented` + ); + } + } + + const buckets = Object.entries(credentialsById) + .filter(([, credentials]) => credentialsEqual(credentials, matched)) + .map(([id]) => ({ Name: id })); + return xmlResponse("ListAllMyBucketsResult", { + Buckets: buckets.length > 0 ? { Bucket: buckets } : {}, + }); +} diff --git a/packages/miniflare/src/workers/r2/s3/auth.worker.ts b/packages/miniflare/src/workers/r2/s3/auth.worker.ts new file mode 100644 index 0000000000..f025c5d949 --- /dev/null +++ b/packages/miniflare/src/workers/r2/s3/auth.worker.ts @@ -0,0 +1,569 @@ +// AWS Signature Version 4 verification for the local S3-compatible endpoint. +// Implements both authentication methods defined by the SigV4 spec: +// - HTTP `Authorization` header: +// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-authentication-methods.html#aws-signing-authentication-methods-http +// - Presigned URL query parameters: +// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-authentication-methods.html#aws-signing-authentication-methods-query +// Canonical request construction follows +// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html +// with the S3-specific deviation that the canonical URI is the request path +// used verbatim (single-encoded), not re-encoded. +// +// Error codes, messages, and check order mimic R2's S3 endpoint +// (.r2.cloudflarestorage.com), which differs from AWS S3: R2 uses +// `InvalidArgument`/`InvalidRequest` for malformed auth, 401 `Unauthorized` +// for unknown access keys (never `InvalidAccessKeyId`), 403 +// `SignatureDoesNotMatch` (with the canonical request and string-to-sign +// echoed for debugging, like AWS) for signature mismatches, and 403 +// `ExpiredRequest` for expired presigned URLs. + +import assert from "node:assert"; +import { hex } from "./common.worker"; +import { errorResponse } from "./errors.worker"; +import type { S3Credentials } from "../constants"; + +const ALGORITHM = "AWS4-HMAC-SHA256"; +const MAX_EXPIRES_SECONDS = 604_800; +const MAX_SKEW_MILLIS = 15 * 60 * 1000; + +const encoder = new TextEncoder(); + +async function sha256Hex(data: BufferSource): Promise { + return hex(await crypto.subtle.digest("SHA-256", data)); +} + +async function hmac(key: BufferSource, data: string): Promise { + const cryptoKey = await crypto.subtle.importKey( + "raw", + key, + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign"] + ); + return crypto.subtle.sign("HMAC", cryptoKey, encoder.encode(data)); +} + +/** + * AWS `UriEncode`: percent-encode everything except RFC 3986 unreserved + * characters, with uppercase hex. `encodeURIComponent` matches except it + * leaves `!'()*` unescaped. Used for query parameters only; the S3 + * canonical URI is the request path verbatim. + */ +export function awsUriEncode(value: string): string { + return encodeURIComponent(value).replace( + /[!'()*]/g, + (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}` + ); +} + +const invalidArgument = (message: string) => + errorResponse(400, "InvalidArgument", message); + +const unauthorized = () => errorResponse(401, "Unauthorized", "Unauthorized"); + +function byteDump(value: string): string { + return Array.from(encoder.encode(value), (byte) => + byte.toString(16).padStart(2, "0") + ).join(" "); +} + +function signatureDoesNotMatch( + computed: ComputedSignature, + provided: string +): Response { + return errorResponse( + 403, + "SignatureDoesNotMatch", + "The request signature we calculated does not match the signature you provided. Check your secret access key and signing method.", + { + StringToSign: computed.stringToSign, + StringToSignBytes: byteDump(computed.stringToSign), + CanonicalRequest: computed.canonicalRequest, + CanonicalRequestBytes: byteDump(computed.canonicalRequest), + SignatureProvided: provided, + } + ); +} + +const unsupportedAlgorithm = () => + errorResponse(400, "InvalidRequest", `Please use ${ALGORITHM}`); + +interface ParsedCredential { + accessKeyId: string; + date: string; + region: string; + service: string; +} + +/** + * Parses `////aws4_request`, + * validating in R2's order: part count, then service, then termination + * string. Access key validity is checked by the caller (R2 checks the key's + * length here, but local credentials are user-configured so any mismatch is + * just an invalid key, reported as 401 after the scope checks). + */ +function parseCredential( + credential: string +): ParsedCredential | { error: Response } { + const parts = credential.split("/"); + if (parts.length < 5) { + return { + error: invalidArgument( + `Credential sigv4 header should have at least 5 slash-separated parts, not ${parts.length}` + ), + }; + } + + // R2 allows extra slashes by treating everything before the last four + // parts as the access key + const accessKeyId = parts.slice(0, -4).join("/"); + const [date, region, service, terminator] = parts.slice(-4); + if (service !== "s3") { + return { + error: invalidArgument(`Credential service should be s3, not ${service}`), + }; + } + if (terminator !== "aws4_request") { + return { + error: invalidArgument( + `Credential termination string should be aws4_request, not ${terminator}` + ), + }; + } + if (date === undefined || region === undefined) { + return { error: unauthorized() }; + } + + return { accessKeyId, date, region, service }; +} + +/** Dates must strictly be basic ISO 8601 format */ +function parseAmzDate(value: string): Date | undefined { + const match = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(value); + if (match === null) { + return undefined; + } + + const [, year, month, day, hour, minute, second] = match; + const date = new Date( + Date.UTC( + Number(year), + Number(month) - 1, + Number(day), + Number(hour), + Number(minute), + Number(second) + ) + ); + + return Number.isNaN(date.getTime()) ? undefined : date; +} + +/** + * Canonical query string: each name and value `UriEncode`d, sorted by + * encoded name (then value), joined with `&`. For presigned requests, + * `X-Amz-Signature` is excluded. + */ +function canonicalQueryString(url: URL, excludeSignature: boolean): string { + const params: [string, string][] = []; + for (const [name, value] of url.searchParams) { + if (excludeSignature && name === "X-Amz-Signature") { + continue; + } + params.push([awsUriEncode(name), awsUriEncode(value)]); + } + params.sort(([name1, value1], [name2, value2]) => { + if (name1 !== name2) { + return name1 < name2 ? -1 : 1; + } + return value1 < value2 ? -1 : value1 > value2 ? 1 : 0; + }); + return params.map(([name, value]) => `${name}=${value}`).join("&"); +} + +/** + * Canonical headers: lowercase name, `:`, trimmed value with sequential + * spaces collapsed, `\n`, for each signed header in the order given + * (the spec requires the SignedHeaders list to already be sorted). + * + * Known limitation: `Headers.get()` joins repeated headers with `", "`, + * while SigV4 canonicalizes them joined with a bare comma. The Fetch API + * offers no way to recover the original values, so signing a repeated + * header produces a signature mismatch. + */ +function canonicalHeaders( + request: Request, + url: URL, + signedHeaders: string[] +): string { + let result = ""; + for (const name of signedHeaders) { + const value = + request.headers.get(name) ?? (name === "host" ? url.host : ""); + result += `${name}:${value.trim().replace(/ +/g, " ")}\n`; + } + return result; +} + +interface ComputedSignature { + signature: string; + canonicalRequest: string; + stringToSign: string; +} + +async function computeSignature( + request: Request, + url: URL, + secretAccessKey: string, + credential: ParsedCredential, + amzDate: string, + signedHeaders: string[], + payloadHash: string, + presigned: boolean +): Promise { + const canonicalRequest = [ + request.method, + url.pathname, + canonicalQueryString(url, presigned), + canonicalHeaders(request, url, signedHeaders), + signedHeaders.join(";"), + payloadHash, + ].join("\n"); + + const scope = `${credential.date}/${credential.region}/${credential.service}/aws4_request`; + const stringToSign = [ + ALGORITHM, + amzDate, + scope, + await sha256Hex(encoder.encode(canonicalRequest)), + ].join("\n"); + + // DateKey = HMAC("AWS4" + secret, date); DateRegionKey = HMAC(DateKey, + // region); DateRegionServiceKey = HMAC(.., service); SigningKey = + // HMAC(.., "aws4_request") + let key = await hmac( + encoder.encode(`AWS4${secretAccessKey}`), + credential.date + ); + key = await hmac(key, credential.region); + key = await hmac(key, credential.service); + key = await hmac(key, "aws4_request"); + + return { + signature: hex(await hmac(key, stringToSign)), + canonicalRequest, + stringToSign, + }; +} + +function timingSafeStringsEqual(expected: string, actual: string): boolean { + const expectedBytes = encoder.encode(expected); + const actualBytes = encoder.encode(actual); + + return actualBytes.byteLength === expectedBytes.byteLength + ? crypto.subtle.timingSafeEqual(expectedBytes, actualBytes) + : !crypto.subtle.timingSafeEqual(expectedBytes, expectedBytes); +} + +/** + * Shared scope checks for both authentication methods: parse the credential, + * match its signed date against the request date, then match the access key. + * Real R2 reports the date mismatch as coming from the 'x-amz-date' header + * even when the date came from the X-Amz-Date query parameter. + */ +function checkCredentialScope( + credentialField: string, + amzDate: string, + credentials: S3Credentials +): ParsedCredential | { error: Response } { + const credential = parseCredential(credentialField); + if ("error" in credential) { + return credential; + } + if (!amzDate.startsWith(credential.date)) { + return { + error: invalidArgument( + `Credential signed date ${credential.date} does not match ${amzDate.slice(0, 8)} from 'x-amz-date' header` + ), + }; + } + if (credential.accessKeyId !== credentials.accessKeyId) { + return { error: unauthorized() }; + } + return credential; +} + +function parseSignedHeaders(field: string): string[] | { error: Response } { + const signedHeaders = field.split(";").map((name) => name.toLowerCase()); + // SigV4 requires the SignedHeaders list to include `host` + if (!signedHeaders.includes("host")) { + return { error: unauthorized() }; + } + return signedHeaders; +} + +async function checkSignature( + request: Request, + url: URL, + credentials: S3Credentials, + credential: ParsedCredential, + amzDate: string, + signedHeaders: string[], + payloadHash: string, + presigned: boolean, + provided: string +): Promise { + const computed = await computeSignature( + request, + url, + credentials.secretAccessKey, + credential, + amzDate, + signedHeaders, + payloadHash, + presigned + ); + return timingSafeStringsEqual(computed.signature, provided) + ? undefined + : signatureDoesNotMatch(computed, provided); +} + +async function verifyAuthorizationHeader( + request: Request, + url: URL, + credentials: S3Credentials, + authorization: string +): Promise { + const payloadHash = request.headers.get("x-amz-content-sha256"); + if (payloadHash === null) { + return errorResponse(400, "InvalidRequest", "Missing x-amz-content-sha256"); + } + + let amzDate = request.headers.get("x-amz-date"); + let dateSource = "'x-amz-date' header"; + if (amzDate === null) { + amzDate = request.headers.get("date"); + dateSource = "'date' header"; + } + if (amzDate === null) { + return invalidArgument("No date provided in x-amz-date nor date header"); + } + const date = parseAmzDate(amzDate); + if (date === undefined) { + return invalidArgument( + `Date provided in ${dateSource} (${amzDate}) didn't parse successfully` + ); + } + if (Math.abs(Date.now() - date.getTime()) > MAX_SKEW_MILLIS) { + return errorResponse( + 403, + "RequestTimeTooSkewed", + "The difference between the request time and the server's time is too large." + ); + } + + // `AWS4-HMAC-SHA256 Credential=, SignedHeaders=, Signature=` + const fields = new Map(); + if (authorization.startsWith(`${ALGORITHM} `)) { + for (const component of authorization.slice(ALGORITHM.length).split(",")) { + const separator = component.indexOf("="); + if (separator === -1) { + continue; + } + fields.set( + component.slice(0, separator).trim(), + component.slice(separator + 1).trim() + ); + } + } + const credentialField = fields.get("Credential"); + const signedHeadersField = fields.get("SignedHeaders"); + const signatureField = fields.get("Signature"); + if ( + credentialField === undefined || + signedHeadersField === undefined || + signatureField === undefined + ) { + return unsupportedAlgorithm(); + } + + const credential = checkCredentialScope( + credentialField, + amzDate, + credentials + ); + if ("error" in credential) { + return credential.error; + } + + const signedHeaders = parseSignedHeaders(signedHeadersField); + if ("error" in signedHeaders) { + return signedHeaders.error; + } + + const mismatch = await checkSignature( + request, + url, + credentials, + credential, + amzDate, + signedHeaders, + payloadHash, + false, + signatureField + ); + if (mismatch !== undefined) { + return mismatch; + } + + // When a literal payload hash was signed (rather than UNSIGNED-PAYLOAD or + // a streaming sentinel), verify the body actually matches it + if (/^[0-9a-f]{64}$/.test(payloadHash)) { + const body = await request.clone().arrayBuffer(); + if ((await sha256Hex(body)) !== payloadHash) { + return errorResponse( + 400, + "XAmzContentSHA256Mismatch", + "The provided 'x-amz-content-sha256' header does not match what was computed." + ); + } + } + + return undefined; +} + +async function verifyPresigned( + request: Request, + url: URL, + credentials: S3Credentials +): Promise { + const params = url.searchParams; + + const missing = [ + "X-Amz-Algorithm", + "X-Amz-Signature", + "X-Amz-Date", + "X-Amz-SignedHeaders", + "X-Amz-Expires", + ].filter((name) => !params.has(name)); + if (missing.length === 1) { + return invalidArgument(`Required search parameter ${missing[0]} missing`); + } + if (missing.length > 1) { + return invalidArgument( + `Required search parameters ${missing.join(", ")} missing` + ); + } + + if (params.get("X-Amz-Algorithm") !== ALGORITHM) { + return unsupportedAlgorithm(); + } + + const amzDate = params.get("X-Amz-Date"); + const expiresParam = params.get("X-Amz-Expires"); + const signedHeadersParam = params.get("X-Amz-SignedHeaders"); + const provided = params.get("X-Amz-Signature"); + assert( + amzDate !== null && + expiresParam !== null && + signedHeadersParam !== null && + provided !== null + ); + + const date = parseAmzDate(amzDate); + if (date === undefined) { + return invalidArgument( + `Date provided in X-Amz-Date (${amzDate}) didn't parse successfully` + ); + } + + const credentialParam = params.get("X-Amz-Credential"); + assert(credentialParam !== null); + const credential = checkCredentialScope( + credentialParam, + amzDate, + credentials + ); + if ("error" in credential) { + return credential.error; + } + + // `Number("")` is 0, but we must reject an empty X-Amz-Expires. + const expires = + expiresParam.trim() === "" ? Number.NaN : Number(expiresParam); + if (Number.isNaN(expires)) { + return invalidArgument("X-Amz-Expires should be a number"); + } + if (expires > MAX_EXPIRES_SECONDS) { + return invalidArgument( + `X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than ${MAX_EXPIRES_SECONDS} seconds` + ); + } + if (expires < 1 || Date.now() > date.getTime() + expires * 1000) { + return errorResponse(403, "ExpiredRequest", "Request has expired"); + } + + const signedHeaders = parseSignedHeaders(signedHeadersParam); + if ("error" in signedHeaders) { + return signedHeaders.error; + } + + // Presigned URLs sign the payload as UNSIGNED-PAYLOAD since the body is + // not known at signing time + return checkSignature( + request, + url, + credentials, + credential, + amzDate, + signedHeaders, + "UNSIGNED-PAYLOAD", + true, + provided + ); +} + +/** Timing-safe comparison of credential pairs */ +export function credentialsEqual( + expected: S3Credentials, + provided: S3Credentials +): boolean { + return ( + timingSafeStringsEqual(expected.accessKeyId, provided.accessKeyId) && + timingSafeStringsEqual(expected.secretAccessKey, provided.secretAccessKey) + ); +} + +/** Whether the request carries either SigV4 authentication method */ +export function hasAuthentication( + request: Request, + params: URLSearchParams +): boolean { + return ( + request.headers.get("Authorization") !== null || + params.has("X-Amz-Credential") + ); +} + +/** + * Verifies a request against AWS Signature Version 4, returning an R2-style + * XML error `Response` on failure, or `undefined` if authentication succeeds. + */ +export async function verifyRequest( + request: Request, + credentials: S3Credentials +): Promise { + const url = new URL(request.url); + + // The Authorization header takes precedence over presigned query params. + const authorization = request.headers.get("Authorization"); + if (authorization !== null) { + return verifyAuthorizationHeader(request, url, credentials, authorization); + } + + if (url.searchParams.has("X-Amz-Credential")) { + return verifyPresigned(request, url, credentials); + } + + return invalidArgument("Authorization"); +} diff --git a/packages/miniflare/src/workers/r2/s3/common.worker.ts b/packages/miniflare/src/workers/r2/s3/common.worker.ts new file mode 100644 index 0000000000..9028d09ab4 --- /dev/null +++ b/packages/miniflare/src/workers/r2/s3/common.worker.ts @@ -0,0 +1,56 @@ +import { Buffer } from "node:buffer"; +import { XMLBuilder, XMLParser } from "fast-xml-parser"; +import type { R2S3Bindings, S3Credentials } from "../constants"; +import type { Context } from "hono"; + +export interface Env { + [R2S3Bindings.JSON_CREDENTIALS]: Record; + [bucket: `${typeof R2S3Bindings.BUCKET_PREFIX}${string}`]: + | R2Bucket + | undefined; +} + +export type S3Context = Context<{ Bindings: Env }>; + +/** HEAD responses must not include a body */ +export function stripBodyForHead(c: S3Context, response: Response): Response { + return c.req.method === "HEAD" && response.body !== null + ? new Response(null, response) + : response; +} + +const XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"; +export const MAX_LIST_KEYS = 1000; +export const MAX_DELETE_KEYS = 1000; + +const xmlBuilder = new XMLBuilder({ ignoreAttributes: false }); +export const xmlParser = new XMLParser({ + ignoreAttributes: true, + parseTagValue: false, +}); + +export function xmlResponse( + root: string, + content: Record, + status = 200 +): Response { + const body = `${xmlBuilder.build({ + [root]: { "@_xmlns": XMLNS, ...content }, + })}`; + return new Response(body, { + status, + headers: { "Content-Type": "application/xml" }, + }); +} + +export function hex(bytes: ArrayLike | ArrayBuffer): string { + return Buffer.from(bytes).toString("hex"); +} + +export function coerceArray(value: T | T[] | undefined): T[] { + if (value === undefined) { + return []; + } + + return Array.isArray(value) ? value : [value]; +} diff --git a/packages/miniflare/src/workers/r2/s3/detect.worker.ts b/packages/miniflare/src/workers/r2/s3/detect.worker.ts new file mode 100644 index 0000000000..af4287364b --- /dev/null +++ b/packages/miniflare/src/workers/r2/s3/detect.worker.ts @@ -0,0 +1,414 @@ +import { errorResponse, notImplemented, routeNotFound } from "./errors.worker"; +import type { S3Context } from "./common.worker"; + +const notImplementedOperation = (name: string) => + notImplemented(`${name} not implemented`); + +export type BucketOperation = + | "HeadBucket" + | "GetBucketLocation" + | "GetBucketEncryption" + | "GetBucketVersioning" + | "GetBucketTagging" + | "GetObjectLockConfiguration" + | "GetBucketReplication" + | "ListObjects" + | "ListObjectsV2" + | "DeleteObjects"; + +export type ObjectOperation = + | "GetObject" + | "HeadObject" + | "PutObject" + | "CopyObject" + | "DeleteObject" + | "CreateMultipartUpload"; + +/** Operations on an in-progress multipart upload, addressed by ?uploadId */ +export type MultipartOperation = + | "UploadPart" + | "UploadPartCopy" + | "CompleteMultipartUpload" + | "AbortMultipartUpload"; + +export type S3Operation = + | BucketOperation + | ObjectOperation + | MultipartOperation; + +export interface MultipartDetection { + operation: MultipartOperation; + uploadId: string; +} + +/** + * Object-level subresource query parameters map onto operations R2's S3 + * endpoint recognizes but does not implement. Subresource parameters + * without an entry for the request method are silently ignored (e.g. + * `DELETE ?tagging` is plain DeleteObject on R2). + */ +const OBJECT_SUBRESOURCES: Record>> = { + tagging: { GET: "GetObjectTagging", PUT: "PutObjectTagging" }, + acl: { GET: "GetObjectAcl", PUT: "PutObjectAcl" }, + attributes: { GET: "GetObjectAttributes" }, + torrent: { GET: "GetObjectTorrent" }, + retention: { GET: "GetObjectRetention", PUT: "PutObjectRetention" }, + "legal-hold": { GET: "GetObjectLegalHold", PUT: "PutObjectLegalHold" }, +}; + +/** + * Bucket-level GET subresources that respond with R2's templated + * " not implemented" error. Most are unimplemented on R2 too. + * acl/cors/lifecycle are implemented by real R2 but return per-bucket state + * (CORS rules, lifecycle rules, the account id in the ACL owner) + * that the R2 binding does not expose. + */ +const BUCKET_GET_NOT_IMPLEMENTED: Partial> = { + versions: "ListObjectVersions", + policy: "GetBucketPolicy", + website: "GetBucketWebsite", + notification: "GetBucketNotificationConfiguration", + requestPayment: "GetBucketRequestPayment", + logging: "GetBucketLogging", + accelerate: "GetBucketAccelerateConfiguration", + publicAccessBlock: "GetPublicAccessBlock", + ownershipControls: "GetBucketOwnershipControls", + "intelligent-tiering": "GetBucketIntelligentTieringConfiguration", + inventory: "GetBucketInventoryConfiguration", + metrics: "GetBucketMetricsConfiguration", + analytics: "GetBucketAnalyticsConfiguration", + // R2's typo, reproduced verbatim + policyStatus: "GetGetBucketPolicyStatus", + acl: "GetBucketAcl", + cors: "GetBucketCors", + lifecycle: "GetBucketLifecycleConfiguration", +}; + +/** + * Bucket-level PUT subresources, all answered with R2's templated named + * error. cors/encryption/lifecycle/versioning are implemented by real R2 + * (it validates the XML body before anything else) but write per-bucket + * state the R2 binding cannot manage; since prod never reveals its names + * for them, they use the AWS operation names. + */ +const BUCKET_PUT_NOT_IMPLEMENTED: Partial> = { + accelerate: "PutBucketAccelerateConfiguration", + acl: "PutBucketAcl", + analytics: "PutBucketAnalyticsConfiguration", + "intelligent-tiering": "PutBucketIntelligentTieringConfiguration", + inventory: "PutBucketInventoryConfiguration", + logging: "PutBucketLogging", + metrics: "PutBucketMetricsConfiguration", + notification: "PutBucketNotificationConfiguration", + "object-lock": "PutObjectLockConfiguration", + ownershipControls: "PutBucketOwnershipControls", + policy: "PutBucketPolicy", + publicAccessBlock: "PutPublicAccessBlock", + replication: "PutBucketReplication", + requestPayment: "PutBucketRequestPayment", + tagging: "PutBucketTagging", + website: "PutBucketWebsite", + cors: "PutBucketCors", + encryption: "PutBucketEncryption", + lifecycle: "PutBucketLifecycleConfiguration", + versioning: "PutBucketVersioning", +}; + +/** + * Bucket-level DELETE subresources. cors/encryption/lifecycle are + * implemented by real R2 (permission-gated) but have no binding equivalent; + * AWS operation names, as above. Subresources that exist only for other + * methods (?versioning, ?acl, ...) are NOT special-cased on DELETE; R2 + * reports them via the "Unsupported search param(s)" error. + */ +const BUCKET_DELETE_NOT_IMPLEMENTED: Partial> = { + analytics: "DeleteBucketAnalyticsConfiguration", + "intelligent-tiering": "DeleteBucketIntelligentTieringConfiguration", + inventory: "DeleteBucketInventoryConfiguration", + metrics: "DeleteBucketMetricsConfiguration", + ownershipControls: "DeleteBucketOwnershipControls", + policy: "DeleteBucketPolicy", + replication: "DeleteBucketReplication", + tagging: "DeleteBucketTagging", + website: "DeleteBucketWebsite", + cors: "DeleteBucketCors", + encryption: "DeleteBucketEncryption", + lifecycle: "DeleteBucketLifecycle", +}; + +/** + * Bucket-configuration reads with static responses on R2 (identical + * for every bucket, since R2 has no versioning/tagging/object + * lock/replication and always encrypts with AES256) + */ +const BUCKET_GET_STATIC: Partial> = { + encryption: "GetBucketEncryption", + versioning: "GetBucketVersioning", + tagging: "GetBucketTagging", + "object-lock": "GetObjectLockConfiguration", + replication: "GetBucketReplication", +}; + +/** + * Query parameters R2 accepts on list requests. Reject anything else + * (excluding presign parameters and the SDK's x-id) with R2's + * "search parameter" error. + */ +const LIST_OBJECTS_PARAMS = new Set([ + "prefix", + "delimiter", + "marker", + "max-keys", + "encoding-type", +]); +const LIST_OBJECTS_V2_PARAMS = new Set([ + "list-type", + "prefix", + "delimiter", + "continuation-token", + "start-after", + "max-keys", + "encoding-type", + "fetch-owner", +]); + +export function isScreenedParam(name: string): boolean { + return name.startsWith("X-Amz-") || name === "x-id"; +} + +function detectListOperation( + params: URLSearchParams +): BucketOperation | Response { + const listType = params.get("list-type"); + if (listType !== null && listType !== "2") { + return notImplementedOperation(`ListObjectsV${listType}`); + } + + const v2 = listType === "2"; + // Reject V2-only pagination on V1 + if (!v2 && params.has("continuation-token")) { + return errorResponse( + 400, + "InvalidArgument", + "continuation-token not supported in ListObjects" + ); + } + + const allowed = v2 ? LIST_OBJECTS_V2_PARAMS : LIST_OBJECTS_PARAMS; + for (const name of params.keys()) { + if (!allowed.has(name) && !isScreenedParam(name)) { + return notImplemented( + `ListObjectsV${v2 ? "2" : "1"} search parameter ${name} not implemented` + ); + } + } + + return v2 ? "ListObjectsV2" : "ListObjects"; +} + +/** + * Bucket-level PUT/DELETE: a recognized subresource (in any position) wins + * with its named templated error; otherwise every non-presign param is + * rejected together. Only a bare PUT/DELETE reaches CreateBucket / + * DeleteBucket; both are implemented by real R2, but local buckets are + * statically configured, so they get the templated named error instead. + */ +function detectBucketMutation( + method: string, + params: URLSearchParams, + subresources: Partial>, + bareOperation: string +): Response { + for (const name of params.keys()) { + const operation = subresources[name]; + if (operation !== undefined) { + return notImplementedOperation(operation); + } + } + + const unsupported = [...new Set(params.keys())].filter( + (name) => !isScreenedParam(name) + ); + if (unsupported.length > 0) { + return errorResponse( + 400, + "InvalidArgument", + `Unsupported search param(s) ${unsupported + .map((name) => `"${name}"`) + .join(", ")} on a ${method} bucket route` + ); + } + + return notImplementedOperation(bareOperation); +} + +export function detectBucketOperation( + method: string, + params: URLSearchParams +): BucketOperation | Response { + switch (method) { + case "HEAD": + return "HeadBucket"; + case "PUT": + return detectBucketMutation( + "PUT", + params, + BUCKET_PUT_NOT_IMPLEMENTED, + "CreateBucket" + ); + case "DELETE": + return detectBucketMutation( + "DELETE", + params, + BUCKET_DELETE_NOT_IMPLEMENTED, + "DeleteBucket" + ); + case "POST": + // ?delete is DeleteObjects; respond 200 with an empty body to + // any other bucket-level POST + return params.has("delete") + ? "DeleteObjects" + : new Response(null, { status: 200 }); + case "GET": { + if (params.has("uploads")) { + return notImplementedOperation("ListMultipartUploads"); + } + + if (params.has("location")) { + for (const name of params.keys()) { + if (name !== "location" && !isScreenedParam(name)) { + return errorResponse( + 400, + "InvalidArgument", + `Search param ${name} is unsupported for bucket location` + ); + } + } + + return "GetBucketLocation"; + } + + for (const name of params.keys()) { + const staticOperation = BUCKET_GET_STATIC[name]; + if (staticOperation !== undefined) { + return staticOperation; + } + + const notImplementedName = BUCKET_GET_NOT_IMPLEMENTED[name]; + if (notImplementedName !== undefined) { + return notImplementedOperation(notImplementedName); + } + } + + return detectListOperation(params); + } + default: + return routeNotFound(); + } +} + +/** A recognized object subresource wins with its named templated error */ +function objectSubresourceError( + params: URLSearchParams, + method: string +): Response | undefined { + for (const name of params.keys()) { + const subresource = OBJECT_SUBRESOURCES[name]?.[method]; + if (subresource !== undefined) { + return notImplementedOperation(subresource); + } + } + return undefined; +} + +export function detectObjectOperation( + c: S3Context, + params: URLSearchParams +): ObjectOperation | MultipartDetection | Response { + const method = c.req.method; + + const uploadId = params.get("uploadId"); + if (uploadId !== null) { + // R2's part routes only match when partNumber looks like an + // (optionally space-padded, signed) integer; other values (e.g. + // `1.0`) match no route at all, and range validation happens later + const partNumber = params.get("partNumber"); + if (partNumber !== null && !/^ *-?\d*$/.test(partNumber)) { + return routeNotFound(); + } + switch (method) { + case "GET": + // Real R2 implements ListParts when partNumber is also present + // (and reports it unimplemented otherwise). The local R2 + // binding cannot list parts either way. + return notImplementedOperation("ListParts"); + case "PUT": + if (partNumber === null) { + // Without partNumber, ignore `uploadId` entirely and + // treat the request as a plain PutObject/CopyObject + break; + } + + return { + operation: + c.req.header("x-amz-copy-source") !== undefined + ? "UploadPartCopy" + : "UploadPart", + uploadId, + }; + case "POST": + return { operation: "CompleteMultipartUpload", uploadId }; + case "DELETE": + return { operation: "AbortMultipartUpload", uploadId }; + case "HEAD": + return "HeadObject"; + default: + return routeNotFound(); + } + } + if (params.has("uploads")) { + if (method === "POST") { + return "CreateMultipartUpload"; + } + + // Serve the bucket's upload list even on object paths + if (method === "GET") { + return notImplementedOperation("ListMultipartUploads"); + } + } + switch (method) { + case "GET": + case "HEAD": + // Real R2 serves individual parts; `bucket.get()` cannot + if (params.has("partNumber")) { + return notImplementedOperation("partNumber"); + } + + if (method === "HEAD") { + // Real R2 HEAD ignores subresource parameters + return "HeadObject"; + } + + return objectSubresourceError(params, method) ?? "GetObject"; + case "PUT": { + const subresource = objectSubresourceError(params, method); + if (subresource !== undefined) { + return subresource; + } + + return c.req.header("x-amz-copy-source") !== undefined + ? "CopyObject" + : "PutObject"; + } + case "POST": + // R2 treats POST on an object key as PutObject, ignoring + // subresource parameters. Unlike PUT, x-amz-copy-source does NOT + // make it a CopyObject: R2 ignores the header and stores the + // request body. + return "PutObject"; + case "DELETE": + return "DeleteObject"; + default: + return routeNotFound(); + } +} diff --git a/packages/miniflare/src/workers/r2/s3/dispatch.worker.ts b/packages/miniflare/src/workers/r2/s3/dispatch.worker.ts new file mode 100644 index 0000000000..bd44f4a915 --- /dev/null +++ b/packages/miniflare/src/workers/r2/s3/dispatch.worker.ts @@ -0,0 +1,140 @@ +// The per-bucket request pipeline: resolve the bucket and its credentials, +// authenticate, detect the operation, screen its headers, then run it. +import assert from "node:assert"; +import { R2S3Bindings } from "../constants"; +import { hasAuthentication, verifyRequest } from "./auth.worker"; +import { stripBodyForHead } from "./common.worker"; +import { detectBucketOperation, detectObjectOperation } from "./detect.worker"; +import { noSuchBucket, notImplemented } from "./errors.worker"; +import { + BUCKET_OPERATIONS, + MULTIPART_OPERATIONS, + OBJECT_OPERATIONS, + screenHeaders, +} from "./operations.worker"; +import type { S3Context } from "./common.worker"; +import type { S3Operation } from "./detect.worker"; +import type { OperationDefinition, ScreeningRules } from "./operations.worker"; +import type { Awaitable } from "miniflare:shared"; + +export async function dispatch( + c: S3Context, + key: string | undefined +): Promise { + return stripBodyForHead(c, await dispatchInner(c, key)); +} + +async function dispatchInner( + c: S3Context, + key: string | undefined +): Promise { + // Both routes that reach `dispatch()` carry a :bucketId segment + const bucketId = c.req.param("bucketId"); + assert(bucketId !== undefined); + + const credentials = c.env[R2S3Bindings.JSON_CREDENTIALS][bucketId]; + if (credentials === undefined) { + // Real R2 verifies the signature before bucket existence (auth errors + // win over the 404), but its credentials are account-scoped while + // local credentials are per-bucket: an unknown bucket has no + // credential set to verify against, so existence is reported first. + return noSuchBucket(); + } + + // The plugin binds exactly the buckets in the credential map + const bucket = c.env[`${R2S3Bindings.BUCKET_PREFIX}${bucketId}`]; + assert(bucket !== undefined); + + const params = new URL(c.req.url).searchParams; + + // R2 interprets a bucket-level POST carrying no auth at all as an + // attempted browser form upload (AWS's POST Object, where auth travels in + // the form fields), which it recognizes but does not implement. The + // doubled "not implemented" is R2's, verbatim. `?delete` (DeleteObjects) + // gets the normal missing-auth error instead. + if ( + key === undefined && + c.req.method === "POST" && + !params.has("delete") && + !hasAuthentication(c.req.raw, params) + ) { + return notImplemented( + "Presigned post requests are not yet implemented not implemented" + ); + } + + const authError = await verifyRequest(c.req.raw, credentials); + if (authError !== undefined) { + return authError; + } + + const detected = detectOperation(c, bucket, bucketId, key, params); + if (detected instanceof Response) { + return detected; + } + + const screenError = screenHeaders(c, detected.operation, detected.rules); + if (screenError !== undefined) { + return screenError; + } + + return detected.run(); +} + +/** A detected operation, bound to the context its handler needs */ +interface BoundOperation { + operation: S3Operation; + rules: ScreeningRules; + run(): Awaitable; +} + +/** + * Detection and the operation tables are split by addressing level (bucket-, + * object-, and multipart-level operations take different contexts). Binding + * the context here keeps those splits out of `dispatch()`. + */ +function detectOperation( + c: S3Context, + bucket: R2Bucket, + bucketId: string, + key: string | undefined, + params: URLSearchParams +): BoundOperation | Response { + if (key === undefined) { + const detected = detectBucketOperation(c.req.method, params); + if (detected instanceof Response) { + return detected; + } + + return bind(detected, BUCKET_OPERATIONS, { c, bucket, bucketId, params }); + } + + const detected = detectObjectOperation(c, params); + if (detected instanceof Response) { + return detected; + } + + const context = { c, bucket, bucketId, key, params }; + // Multipart operations, uniquely, are detected as an `object` + if (typeof detected === "object") { + return bind(detected.operation, MULTIPART_OPERATIONS, { + ...context, + uploadId: detected.uploadId, + }); + } + + return bind(detected, OBJECT_OPERATIONS, context); +} + +function bind( + operation: Operation, + table: Record & ScreeningRules>, + context: Context +): BoundOperation { + const definition = table[operation]; + return { + operation, + rules: definition, + run: () => definition.handle(context), + }; +} diff --git a/packages/miniflare/src/workers/r2/s3/errors.worker.ts b/packages/miniflare/src/workers/r2/s3/errors.worker.ts new file mode 100644 index 0000000000..61976586bf --- /dev/null +++ b/packages/miniflare/src/workers/r2/s3/errors.worker.ts @@ -0,0 +1,43 @@ +function xmlEscape(value: string): string { + return value + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + +/** + * https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html + * Unlike success documents, R2's documents carry no xmlns. + * `extraFields` are appended after in entry order (e.g. + * SignatureDoesNotMatch's debug fields). + * + * Hand-built rather than with common.worker's XMLBuilder: R2 escapes `'` as + * `'` in error messages, which fast-xml-parser's entity processing + * does not reproduce. + */ +export function errorResponse( + status: number, + code: string, + message: string, + extraFields: Record = {} +) { + const extra = Object.entries(extraFields) + .map(([name, value]) => `<${name}>${xmlEscape(value)}`) + .join(""); + const body = `${code}${xmlEscape(message)}${extra}`; + return new Response(body, { + status, + headers: { "Content-Type": "application/xml" }, + }); +} + +export const noSuchBucket = () => + errorResponse(404, "NoSuchBucket", "The specified bucket does not exist."); + +export const notImplemented = (message: string) => + errorResponse(501, "NotImplemented", message); + +export const routeNotFound = () => + errorResponse(404, "RouteNotFound", "No route matches this url."); diff --git a/packages/miniflare/src/workers/r2/s3/index.worker.ts b/packages/miniflare/src/workers/r2/s3/index.worker.ts new file mode 100644 index 0000000000..9a2b20cf67 --- /dev/null +++ b/packages/miniflare/src/workers/r2/s3/index.worker.ts @@ -0,0 +1,85 @@ +// S3-compatible API for local R2 buckets. Status codes, headers, XML bodies, +// and header-screening behavior mimic R2's S3 endpoint, captured from a real +// bucket (2026-06-11). +// +// ## Gaps: surfaces real R2 implements, with no local equivalent +// +// These exist in real R2 but cannot be served faithfully here because the R2 +// binding backing this endpoint (and Miniflare's R2 simulator behind it) +// exposes no equivalent. Operations among them respond exactly like the +// operations R2's S3 endpoint itself does not implement: R2's templated +// " not implemented" error (plus list/bucket-route "search parameter" +// errors and RouteNotFound for unknown methods). +// +// - SSE-C: the simulator ignores encryption keys entirely, so writes with +// SSE-C headers are rejected with NotImplemented rather than silently +// storing plaintext. Reads with SSE-C headers return the same error real +// R2 gives for unencrypted objects. +// - x-amz-storage-class: the simulator can't persist storage classes, so +// non-default classes (STANDARD_IA) are rejected with NotImplemented +// rather than silently stored as STANDARD. +// - Flexible checksums: real R2 validates, stores, and echoes +// x-amz-checksum-* request headers (returning BadDigest on mismatch); the +// local binding has no checksum surface beyond MD5, so they are silently +// ignored like other unrecognized x-amz-* headers (Content-MD5 IS +// verified). The aws-chunked content encoding SDKs use to stream trailing +// checksums is not decoded either; clients must send plain bodies. +// - ListParts, ListMultipartUploads, the partNumber query parameter, the +// bucket-configuration reads ?acl, ?cors, and ?lifecycle, and the +// bucket-configuration writes/deletes for ?cors, ?encryption, ?lifecycle, +// and ?versioning answer the templated not-implemented error described +// above. +// - Owner in listings is omitted; there's no account id to report locally. +// - ListBuckets omits CreationDate; local buckets have no creation time. +// - GetBucketLocation reports "auto"; real R2 reports the bucket's location +// hint (e.g. ENAM). +// +// ## By design: surfaces where the local architecture differs +// +// Local buckets and credentials are declared in Wrangler/Miniflare config, +// not provisioned under a Cloudflare account, so some of real R2's account +// semantics deliberately do not apply: +// +// - CreateBucket and DeleteBucket answer the templated not-implemented +// error: a bucket exists locally by being configured as a binding, and a +// bucket created over HTTP would not be reachable from any Worker. +// - Credentials are per-bucket, not account-scoped. Requests for unknown +// buckets return NoSuchBucket without checking the signature (real R2 +// verifies auth first), and ListBuckets lists the buckets sharing the +// presented credential pair rather than every bucket on an account. +// - CORS: cross-origin use is always allowed, as if the bucket had a +// permissive CORS policy configured. Real R2 answers preflights according +// to the bucket's per-bucket CORS configuration, which has no R2 binding +// equivalent. Without preflight approval for `Authorization` and +// `x-amz-*` headers, no browser request could ever reach this endpoint +// (e.g. presigned uploads from a frontend dev server). +// +// ## Client quirks +// +// - workerd never responds with `100 Continue`, so clients sending +// `Expect: 100-continue` (the AWS SDKs do by default for requests with +// bodies) hang waiting for it. With @aws-sdk/client-s3, remove the +// `addExpectContinueMiddleware` from the client's middleware stack. + +import { cors } from "hono/cors"; +import { Hono } from "hono/tiny"; +import { CorePaths } from "../../core/constants"; +import { listBuckets } from "./account.worker"; +import { dispatch } from "./dispatch.worker"; +import type { Env } from "./common.worker"; + +const app = new Hono<{ Bindings: Env }>().basePath(CorePaths.R2_S3); + +app.use( + cors({ + origin: "*", + allowMethods: ["GET", "HEAD", "PUT", "POST", "DELETE"], + exposeHeaders: ["*"], + }) +); + +app.all("/:bucketId/:key{.+}", (c) => dispatch(c, c.req.param("key"))); +app.all("/:bucketId", (c) => dispatch(c, undefined)); +app.all("/", (c) => listBuckets(c)); + +export default app; diff --git a/packages/miniflare/src/workers/r2/s3/operations.worker.ts b/packages/miniflare/src/workers/r2/s3/operations.worker.ts new file mode 100644 index 0000000000..62e25a64ee --- /dev/null +++ b/packages/miniflare/src/workers/r2/s3/operations.worker.ts @@ -0,0 +1,1075 @@ +import assert from "node:assert"; +import { XMLValidator } from "fast-xml-parser"; +import { CorePaths } from "../../core/constants"; +import { R2S3Bindings } from "../constants"; +import { parseRangeHeader, serveR2Object } from "../serve.worker"; +import { awsUriEncode, credentialsEqual } from "./auth.worker"; +import { + coerceArray, + hex, + MAX_DELETE_KEYS, + MAX_LIST_KEYS, + xmlParser, + xmlResponse, +} from "./common.worker"; +import { errorResponse, noSuchBucket, notImplemented } from "./errors.worker"; +import type { S3Context } from "./common.worker"; +import type { + BucketOperation, + MultipartOperation, + ObjectOperation, + S3Operation, +} from "./detect.worker"; +import type { Awaitable } from "miniflare:shared"; + +interface S3Error { + status: number; + code: string; + message: string; +} + +const NO_SUCH_KEY: S3Error = { + status: 404, + code: "NoSuchKey", + message: "The specified key does not exist.", +}; +const PRECONDITION_FAILED: S3Error = { + status: 412, + code: "PreconditionFailed", + message: "At least one of the pre-conditions you specified did not hold.", +}; +const NO_SUCH_UPLOAD: S3Error = { + status: 404, + code: "NoSuchUpload", + message: "The specified multipart upload does not exist.", +}; + +const s3Error = (error: S3Error) => + errorResponse(error.status, error.code, error.message); + +const noSuchKey = () => s3Error(NO_SUCH_KEY); + +const preconditionFailed = () => s3Error(PRECONDITION_FAILED); + +const malformedXml = () => + errorResponse( + 400, + "MalformedXML", + "The XML you provided was not well formed or did not validate against our published schema." + ); + +const notImplementedHeader = (name: string, value: string) => + errorResponse( + 501, + "NotImplemented", + `Header '${name}' with value '${value}' not implemented` + ); + +/** + * R2 binding errors carry a stable v4 error code in their message (e.g. + * "completeMultipartUpload: The specified multipart upload does not exist. + * (10024)"); map those onto the S3 error responses real R2 returns. + */ +const BINDING_ERRORS: Partial> = { + // NO_SUCH_OBJECT_KEY + 10007: NO_SUCH_KEY, + // ENTITY_TOO_SMALL + 10011: { + status: 400, + code: "EntityTooSmall", + message: + "Your proposed upload is smaller than the minimum allowed object size.", + }, + // NO_SUCH_UPLOAD + 10024: NO_SUCH_UPLOAD, + // INVALID_PART + 10025: { + status: 400, + code: "InvalidPart", + message: "One or more of the specified parts could not be found.", + }, + // PRECONDITION_FAILED + 10031: PRECONDITION_FAILED, + // INVALID_RANGE + 10039: { + status: 416, + code: "InvalidRange", + message: "The requested range is not satisfiable", + }, +}; + +/** + * Parsing the message is the only option: workerd deliberately throws plain + * Errors formatted as ": ()"; its structured + * R2Error type is disabled (r2-rpc.c++, "all we can send back to the user + * is a message"). + */ +function bindingError(e: unknown): Response { + const message = e instanceof Error ? e.message : String(e); + const v4Code = /\((\d+)\)$/.exec(message); + const known = v4Code === null ? undefined : BINDING_ERRORS[Number(v4Code[1])]; + if (known !== undefined) { + return s3Error(known); + } + + return errorResponse(500, "InternalError", message); +} + +const BUCKET_OWNER = ["x-amz-expected-bucket-owner"]; +const SOURCE_BUCKET_OWNER = ["x-amz-source-expected-bucket-owner"]; +const MFA_AND_LOCK_BYPASS = ["x-amz-mfa", "x-amz-bypass-governance-retention"]; +const COPY_SOURCE_CONDITIONALS = [ + "x-amz-copy-source-if-match", + "x-amz-copy-source-if-none-match", + "x-amz-copy-source-if-modified-since", + "x-amz-copy-source-if-unmodified-since", +]; +/** Headers R2 recognizes but rejects on every write operation */ +const WRITE_UNSUPPORTED = [ + ...BUCKET_OWNER, + "x-amz-tagging", + "x-amz-grant-full-control", + "x-amz-grant-read", + "x-amz-grant-read-acp", + "x-amz-grant-write", + "x-amz-grant-write-acp", + "x-amz-website-redirect-location", + "x-amz-object-lock-mode", + "x-amz-object-lock-retain-until-date", + "x-amz-object-lock-legal-hold", + "x-amz-server-side-encryption-aws-kms-key-id", + "x-amz-server-side-encryption-context", + "x-amz-server-side-encryption-bucket-key-enabled", +]; + +/** Everything a bucket-level handler might need, resolved once in `dispatch()` */ +export interface BucketOperationContext { + c: S3Context; + bucket: R2Bucket; + bucketId: string; + params: URLSearchParams; +} + +/** Object-level operations additionally address a key within the bucket */ +export interface ObjectOperationContext extends BucketOperationContext { + key: string; +} + +/** Multipart operations additionally address an in-progress upload */ +export interface MultipartOperationContext extends ObjectOperationContext { + uploadId: string; +} + +export interface ScreeningRules { + /** Headers rejected with the templated NotImplemented error */ + unsupportedHeaders: string[]; + /** Validate x-amz-server-side-encryption / x-amz-acl values */ + validatesWriteHeaders?: true; + /** + * SSE-C handling: real R2 supports SSE-C on these operations, but the + * local simulator ignores encryption keys. "read" returns R2's standard + * error for SSE-C parameters on an unencrypted object; "write" reports + * the header as NotImplemented rather than silently storing plaintext. + */ + ssec?: "read" | "write"; +} + +export interface OperationDefinition { + handle(operation: Context): Awaitable; +} + +/** + * Accept canned ACLs as no-ops (R2 has no object ACLs); reject anything + * else with the templated NotImplemented error + */ +const CANNED_ACLS = new Set([ + "private", + "public-read", + "public-read-write", + "authenticated-read", + "aws-exec-read", + "bucket-owner-read", + "bucket-owner-full-control", +]); + +export function screenHeaders( + c: S3Context, + operation: S3Operation, + rules: ScreeningRules +): Response | undefined { + // Reject session tokens on every operation (auth-level, not + // operation-level) + if (c.req.header("x-amz-security-token") !== undefined) { + return errorResponse(400, "InvalidArgument", "X-Amz-Security-Token"); + } + + for (const name of rules.unsupportedHeaders) { + const value = c.req.header(name); + if (value !== undefined) { + return notImplementedHeader(name, value); + } + } + + if (rules.validatesWriteHeaders === true) { + // R2 encrypts at rest with AES256 anyway, so that value is a truthful + // no-op; KMS and other modes are not implemented + const sse = c.req.header("x-amz-server-side-encryption"); + if (sse !== undefined && sse !== "AES256") { + return notImplementedHeader("x-amz-server-side-encryption", sse); + } + + const acl = c.req.header("x-amz-acl"); + if (acl !== undefined && !CANNED_ACLS.has(acl)) { + return notImplementedHeader("x-amz-acl", acl); + } + } + + if (rules.ssec !== undefined) { + return screenSSECHeaders(c, operation, rules.ssec); + } + + return undefined; +} + +function screenSSECHeaders( + c: S3Context, + operation: S3Operation, + mode: "read" | "write" +): Response | undefined { + const prefixes = + operation === "CopyObject" || operation === "UploadPartCopy" + ? ["x-amz-", "x-amz-copy-source-"] + : ["x-amz-"]; + for (const prefix of prefixes) { + const algorithmName = `${prefix}server-side-encryption-customer-algorithm`; + const algorithm = c.req.header(algorithmName); + const key = c.req.header(`${prefix}server-side-encryption-customer-key`); + const keyMd5 = c.req.header( + `${prefix}server-side-encryption-customer-key-MD5` + ); + if (algorithm === undefined && key === undefined && keyMd5 === undefined) { + continue; + } + + // R2 requires the full header triple, checked in this order + if (key === undefined) { + return errorResponse( + 400, + "InvalidArgument", + "Requests specifying Server Side Encryption with Customer provided keys must provide an appropriate secret key." + ); + } + if (keyMd5 === undefined) { + return errorResponse( + 400, + "InvalidArgument", + "Requests specifying Server Side Encryption with Customer provided keys must provide the client calculated MD5 of the secret key." + ); + } + if (algorithm === undefined) { + return errorResponse( + 400, + "InvalidArgument", + "Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm." + ); + } + // After the triple is complete, R2 validates the algorithm value (on + // reads, writes, and copy sources alike) before anything else + if (algorithm !== "AES256") { + return errorResponse( + 400, + "InvalidEncryptionAlgorithmError", + "The encryption request that you specified is not valid. The valid value is AES256." + ); + } + + return mode === "read" + ? errorResponse( + 400, + "InvalidRequest", + "The encryption parameters are not applicable to this object." + ) + : notImplementedHeader(algorithmName, algorithm); + } + + return undefined; +} + +const CONDITIONAL_HEADERS = [ + "If-Match", + "If-None-Match", + "If-Modified-Since", + "If-Unmodified-Since", +]; + +/** + * The request body for a write: streamed through untouched unless + * Content-MD5 verification requires buffering it. + */ +async function verifiedRequestBody( + c: S3Context +): Promise { + if (c.req.header("Content-MD5") === undefined) { + return c.req.raw.body ?? ""; + } + + const buffered = await c.req.raw.arrayBuffer(); + const digestError = await verifyContentMD5(c, buffered); + return digestError ?? buffered; +} + +async function verifyContentMD5( + c: S3Context, + body: ArrayBuffer +): Promise { + const contentMd5 = c.req.header("Content-MD5"); + if (contentMd5 === undefined) { + return undefined; + } + let provided: Uint8Array; + try { + provided = Uint8Array.from(atob(contentMd5), (char) => char.charCodeAt(0)); + if (provided.length !== 16) { + throw new Error("bad length"); + } + } catch { + return errorResponse( + 400, + "InvalidDigest", + "The checksum or Content-MD5 you specified is not valid." + ); + } + const computed = hex(await crypto.subtle.digest("MD5", body)); + if (hex(provided) !== computed) { + return errorResponse( + 400, + "BadDigest", + `The MD5 checksum you specified did not match what we received.\nYou provided a MD5 checksum with value: ${hex(provided)}\nActual MD5 was: ${computed}` + ); + } + return undefined; +} + +/** Returns the R2 storage class for x-amz-storage-class, or an error */ +function parseStorageClass(c: S3Context): string | undefined | Response { + const header = c.req.header("x-amz-storage-class"); + switch (header) { + case undefined: + return undefined; + case "STANDARD": + return "Standard"; + case "STANDARD_IA": + // The simulator can't persist storage classes, so rather than + // silently storing as STANDARD, reject non-default classes + return notImplementedHeader("x-amz-storage-class", header); + default: + return errorResponse( + 400, + "InvalidStorageClass", + "The storage class specified is not valid." + ); + } +} + +function collectCustomMetadata(c: S3Context): Record { + const customMetadata: Record = {}; + for (const [name, value] of c.req.raw.headers) { + if (name.startsWith("x-amz-meta-")) { + customMetadata[name.slice("x-amz-meta-".length)] = value; + } + } + + return customMetadata; +} + +/** + * Parses `x-amz-copy-source` (`/bucket/key` or `bucket/key`) and resolves + * the source bucket binding. + */ +function parseCopySource( + c: S3Context, + bucketId: string +): { bucket: R2Bucket; key: string } | Response { + // detectObjectOperation() only routes to copy operations when the header + // exists + const header = c.req.header("x-amz-copy-source"); + assert(header !== undefined); + const raw = decodeURIComponent(header); + const source = raw.startsWith("/") ? raw.slice(1) : raw; + const separator = source.indexOf("/"); + if (separator === -1 || separator === source.length - 1) { + return errorResponse(400, "InvalidArgument", "copy source bucket name"); + } + + const sourceBucketId = source.slice(0, separator); + const bucket = c.env[`${R2S3Bindings.BUCKET_PREFIX}${sourceBucketId}`]; + if (bucket === undefined) { + return noSuchBucket(); + } + + // Credentials are per-bucket: the pair the request authenticated with + // (the target bucket's) must also grant access to the source bucket. + // Real R2 credentials are account-scoped, so it has no equivalent check. + const credentials = c.env[R2S3Bindings.JSON_CREDENTIALS]; + const sourceCredentials = credentials[sourceBucketId]; + const targetCredentials = credentials[bucketId]; + assert(sourceCredentials !== undefined && targetCredentials !== undefined); + if (!credentialsEqual(sourceCredentials, targetCredentials)) { + return errorResponse(401, "Unauthorized", "Unauthorized"); + } + + return { bucket, key: source.slice(separator + 1) }; +} + +/** Maps x-amz-copy-source-if-* headers onto standard conditional headers */ +function copySourceConditionals(c: S3Context): Headers | undefined { + let headers: Headers | undefined; + for (const standard of CONDITIONAL_HEADERS) { + const value = c.req.header(`x-amz-copy-source-${standard.toLowerCase()}`); + if (value !== undefined) { + headers ??= new Headers(); + headers.set(standard, value); + } + } + + return headers; +} + +function serveObject( + c: S3Context, + bucket: R2Bucket, + key: string +): Awaitable { + const rangeHeader = c.req.header("Range"); + const range = rangeHeader === undefined ? {} : parseRangeHeader(rangeHeader); + if ("error" in range) { + switch (range.error) { + case "malformed": + // R2's message really ends with `.'` + return errorResponse( + 400, + "InvalidArgument", + "range must be in format 'bytes=start-end', 'bytes=start-' or 'bytes=-suffix'.'" + ); + case "inverted": + return errorResponse(400, "InvalidArgument", "range must be positive."); + case "unsatisfiable": + return errorResponse( + 416, + "InvalidRange", + "The requested range is not satisfiable" + ); + } + } + + return serveR2Object( + c.req.raw, + bucket, + key, + { + notFound: noSuchKey, + preconditionFailed, + invalidRange: () => + errorResponse( + 416, + "InvalidRange", + "The requested range is not satisfiable" + ), + decorateHeaders(object, headers) { + for (const [name, value] of Object.entries( + object.customMetadata ?? {} + )) { + headers.set(`x-amz-meta-${name}`, value); + } + }, + }, + range + ); +} + +async function listObjects( + params: URLSearchParams, + bucket: R2Bucket, + bucketId: string, + v2: boolean +): Promise { + const encodingType = params.get("encoding-type"); + if (encodingType !== null && encodingType !== "url") { + return notImplemented( + `Unrecognized encoding-type "${encodingType}" not implemented` + ); + } + + const encode = (value: string) => + encodingType === "url" ? awsUriEncode(value) : value; + + // R2 floors fractional values and allows 0 and values above the limit + // (clamping the effective page size but echoing the requested MaxKeys) + let maxKeys = MAX_LIST_KEYS; + const maxKeysParam = params.get("max-keys"); + if (maxKeysParam !== null) { + // `Number("")` is 0, but R2 rejects an empty max-keys + const value = + maxKeysParam.trim() === "" ? Number.NaN : Number(maxKeysParam); + if (!Number.isFinite(value) || value < 0) { + return errorResponse( + 400, + "InvalidMaxKeys", + "MaxKeys params must be positive integer <= 1000." + ); + } + + maxKeys = Math.floor(value); + } + const limit = Math.min(maxKeys, MAX_LIST_KEYS); + + const prefix = params.get("prefix") ?? ""; + const delimiter = params.get("delimiter") ?? undefined; + const marker = v2 ? undefined : (params.get("marker") ?? undefined); + const startAfter = v2 ? (params.get("start-after") ?? undefined) : marker; + const continuationToken = v2 + ? (params.get("continuation-token") ?? undefined) + : undefined; + + let objects: R2Object[] = []; + let delimitedPrefixes: string[] = []; + let truncated = false; + let cursor: string | undefined; + let result; + try { + result = await bucket.list({ + prefix, + delimiter, + // For max-keys=0, list one key anyway: R2 reports IsTruncated based + // on whether any matching keys exist + limit: Math.max(limit, 1), + startAfter, + cursor: continuationToken, + }); + } catch (e) { + return bindingError(e); + } + + if (limit === 0) { + truncated = + result.objects.length > 0 || result.delimitedPrefixes.length > 0; + } else { + objects = result.objects; + delimitedPrefixes = result.delimitedPrefixes; + truncated = result.truncated; + cursor = result.truncated ? result.cursor : undefined; + } + + const contents = objects.map((object) => ({ + Key: encode(object.key), + Size: object.size, + LastModified: object.uploaded.toISOString(), + ETag: object.httpEtag, + // The local simulator does not store storage classes + StorageClass: "STANDARD", + })); + const commonPrefixes = delimitedPrefixes.map((value) => ({ + Prefix: encode(value), + })); + // NextMarker is the lexicographically last returned item, which can be a + // CommonPrefix (prefixes sort after the keys they group) + const lastObjectKey = objects[objects.length - 1]?.key; + const lastPrefix = delimitedPrefixes[delimitedPrefixes.length - 1]; + const lastKey = + lastPrefix !== undefined && + (lastObjectKey === undefined || lastPrefix > lastObjectKey) + ? lastPrefix + : lastObjectKey; + + return xmlResponse("ListBucketResult", { + Name: bucketId, + ...(contents.length > 0 ? { Contents: contents } : {}), + IsTruncated: truncated, + ...(commonPrefixes.length > 0 ? { CommonPrefixes: commonPrefixes } : {}), + Prefix: encode(prefix), + ...(delimiter !== undefined ? { Delimiter: encode(delimiter) } : {}), + ...(v2 + ? { + ...(startAfter !== undefined + ? { StartAfter: encode(startAfter) } + : {}), + ...(continuationToken !== undefined + ? { ContinuationToken: continuationToken } + : {}), + ...(cursor !== undefined ? { NextContinuationToken: cursor } : {}), + } + : { + Marker: encode(marker ?? ""), + ...(truncated && lastKey !== undefined + ? { NextMarker: encode(lastKey) } + : {}), + }), + MaxKeys: maxKeys, + ...(v2 ? { KeyCount: contents.length + commonPrefixes.length } : {}), + ...(encodingType !== null ? { EncodingType: encodingType } : {}), + }); +} + +function parsePartNumber(params: URLSearchParams): number | Response { + // detectObjectOperation() only routes to part operations when partNumber is + // present and integer-shaped (possibly empty, padded, or signed) + const raw = params.get("partNumber"); + assert(raw !== null); + const partNumber = raw.trim() === "" ? Number.NaN : Number(raw); + if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > 10000) { + return errorResponse( + 400, + "InvalidArgument", + "Part number must be an integer between 1 and 10000, inclusive." + ); + } + return partNumber; +} + +export const OBJECT_OPERATIONS: Record< + ObjectOperation, + OperationDefinition & ScreeningRules +> = { + GetObject: { + unsupportedHeaders: BUCKET_OWNER, + ssec: "read", + handle: ({ c, bucket, key }) => serveObject(c, bucket, key), + }, + HeadObject: { + unsupportedHeaders: BUCKET_OWNER, + ssec: "read", + handle: ({ c, bucket, key }) => serveObject(c, bucket, key), + }, + PutObject: { + unsupportedHeaders: WRITE_UNSUPPORTED, + validatesWriteHeaders: true, + ssec: "write", + async handle({ c, bucket, key }) { + const storageClass = parseStorageClass(c); + if (storageClass instanceof Response) { + return storageClass; + } + + const body = await verifiedRequestBody(c); + if (body instanceof Response) { + return body; + } + + const hasConditional = CONDITIONAL_HEADERS.some( + (name) => c.req.header(name) !== undefined + ); + const object = await bucket.put(key, body, { + httpMetadata: c.req.raw.headers, + customMetadata: collectCustomMetadata(c), + onlyIf: hasConditional ? c.req.raw.headers : undefined, + storageClass, + }); + if (object === null) { + return preconditionFailed(); + } + return c.body(null, { headers: { ETag: object.httpEtag } }); + }, + }, + CopyObject: { + unsupportedHeaders: [ + ...WRITE_UNSUPPORTED, + ...SOURCE_BUCKET_OWNER, + "x-amz-tagging-directive", + "x-amz-checksum-algorithm", + ], + validatesWriteHeaders: true, + ssec: "write", + async handle({ c, bucket, bucketId, key }) { + const directive = c.req.header("x-amz-metadata-directive") ?? "COPY"; + if (directive !== "COPY" && directive !== "REPLACE") { + return errorResponse( + 400, + "InvalidArgument", + `metadata directive ${directive}.` + ); + } + const storageClass = parseStorageClass(c); + if (storageClass instanceof Response) { + return storageClass; + } + const source = parseCopySource(c, bucketId); + if (source instanceof Response) { + return source; + } + + const sourceObject = await source.bucket.get(source.key, { + onlyIf: copySourceConditionals(c), + }); + if (sourceObject === null) { + return noSuchKey(); + } + if (!("body" in sourceObject)) { + return preconditionFailed(); + } + + const object = await bucket.put(key, sourceObject.body, { + httpMetadata: + directive === "COPY" ? sourceObject.httpMetadata : c.req.raw.headers, + customMetadata: + directive === "COPY" + ? sourceObject.customMetadata + : collectCustomMetadata(c), + storageClass, + }); + if (object === null) { + return preconditionFailed(); + } + return xmlResponse("CopyObjectResult", { + ETag: object.httpEtag, + LastModified: object.uploaded.toISOString(), + }); + }, + }, + DeleteObject: { + unsupportedHeaders: [...BUCKET_OWNER, ...MFA_AND_LOCK_BYPASS], + async handle({ c, bucket, key }) { + await bucket.delete(key); + return c.body(null, 204); + }, + }, + CreateMultipartUpload: { + unsupportedHeaders: WRITE_UNSUPPORTED, + validatesWriteHeaders: true, + ssec: "write", + async handle({ c, bucket, bucketId, key }) { + const storageClass = parseStorageClass(c); + if (storageClass instanceof Response) { + return storageClass; + } + const upload = await bucket.createMultipartUpload(key, { + httpMetadata: c.req.raw.headers, + customMetadata: collectCustomMetadata(c), + storageClass, + }); + return xmlResponse("InitiateMultipartUploadResult", { + UploadId: upload.uploadId, + Bucket: bucketId, + Key: key, + }); + }, + }, +}; + +/** Operations on an in-progress multipart upload (?uploadId) */ +export const MULTIPART_OPERATIONS: Record< + MultipartOperation, + OperationDefinition & ScreeningRules +> = { + UploadPart: { + unsupportedHeaders: [...BUCKET_OWNER, "x-amz-server-side-encryption"], + ssec: "write", + async handle({ c, bucket, key, uploadId, params }) { + const partNumber = parsePartNumber(params); + if (partNumber instanceof Response) { + return partNumber; + } + + const body = await verifiedRequestBody(c); + if (body instanceof Response) { + return body; + } + + // `resumeMultipartUpload` never validates: it just constructs the + // upload object; an unknown id only fails once the upload is used + const upload = bucket.resumeMultipartUpload(key, uploadId); + try { + const part = await upload.uploadPart(partNumber, body); + return c.body(null, { headers: { ETag: `"${part.etag}"` } }); + } catch (e) { + return bindingError(e); + } + }, + }, + UploadPartCopy: { + unsupportedHeaders: [ + ...BUCKET_OWNER, + ...SOURCE_BUCKET_OWNER, + ...COPY_SOURCE_CONDITIONALS, + ], + ssec: "write", + async handle({ c, bucket, bucketId, key, uploadId, params }) { + const partNumber = parsePartNumber(params); + if (partNumber instanceof Response) { + return partNumber; + } + const source = parseCopySource(c, bucketId); + if (source instanceof Response) { + return source; + } + let range: R2Range | undefined; + const rangeHeader = c.req.header("x-amz-copy-source-range"); + if (rangeHeader !== undefined) { + const match = /^bytes=(\d+)-(\d+)$/.exec(rangeHeader); + if (match === null) { + return errorResponse( + 400, + "InvalidArgument", + `Invalid x-amz-copy-source-range: ${rangeHeader}` + ); + } + const offset = Number(match[1]); + const end = Number(match[2]); + // R2 clamps out-of-bounds ends (the simulator does too) but + // rejects inverted ranges + if (end < offset) { + return errorResponse( + 400, + "InvalidArgument", + "x-amz-copy-source-range must be positive." + ); + } + range = { offset, length: end - offset + 1 }; + } + + let sourceObject; + try { + sourceObject = await source.bucket.get(source.key, { range }); + } catch (e) { + return bindingError(e); + } + if (sourceObject === null) { + return noSuchKey(); + } + + const upload = bucket.resumeMultipartUpload(key, uploadId); + try { + const part = await upload.uploadPart(partNumber, sourceObject.body); + return xmlResponse("CopyPartResult", { + ETag: `"${part.etag}"`, + LastModified: new Date().toISOString(), + }); + } catch (e) { + return bindingError(e); + } + }, + }, + CompleteMultipartUpload: { + unsupportedHeaders: BUCKET_OWNER, + async handle({ c, bucket, bucketId, key, uploadId }) { + const text = await c.req.raw.text(); + if (XMLValidator.validate(text) !== true) { + return malformedXml(); + } + + const parsed: unknown = xmlParser.parse(text); + const request = ( + parsed as { CompleteMultipartUpload?: { Part?: unknown } } + ).CompleteMultipartUpload; + if (request === undefined) { + return malformedXml(); + } + + const parts: R2UploadedPart[] = []; + const seenPartNumbers = new Set(); + for (const part of coerceArray(request.Part)) { + const { PartNumber, ETag } = part as { + PartNumber?: unknown; + ETag?: unknown; + }; + + const partNumber = Number(PartNumber); + if (!Number.isInteger(partNumber) || typeof ETag !== "string") { + return malformedXml(); + } + + // Real R2 treats out-of-range part numbers in the XML as parts that + // cannot exist, not as malformed arguments (unlike ?partNumber=) + if (partNumber < 1 || partNumber > 10000) { + return errorResponse( + 400, + "InvalidPart", + "One or more of the specified parts could not be found." + ); + } + + // The simulator reports duplicate part numbers with the same + // internal error (10001) as an unknown upload id, so screen them + // here with the error real R2 returns for duplicates + if (seenPartNumbers.has(partNumber)) { + return errorResponse( + 400, + "InvalidPart", + "There was a problem with the multipart upload." + ); + } + seenPartNumbers.add(partNumber); + + parts.push({ partNumber, etag: ETag.replace(/^"|"$/g, "") }); + } + + if (parts.length === 0) { + return malformedXml(); + } + + const upload = bucket.resumeMultipartUpload(key, uploadId); + try { + const object = await upload.complete(parts); + const url = new URL(c.req.url); + return xmlResponse("CompleteMultipartUploadResult", { + Bucket: bucketId, + Key: key, + ETag: object.httpEtag, + Location: `${url.origin}${CorePaths.R2_S3}/${bucketId}/${awsUriEncode(key)}`, + }); + } catch (e) { + const response = bindingError(e); + + // Like abort, the simulator reports an unknown upload id on + // complete as an internal error (10001) rather than + // NO_SUCH_UPLOAD. Duplicate part numbers, its only other 10001 + // source here, are screened during parsing above. + if (response.status === 500) { + return s3Error(NO_SUCH_UPLOAD); + } + + return response; + } + }, + }, + AbortMultipartUpload: { + unsupportedHeaders: [], + async handle({ c, bucket, key, uploadId }) { + const upload = bucket.resumeMultipartUpload(key, uploadId); + try { + await upload.abort(); + } catch (e) { + const response = bindingError(e); + // The simulator reports an unknown upload id on abort as an + // internal error (10001) rather than NO_SUCH_UPLOAD; aborting is + // its only failure mode, so map it onto the NoSuchUpload R2's S3 + // API returns + if (response.status === 500) { + return s3Error(NO_SUCH_UPLOAD); + } + return response; + } + return c.body(null, 204); + }, + }, +}; + +export const BUCKET_OPERATIONS: Record< + BucketOperation, + OperationDefinition & ScreeningRules +> = { + HeadBucket: { + unsupportedHeaders: BUCKET_OWNER, + // The bucket exists, or routing would have 404ed already + handle: ({ c }) => c.body(null, 200), + }, + GetBucketLocation: { + unsupportedHeaders: BUCKET_OWNER, + // Real R2 reports the bucket's location hint (e.g. ENAM); local + // buckets have no location + handle: () => xmlResponse("LocationConstraint", { "#text": "auto" }), + }, + // R2 always encrypts at rest with AES256; there is no per-bucket config + GetBucketEncryption: { + unsupportedHeaders: BUCKET_OWNER, + handle: () => + xmlResponse("ServerSideEncryptionConfiguration", { + Rule: { + ApplyServerSideEncryptionByDefault: { SSEAlgorithm: "AES256" }, + BucketKeyEnabled: true, + }, + }), + }, + // R2 has no bucket versioning, tagging, object lock, or replication; + // these responses are identical for every bucket + GetBucketVersioning: { + unsupportedHeaders: BUCKET_OWNER, + handle: () => xmlResponse("VersioningConfiguration", {}), + }, + GetBucketTagging: { + unsupportedHeaders: BUCKET_OWNER, + handle: () => + errorResponse(404, "NoSuchTagSet", "The TagSet does not exist."), + }, + GetObjectLockConfiguration: { + unsupportedHeaders: BUCKET_OWNER, + handle: () => + errorResponse( + 404, + "ObjectLockConfigurationNotFoundError", + "Object Lock configuration does not exist for this bucket." + ), + }, + GetBucketReplication: { + unsupportedHeaders: BUCKET_OWNER, + handle: () => + errorResponse( + 404, + "ReplicationConfigurationNotFoundError", + "The replication configuration was not found." + ), + }, + ListObjects: { + unsupportedHeaders: BUCKET_OWNER, + handle: ({ params, bucket, bucketId }) => + listObjects(params, bucket, bucketId, false), + }, + ListObjectsV2: { + unsupportedHeaders: BUCKET_OWNER, + handle: ({ params, bucket, bucketId }) => + listObjects(params, bucket, bucketId, true), + }, + DeleteObjects: { + unsupportedHeaders: [...BUCKET_OWNER, ...MFA_AND_LOCK_BYPASS], + async handle({ c, bucket }) { + const body = await c.req.raw.arrayBuffer(); + const digestError = await verifyContentMD5(c, body); + if (digestError !== undefined) { + return digestError; + } + + const text = new TextDecoder().decode(body); + if (XMLValidator.validate(text) !== true) { + return malformedXml(); + } + + const parsed: unknown = xmlParser.parse(text); + const request = ( + parsed as { Delete?: { Object?: unknown; Quiet?: unknown } } + ).Delete; + if (request === undefined) { + return malformedXml(); + } + + const keys: string[] = []; + for (const object of coerceArray(request.Object)) { + const key = (object as { Key?: unknown }).Key; + if (typeof key !== "string") { + return malformedXml(); + } + + keys.push(key); + } + + if (keys.length === 0 || keys.length > MAX_DELETE_KEYS) { + return malformedXml(); + } + + // R2 validates Quiet strictly but then ignores it. + // The Deleted list is returned even in quiet mode. + if ( + request.Quiet !== undefined && + request.Quiet !== "true" && + request.Quiet !== "false" + ) { + return malformedXml(); + } + + await bucket.delete(keys); + + // Deletes are idempotent: missing keys are still reported as Deleted + return xmlResponse("DeleteResult", { + Deleted: keys.map((key) => ({ Key: key })), + }); + }, + }, +}; diff --git a/packages/miniflare/src/workers/r2/serve.worker.ts b/packages/miniflare/src/workers/r2/serve.worker.ts new file mode 100644 index 0000000000..1bdacf42ea --- /dev/null +++ b/packages/miniflare/src/workers/r2/serve.worker.ts @@ -0,0 +1,164 @@ +// Accept only a single range with start <= end; reject anything else +// (including multiple ranges) with an endpoint-specific 400 rather than +// ignoring it +const RANGE_HEADER = /^bytes=(?:(\d+)-(\d+)?|-(\d+))$/; + +export type ParsedRangeHeader = + | { error: "malformed" | "inverted" | "unsatisfiable" } + // `start` is undefined for suffix ranges (`bytes=-N`) + | { start?: number }; + +export function parseRangeHeader(header: string): ParsedRangeHeader { + const match = RANGE_HEADER.exec(header); + if (match === null) { + return { error: "malformed" }; + } + const [, start, end, suffix] = match; + if (start === undefined) { + // A zero suffix length (`bytes=-0`) is unsatisfiable for any object; + // the simulator would ignore the range and serve the full body + return Number(suffix) === 0 ? { error: "unsatisfiable" } : {}; + } + if (end !== undefined && Number(start) > Number(end)) { + return { error: "inverted" }; + } + return { start: Number(start) }; +} + +function objectHeaders(object: R2Object): Headers { + const headers = new Headers(); + object.writeHttpMetadata(headers); + headers.set("ETag", object.httpEtag); + headers.set("Last-Modified", object.uploaded.toUTCString()); + headers.set("Accept-Ranges", "bytes"); + return headers; +} + +export interface ServeHandlers { + notFound(): Response | Promise; + preconditionFailed(): Response; + /** + * When provided, a range starting at or beyond the object size returns + * this error (like R2's S3 API). When omitted, the simulator's behavior + * of clamping the range applies. + */ + invalidRange?(): Response; + /** Adds endpoint-specific headers to successful (2xx/304) responses */ + decorateHeaders?(object: R2Object, headers: Headers): void; +} + +export async function serveR2Object( + request: Request, + bucket: R2Bucket, + key: string, + handlers: ServeHandlers, + /** The caller's parse of the request's Range header, error-screened */ + parsedRange?: ParsedRangeHeader +): Promise { + // R2 honors Range on HEAD too (206 + Content-Range, no body) + const rangeHeader = request.headers.get("Range"); + const hasRange = rangeHeader !== null; + + // `bucket.head()` cannot evaluate conditional headers (the R2 head + // operation only carries the key), so HEAD also uses `bucket.get()` and + // discards the body. + const object = await bucket.get(key, { + onlyIf: request.headers, + range: hasRange ? request.headers : undefined, + }); + + if (object === null) { + return handlers.notFound(); + } + + const headers = objectHeaders(object); + handlers.decorateHeaders?.(object, headers); + + if (!("body" in object)) { + // Some conditional header failed, but `bucket.get()` reports the + // failure without naming the header. We need to determine which header + // failed to determine the status code to return. + // + // https://datatracker.ietf.org/doc/html/rfc7232#section-6 gives the + // order for checking headers. We know at least one header failed. + // We must first check for a precondition header failure. + // + // The logic in `_testR2Conditional` ensures we can simultaneously + // check both "If-Match" and "If-Unmodified-Since" (since a failure in + // "If-Unmodified-Since" can be suppressed by success for a present + // "If-Match"). These both yield status 412s upon failure. + let preconditions: Headers | undefined; + for (const name of ["If-Match", "If-Unmodified-Since"]) { + const value = request.headers.get(name); + if (value !== null) { + preconditions ??= new Headers(); + preconditions.set(name, value); + } + } + if (preconditions !== undefined) { + const recheck = await bucket.get(key, { onlyIf: preconditions }); + if (recheck === null) { + return handlers.notFound(); + } + if (!("body" in recheck)) { + return handlers.preconditionFailed(); + } + // An unread recheck body would hold a read stream open + void recheck.body.cancel(); + } + + // Otherwise, the preconditions hold, so the failure came from a cache + // validator. + return new Response(null, { status: 304, headers }); + } + + const body = request.method === "HEAD" ? null : object.body; + if (body === null) { + // An unread body would hold a read stream open + void object.body.cancel(); + } + + const range = object.range; + if (hasRange && range !== undefined) { + // The simulator clamps out-of-bounds ranges and serves a zero-length + // range for empty objects; R2 rejects both with 416 (any range on a + // 0-byte object, or a range starting at or beyond the object size) + if (handlers.invalidRange !== undefined) { + const parsed = parsedRange ?? parseRangeHeader(rangeHeader); + if ( + !("error" in parsed) && + (object.size === 0 || + (parsed.start !== undefined && parsed.start >= object.size)) + ) { + if (body !== null) { + void body.cancel(); + } + return handlers.invalidRange(); + } + } + // The returned range may carry all keys with some `undefined` (e.g. + // `suffix` present but undefined on an offset range), so normalize by + // value rather than by key presence + const normalized: { offset?: number; length?: number; suffix?: number } = { + ...range, + }; + let offset: number; + let length: number; + if (normalized.suffix !== undefined) { + length = Math.min(normalized.suffix, object.size); + offset = object.size - length; + } else { + offset = normalized.offset ?? 0; + length = normalized.length ?? object.size - offset; + } + headers.set( + "Content-Range", + `bytes ${offset}-${offset + length - 1}/${object.size}` + ); + headers.set("Content-Length", `${length}`); + return new Response(body, { status: 206, headers }); + } + + headers.set("Content-Length", `${object.size}`); + return new Response(body, { headers }); +} diff --git a/packages/miniflare/test/plugins/r2/public.spec.ts b/packages/miniflare/test/plugins/r2/public.spec.ts index 9424731c30..2933778727 100644 --- a/packages/miniflare/test/plugins/r2/public.spec.ts +++ b/packages/miniflare/test/plugins/r2/public.spec.ts @@ -102,6 +102,12 @@ test("decodes percent-encoded keys", async ({ expect }) => { expect(res.status).toBe(200); expect(await res.text()).toBe("nested"); + + // Keys containing `%` must be decoded exactly once + await r2.put("100%/a%2Bb.txt", "percent"); + const percent = await fetch(bucketUrl("/100%25/a%252Bb.txt", ctx.url)); + expect(percent.status).toBe(200); + expect(await percent.text()).toBe("percent"); }); test("GET supports range requests", async ({ expect }) => { @@ -118,6 +124,35 @@ test("GET supports range requests", async ({ expect }) => { expect(res.headers.get("Content-Length")).toBe("4"); }); +test("GET honors suffix ranges with a partial 206", async ({ expect }) => { + const r2 = await ctx.mf.getR2Bucket("BUCKET"); + await r2.put("suffix-range-key", "0123456789"); + + const res = await fetch(bucketUrl("/suffix-range-key", ctx.url), { + headers: { Range: "bytes=-4" }, + }); + + expect(res.status).toBe(206); + expect(await res.text()).toBe("6789"); + expect(res.headers.get("Content-Range")).toBe("bytes 6-9/10"); + expect(res.headers.get("Content-Length")).toBe("4"); +}); + +test("HEAD honors Range with a bodyless 206", async ({ expect }) => { + const r2 = await ctx.mf.getR2Bucket("BUCKET"); + await r2.put("head-range-key", "0123456789"); + + const res = await fetch(bucketUrl("/head-range-key", ctx.url), { + method: "HEAD", + headers: { Range: "bytes=0-3" }, + }); + + expect(res.status).toBe(206); + expect(res.headers.get("Content-Range")).toBe("bytes 0-3/10"); + expect(res.headers.get("Content-Length")).toBe("4"); + expect(await res.text()).toBe(""); +}); + test("HEAD returns headers without a body", async ({ expect }) => { const r2 = await ctx.mf.getR2Bucket("BUCKET"); await r2.put("head-key", "abcdef", { @@ -144,9 +179,7 @@ test("HEAD returns 404 for a missing key", async ({ expect }) => { await res.arrayBuffer(); }); -test("rejects write methods with 405 and an Allow header", async ({ - expect, -}) => { +test("rejects write methods with 401", async ({ expect }) => { const r2 = await ctx.mf.getR2Bucket("BUCKET"); await r2.put("readonly-key", "untouched"); @@ -155,8 +188,7 @@ test("rejects write methods with 405 and an Allow header", async ({ method, body: method === "DELETE" ? undefined : "tampered", }); - expect(res.status, `${method} should be rejected`).toBe(405); - expect(res.headers.get("Allow")).toBe("GET, HEAD, OPTIONS"); + expect(res.status, `${method} should be rejected`).toBe(401); await res.arrayBuffer(); } @@ -164,6 +196,46 @@ test("rejects write methods with 405 and an Allow header", async ({ expect(await after?.text()).toBe("untouched"); }); +test("rejects malformed and multiple ranges with 400", async ({ expect }) => { + const r2 = await ctx.mf.getR2Bucket("BUCKET"); + await r2.put("bad-range-key", "0123456789"); + + for (const range of ["bytes=zzz", "bytes=0-1,3-4", "0-3", "bytes=5-2"]) { + const res = await fetch(bucketUrl("/bad-range-key", ctx.url), { + headers: { Range: range }, + }); + expect(res.status, `"${range}" should be rejected`).toBe(400); + await res.arrayBuffer(); + } +}); + +test("rejects unsatisfiable ranges with 416", async ({ expect }) => { + const r2 = await ctx.mf.getR2Bucket("BUCKET"); + await r2.put("unsat-range-key", "0123456789"); + + const res = await fetch(bucketUrl("/unsat-range-key", ctx.url), { + headers: { Range: "bytes=99999-" }, + }); + expect(res.status).toBe(416); + await res.arrayBuffer(); + + // A zero suffix length is unsatisfiable for any object + const zeroSuffix = await fetch(bucketUrl("/unsat-range-key", ctx.url), { + headers: { Range: "bytes=-0" }, + }); + expect(zeroSuffix.status).toBe(416); + await zeroSuffix.arrayBuffer(); + + // Any range on a zero-length object is unsatisfiable, including a suffix + // range (which the simulator would otherwise serve as a zero-length 206) + await r2.put("empty-range-key", ""); + const emptySuffix = await fetch(bucketUrl("/empty-range-key", ctx.url), { + headers: { Range: "bytes=-5" }, + }); + expect(emptySuffix.status).toBe(416); + await emptySuffix.arrayBuffer(); +}); + // The entry worker rejects /cdn-cgi/* requests from non-localhost origins // before they reach this worker, so cross-origin here means a different // localhost port (e.g. a frontend dev server). diff --git a/packages/miniflare/test/plugins/r2/s3.spec.ts b/packages/miniflare/test/plugins/r2/s3.spec.ts new file mode 100644 index 0000000000..76619378ec --- /dev/null +++ b/packages/miniflare/test/plugins/r2/s3.spec.ts @@ -0,0 +1,2162 @@ +import crypto from "node:crypto"; +import { Sha256 } from "@aws-crypto/sha256-js"; +import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CopyObjectCommand, + CreateMultipartUploadCommand, + DeleteObjectCommand, + DeleteObjectsCommand, + GetObjectCommand, + GetBucketEncryptionCommand, + GetBucketLocationCommand, + GetBucketVersioningCommand, + HeadBucketCommand, + HeadObjectCommand, + ListMultipartUploadsCommand, + ListBucketsCommand, + ListObjectsCommand, + ListObjectsV2Command, + ListPartsCommand, + PutObjectCommand, + S3Client, + S3ServiceException, + UploadPartCommand, + UploadPartCopyCommand, +} from "@aws-sdk/client-s3"; +import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; +import { SignatureV4 } from "@smithy/signature-v4"; +import { Miniflare } from "miniflare"; +import { assert, onTestFinished, test } from "vitest"; +import { miniflareTest } from "../../test-shared"; +import type { MiniflareTestContext } from "../../test-shared"; +import type { R2Bucket } from "@cloudflare/workers-types/experimental"; +import type { ExpectStatic } from "vitest"; + +// Operations are exercised through the official AWS SDK to prove client +// interop. Requests the SDK cannot produce (forged payload hashes, malformed +// bodies, unsupported headers) are signed directly with @smithy/signature-v4 +// via `s3Fetch()`. + +const CREDENTIALS = { + accessKeyId: "A".repeat(32), + secretAccessKey: "test-secret-key", +}; +const THIRD_CREDENTIALS = { + accessKeyId: "C".repeat(32), + secretAccessKey: "third-secret-key", +}; + +const ctx = miniflareTest<{ BUCKET: R2Bucket }, MiniflareTestContext>( + { + r2Buckets: { + BUCKET: { id: "bucket", s3Credentials: CREDENTIALS }, + OTHER: { id: "other-bucket", s3Credentials: CREDENTIALS }, + THIRD: { id: "third-bucket", s3Credentials: THIRD_CREDENTIALS }, + }, + }, + async (global) => new global.Response(null, { status: 404 }) +); + +function s3Url(path: string): URL { + return new URL(`/cdn-cgi/local/r2/s3/${path}`, ctx.url); +} + +function s3(options: { credentials?: typeof CREDENTIALS } = {}): S3Client { + const client = new S3Client({ + region: "auto", + endpoint: s3Url("").href, + credentials: options.credentials ?? CREDENTIALS, + forcePathStyle: true, + }); + + // The SDK sends `Expect: 100-continue` on requests with bodies, but + // workerd never responds with `100 Continue`, so the SDK would wait for it + // indefinitely before sending the body + client.middlewareStack.remove("addExpectContinueMiddleware"); + onTestFinished(() => client.destroy()); + return client; +} + +async function expectSdkError( + promise: Promise, + status: number, + code: string, + expect: ExpectStatic +) { + const error = await promise.then( + () => undefined, + (e: unknown) => e + ); + assert(error instanceof S3ServiceException); + expect(error.$metadata.httpStatusCode).toBe(status); + expect(error.name).toBe(code); + return error; +} + +function bucket() { + return ctx.mf.getR2Bucket("BUCKET"); +} +function sha256Hex(data: string | Uint8Array): string { + return crypto.createHash("sha256").update(data).digest("hex"); +} + +function toAmzDate(date: Date): string { + return date + .toISOString() + .replace(/[-:]/g, "") + .replace(/\.\d{3}/, ""); +} + +interface S3FetchOptions { + method?: string; + headers?: Record; + body?: string | Uint8Array; + /** Overrides the signed payload hash (for mismatch tests) */ + payloadHash?: string; + credentials?: typeof CREDENTIALS; +} + +async function s3Fetch(path: string, opts: S3FetchOptions = {}) { + const url = s3Url(path); + const method = opts.method ?? "GET"; + const body = opts.body ?? ""; + + const signer = new SignatureV4({ + credentials: opts.credentials ?? CREDENTIALS, + region: "auto", + service: "s3", + sha256: Sha256, + // S3 canonical URIs are single-encoded (the path as sent) + uriEscapePath: false, + applyChecksum: true, + }); + const query: Record = {}; + for (const [name, value] of url.searchParams) { + (query[name] ??= []).push(value); + } + const signed = await signer.sign({ + method, + protocol: url.protocol, + hostname: url.hostname, + port: url.port === "" ? undefined : Number(url.port), + path: url.pathname, + query, + headers: { + host: url.host, + ...opts.headers, + // A pre-set payload hash is signed as-is, so a wrong hash here + // makes the signature cover a hash the body won't match + ...(opts.payloadHash !== undefined && { + "x-amz-content-sha256": opts.payloadHash, + }), + }, + body, + }); + + return fetch(url, { + method, + headers: signed.headers, + body: method === "GET" || method === "HEAD" ? undefined : body, + }); +} + +async function expectError( + res: Response, + status: number, + code: string, + expect: ExpectStatic +) { + expect(res.status).toBe(status); + expect(await res.text()).toContain(`${code}`); +} + +test("rejects anonymous requests", async ({ expect }) => { + const res = await fetch(s3Url("bucket/key.txt")); + expect(res.status).toBe(400); + expect(await res.text()).toContain( + "InvalidArgumentAuthorization" + ); +}); + +test("rejects a wrong secret with SignatureDoesNotMatch", async ({ + expect, +}) => { + const client = s3({ + credentials: { ...CREDENTIALS, secretAccessKey: "wrong" }, + }); + const error = await expectSdkError( + client.send(new GetObjectCommand({ Bucket: "bucket", Key: "key.txt" })), + 403, + "SignatureDoesNotMatch", + expect + ); + expect(error.message).toBe( + "The request signature we calculated does not match the signature you provided. Check your secret access key and signing method." + ); +}); + +test("rejects an unknown access key id with 401 Unauthorized", async ({ + expect, +}) => { + const client = s3({ + credentials: { ...CREDENTIALS, accessKeyId: "B".repeat(32) }, + }); + await expectSdkError( + client.send(new GetObjectCommand({ Bucket: "bucket", Key: "key.txt" })), + 401, + "Unauthorized", + expect + ); +}); + +test("accepts valid header auth", async ({ expect }) => { + await expectSdkError( + s3().send(new GetObjectCommand({ Bucket: "bucket", Key: "missing.txt" })), + 404, + "NoSuchKey", + expect + ); +}); + +test("requires x-amz-content-sha256", async ({ expect }) => { + const url = s3Url("bucket/key.txt"); + const res = await fetch(url, { + headers: { authorization: "AWS4-HMAC-SHA256 garbage" }, + }); + expect(res.status).toBe(400); + expect(await res.text()).toContain("Missing x-amz-content-sha256"); +}); + +test("rejects a skewed request time", async ({ expect }) => { + const url = s3Url("bucket/key.txt"); + const amzDate = toAmzDate(new Date(Date.now() - 3600_000)); + const res = await fetch(url, { + headers: { + authorization: `AWS4-HMAC-SHA256 Credential=${CREDENTIALS.accessKeyId}/${amzDate.slice(0, 8)}/auto/s3/aws4_request, SignedHeaders=host, Signature=${"0".repeat(64)}`, + "x-amz-date": amzDate, + "x-amz-content-sha256": "UNSIGNED-PAYLOAD", + }, + }); + await expectError(res, 403, "RequestTimeTooSkewed", expect); +}); + +test("reports date errors like R2", async ({ expect }) => { + const url = s3Url("bucket/key.txt"); + const amzDate = toAmzDate(new Date()); + const day = amzDate.slice(0, 8); + const sha = { "x-amz-content-sha256": "UNSIGNED-PAYLOAD" }; + const authorization = `AWS4-HMAC-SHA256 Credential=${CREDENTIALS.accessKeyId}/${day}/auto/s3/aws4_request, SignedHeaders=host;x-amz-date, Signature=${"0".repeat(64)}`; + + const noDate = await fetch(url, { headers: { ...sha, authorization } }); + expect(noDate.status).toBe(400); + expect(await noDate.text()).toContain( + "No date provided in x-amz-date nor date header" + ); + + // The `date` header is a fallback, but accepts only ISO 8601 basic + // format too (RFC 1123 dates are rejected) + const rfc1123Date = new Date().toUTCString(); + const rfc1123 = await fetch(url, { + headers: { ...sha, authorization, date: rfc1123Date }, + }); + expect(rfc1123.status).toBe(400); + expect(await rfc1123.text()).toContain( + `Date provided in 'date' header (${rfc1123Date}) didn't parse successfully` + ); + + const extendedIso = await fetch(url, { + headers: { ...sha, authorization, "x-amz-date": "2026-06-12T00:00:00Z" }, + }); + expect(extendedIso.status).toBe(400); + expect(await extendedIso.text()).toContain( + "Date provided in 'x-amz-date' header (2026-06-12T00:00:00Z) didn't parse successfully" + ); +}); + +test("reports credential scope errors like R2", async ({ expect }) => { + const url = s3Url("bucket/key.txt"); + const amzDate = toAmzDate(new Date()); + const day = amzDate.slice(0, 8); + const headers = (authorization: string) => ({ + "x-amz-content-sha256": "UNSIGNED-PAYLOAD", + "x-amz-date": amzDate, + authorization, + }); + const auth = (credential: string, signedHeaders = "host;x-amz-date") => + `AWS4-HMAC-SHA256 Credential=${credential}, SignedHeaders=${signedHeaders}, Signature=${"0".repeat(64)}`; + + // Non-SigV4 schemes and missing SigV4 fields get the algorithm error + const basic = await fetch(url, { headers: headers("Basic dXNlcjpwYXNz") }); + expect(basic.status).toBe(400); + expect(await basic.text()).toContain( + "InvalidRequestPlease use AWS4-HMAC-SHA256" + ); + + const short = await fetch(url, { + headers: headers(auth(`${CREDENTIALS.accessKeyId}/${day}/auto`)), + }); + expect(short.status).toBe(400); + expect(await short.text()).toContain( + "Credential sigv4 header should have at least 5 slash-separated parts, not 3" + ); + + const service = await fetch(url, { + headers: headers( + auth(`${CREDENTIALS.accessKeyId}/${day}/auto/sqs/aws4_request`) + ), + }); + expect(service.status).toBe(400); + expect(await service.text()).toContain( + "Credential service should be s3, not sqs" + ); + + const terminator = await fetch(url, { + headers: headers(auth(`${CREDENTIALS.accessKeyId}/${day}/auto/s3/wat`)), + }); + expect(terminator.status).toBe(400); + expect(await terminator.text()).toContain( + "Credential termination string should be aws4_request, not wat" + ); + + const staleScope = await fetch(url, { + headers: headers( + auth(`${CREDENTIALS.accessKeyId}/19990101/auto/s3/aws4_request`) + ), + }); + expect(staleScope.status).toBe(400); + expect(await staleScope.text()).toContain( + `Credential signed date 19990101 does not match ${day} from 'x-amz-date' header` + ); + + // SignedHeaders must include host + const noHost = await fetch(url, { + headers: headers( + auth( + `${CREDENTIALS.accessKeyId}/${day}/auto/s3/aws4_request`, + "x-amz-date" + ) + ), + }); + await expectError(noHost, 401, "Unauthorized", expect); +}); + +test("rejects x-amz-security-token", async ({ expect }) => { + const res = await s3Fetch("bucket/key.txt", { + headers: { "x-amz-security-token": "bogus" }, + }); + expect(res.status).toBe(400); + expect(await res.text()).toContain("X-Amz-Security-Token"); +}); + +test("detects a payload hash mismatch", async ({ expect }) => { + const res = await s3Fetch("bucket/hash.txt", { + method: "PUT", + body: "actual body", + payloadHash: sha256Hex("a different body"), + }); + await expectError(res, 400, "XAmzContentSHA256Mismatch", expect); + + // The UNSIGNED-PAYLOAD sentinel skips body verification entirely + const unsigned = await s3Fetch("bucket/hash.txt", { + method: "PUT", + body: "any body at all", + payloadHash: "UNSIGNED-PAYLOAD", + }); + expect(unsigned.status).toBe(200); +}); + +test("serves presigned GETs", async ({ expect }) => { + const r2 = await bucket(); + await r2.put("presigned.txt", "presigned content"); + const url = await getSignedUrl( + s3(), + new GetObjectCommand({ Bucket: "bucket", Key: "presigned.txt" }) + ); + const res = await fetch(url); + expect(res.status).toBe(200); + expect(await res.text()).toBe("presigned content"); +}); + +test("supports presigned PUTs", async ({ expect }) => { + const url = await getSignedUrl( + s3(), + new PutObjectCommand({ Bucket: "bucket", Key: "presigned-put.txt" }) + ); + const res = await fetch(url, { method: "PUT", body: "uploaded" }); + expect(res.status).toBe(200); + const r2 = await bucket(); + const object = await r2.get("presigned-put.txt"); + assert(object !== null); + expect(await object.text()).toBe("uploaded"); +}); + +test("rejects an empty X-Amz-Expires as not a number", async ({ expect }) => { + const url = s3Url("bucket/key.txt"); + url.search = new URLSearchParams({ + "X-Amz-Algorithm": "AWS4-HMAC-SHA256", + "X-Amz-Date": "20260612T000000Z", + "X-Amz-Expires": "", + "X-Amz-SignedHeaders": "host", + "X-Amz-Signature": "abc", + "X-Amz-Credential": `${CREDENTIALS.accessKeyId}/20260612/auto/s3/aws4_request`, + }).toString(); + const res = await fetch(url); + expect(res.status).toBe(400); + expect(await res.text()).toContain( + "X-Amz-Expires should be a number" + ); +}); + +test("rejects expired presigned URLs", async ({ expect }) => { + const url = await getSignedUrl( + s3(), + new GetObjectCommand({ Bucket: "bucket", Key: "presigned.txt" }), + { expiresIn: 60, signingDate: new Date(Date.now() - 3600_000) } + ); + const res = await fetch(url); + await expectError(res, 403, "ExpiredRequest", expect); +}); + +test("rejects presigned URLs with too-long expiry", async ({ expect }) => { + const signed = await getSignedUrl( + s3(), + new GetObjectCommand({ Bucket: "bucket", Key: "presigned.txt" }) + ); + const url = new URL(signed); + url.searchParams.set("X-Amz-Expires", "700000"); + const res = await fetch(url); + expect(res.status).toBe(400); + expect(await res.text()).toContain("X-Amz-Expires must be less than a week"); +}); + +test("rejects tampered presigned URLs", async ({ expect }) => { + const signed = await getSignedUrl( + s3(), + new GetObjectCommand({ Bucket: "bucket", Key: "presigned.txt" }) + ); + const url = new URL(signed); + url.pathname = url.pathname.replace("presigned", "evil"); + const res = await fetch(url); + await expectError(res, 403, "SignatureDoesNotMatch", expect); + // (re-fetch since expectError consumed the body) + const body = await (await fetch(url)).text(); + expect(body).toContain(""); + expect(body).toContain(""); + expect(body).toContain(""); +}); + +test("reports missing presigned parameters together", async ({ expect }) => { + const url = s3Url("bucket/key.txt"); + url.searchParams.set( + "X-Amz-Credential", + `${CREDENTIALS.accessKeyId}/20260611/auto/s3/aws4_request` + ); + url.searchParams.set("X-Amz-Date", "20260611T000000Z"); + url.searchParams.set("X-Amz-Expires", "60"); + url.searchParams.set("X-Amz-SignedHeaders", "host"); + const res = await fetch(url); + expect(res.status).toBe(400); + expect(await res.text()).toContain( + "Required search parameters X-Amz-Algorithm, X-Amz-Signature missing" + ); +}); + +test("returns NoSuchBucket for an unknown bucket id", async ({ expect }) => { + const res = await s3Fetch("not-a-bucket/key.txt"); + await expectError(res, 404, "NoSuchBucket", expect); + + // Unlike real R2 (where credentials are account-scoped and auth is + // verified first), local credentials are per-bucket, so an unknown + // bucket reports NoSuchBucket regardless of the signature + const forged = await s3Fetch("not-a-bucket/key.txt", { + credentials: { ...CREDENTIALS, secretAccessKey: "wrong-secret" }, + }); + await expectError(forged, 404, "NoSuchBucket", expect); + + // HEAD errors carry the status but no body + const head = await s3Fetch("not-a-bucket/key.txt", { method: "HEAD" }); + expect(head.status).toBe(404); + expect(await head.text()).toBe(""); +}); + +test("PutObject stores body, metadata, and returns the ETag", async ({ + expect, +}) => { + const client = s3(); + const put = await client.send( + new PutObjectCommand({ + Bucket: "bucket", + Key: "put.txt", + Body: "0123456789", + ContentType: "text/markdown", + CacheControl: "max-age=60", + Metadata: { hello: "world" }, + }) + ); + expect(put.ETag).toBe( + `"${crypto.createHash("md5").update("0123456789").digest("hex")}"` + ); + + const get = await client.send( + new GetObjectCommand({ Bucket: "bucket", Key: "put.txt" }) + ); + assert(get.Body !== undefined); + expect(await get.Body.transformToString()).toBe("0123456789"); + expect(get.ContentType).toBe("text/markdown"); + expect(get.CacheControl).toBe("max-age=60"); + expect(get.Metadata).toEqual({ hello: "world" }); + expect(get.AcceptRanges).toBe("bytes"); +}); + +test("round-trips special-character keys", async ({ expect }) => { + // Exercises canonical-URI handling: the S3 canonical URI is the + // percent-encoded path exactly as sent, never re-encoded + const key = "späcial dir/key with spaces+(parens)&'quote.txt"; + const client = s3(); + await client.send( + new PutObjectCommand({ Bucket: "bucket", Key: key, Body: "special" }) + ); + + const get = await client.send( + new GetObjectCommand({ Bucket: "bucket", Key: key }) + ); + assert(get.Body !== undefined); + expect(await get.Body.transformToString()).toBe("special"); + + // The stored key is the decoded form + const r2 = await bucket(); + const object = await r2.get(key); + expect(await object?.text()).toBe("special"); + + // Keys containing `%` must be decoded exactly once + const percentKey = "literal 100%/a%2Bb.txt"; + await client.send( + new PutObjectCommand({ Bucket: "bucket", Key: percentKey, Body: "percent" }) + ); + const percent = await client.send( + new GetObjectCommand({ Bucket: "bucket", Key: percentKey }) + ); + assert(percent.Body !== undefined); + expect(await percent.Body.transformToString()).toBe("percent"); + expect(await (await r2.get(percentKey))?.text()).toBe("percent"); +}); + +test("POST on an object key behaves like PutObject (R2 quirk)", async ({ + expect, +}) => { + const res = await s3Fetch("bucket/posted.txt", { + method: "POST", + body: "posted", + }); + expect(res.status).toBe(200); + const get = await s3Fetch("bucket/posted.txt"); + expect(await get.text()).toBe("posted"); + + const r2 = await bucket(); + await r2.put("post-copy-source.txt", "source content"); + const postCopy = await s3Fetch("bucket/posted.txt", { + method: "POST", + body: "body wins", + headers: { "x-amz-copy-source": "bucket/post-copy-source.txt" }, + }); + expect(postCopy.status).toBe(200); + expect(await postCopy.text()).toBe(""); + const getCopy = await s3Fetch("bucket/posted.txt"); + expect(await getCopy.text()).toBe("body wins"); +}); + +test("HeadObject returns metadata with an empty body", async ({ expect }) => { + const r2 = await bucket(); + await r2.put("head.txt", "abcde"); + const head = await s3().send( + new HeadObjectCommand({ Bucket: "bucket", Key: "head.txt" }) + ); + expect(head.ContentLength).toBe(5); +}); + +test("GetObject returns NoSuchKey XML; HeadObject errors have no body", async ({ + expect, +}) => { + const get = await s3Fetch("bucket/missing.txt"); + await expectError(get, 404, "NoSuchKey", expect); + const head = await s3Fetch("bucket/missing.txt", { method: "HEAD" }); + expect(head.status).toBe(404); + expect(await head.text()).toBe(""); + // The SDK models bodyless HEAD errors as NotFound + await expectSdkError( + s3().send(new HeadObjectCommand({ Bucket: "bucket", Key: "missing.txt" })), + 404, + "NotFound", + expect + ); + // Auth errors on HEAD are bodyless too + const headAuth = await s3Fetch("bucket/missing.txt", { + method: "HEAD", + credentials: { + accessKeyId: CREDENTIALS.accessKeyId, + secretAccessKey: "wrong-secret", + }, + }); + expect(headAuth.status).toBe(403); + expect(await headAuth.text()).toBe(""); +}); + +test("DeleteObject returns 204, even for missing keys", async ({ expect }) => { + const r2 = await bucket(); + await r2.put("del.txt", "x"); + const client = s3(); + await client.send( + new DeleteObjectCommand({ Bucket: "bucket", Key: "del.txt" }) + ); + expect(await r2.head("del.txt")).toBe(null); + // Deleting a missing key still succeeds + await client.send( + new DeleteObjectCommand({ Bucket: "bucket", Key: "del.txt" }) + ); +}); + +test("conditional GETs return 304 and 412", async ({ expect }) => { + const r2 = await bucket(); + const object = await r2.put("cond.txt", "x"); + assert(object !== null); + + // The SDK surfaces 304s as (unparseable, bodyless) errors + const notModified = await s3() + .send( + new GetObjectCommand({ + Bucket: "bucket", + Key: "cond.txt", + IfNoneMatch: object.httpEtag, + }) + ) + .then( + () => undefined, + (e: unknown) => e + ); + assert(notModified instanceof S3ServiceException); + expect(notModified.$metadata.httpStatusCode).toBe(304); + + await expectSdkError( + s3().send( + new GetObjectCommand({ + Bucket: "bucket", + Key: "cond.txt", + IfMatch: '"0123456789abcdef0123456789abcdef"', + }) + ), + 412, + "PreconditionFailed", + expect + ); +}); + +test("conditional PUTs return 412 on failure", async ({ expect }) => { + const r2 = await bucket(); + await r2.put("cond-put.txt", "x"); + await expectSdkError( + s3().send( + new PutObjectCommand({ + Bucket: "bucket", + Key: "cond-put.txt", + Body: "y", + IfNoneMatch: "*", + }) + ), + 412, + "PreconditionFailed", + expect + ); +}); + +test("range requests work, including suffix ranges", async ({ expect }) => { + const r2 = await bucket(); + await r2.put("range.txt", "0123456789"); + const client = s3(); + + const partial = await client.send( + new GetObjectCommand({ + Bucket: "bucket", + Key: "range.txt", + Range: "bytes=2-4", + }) + ); + expect(partial.$metadata.httpStatusCode).toBe(206); + assert(partial.Body !== undefined); + expect(await partial.Body.transformToString()).toBe("234"); + expect(partial.ContentRange).toBe("bytes 2-4/10"); + + const suffix = await client.send( + new GetObjectCommand({ + Bucket: "bucket", + Key: "range.txt", + Range: "bytes=-3", + }) + ); + assert(suffix.Body !== undefined); + expect(await suffix.Body.transformToString()).toBe("789"); + expect(suffix.ContentRange).toBe("bytes 7-9/10"); +}); + +test("rejects unsatisfiable and malformed ranges", async ({ expect }) => { + const r2 = await bucket(); + await r2.put("range2.txt", "0123456789"); + + await expectSdkError( + s3().send( + new GetObjectCommand({ + Bucket: "bucket", + Key: "range2.txt", + Range: "bytes=99999-", + }) + ), + 416, + "InvalidRange", + expect + ); + + const malformed = await s3Fetch("bucket/range2.txt", { + headers: { Range: "bytes=zzz" }, + }); + expect(malformed.status).toBe(400); + expect(await malformed.text()).toContain( + "'bytes=-suffix'.'" + ); + + const inverted = await s3Fetch("bucket/range2.txt", { + headers: { Range: "bytes=5-2" }, + }); + expect(inverted.status).toBe(400); + expect(await inverted.text()).toContain( + "range must be positive." + ); + + // A zero suffix length is unsatisfiable for any object + const zeroSuffix = await s3Fetch("bucket/range2.txt", { + headers: { Range: "bytes=-0" }, + }); + await expectError(zeroSuffix, 416, "InvalidRange", expect); +}); + +test("HEAD honors Range with a bodyless 206", async ({ expect }) => { + const r2 = await bucket(); + await r2.put("head-range.txt", "0123456789"); + + const res = await s3Fetch("bucket/head-range.txt", { + method: "HEAD", + headers: { Range: "bytes=0-4" }, + }); + expect(res.status).toBe(206); + expect(res.headers.get("Content-Range")).toBe("bytes 0-4/10"); + expect(res.headers.get("Content-Length")).toBe("5"); + expect(await res.text()).toBe(""); +}); + +test("verifies Content-MD5", async ({ expect }) => { + const good = await s3Fetch("bucket/md5.txt", { + method: "PUT", + body: "data", + headers: { + "Content-MD5": crypto.createHash("md5").update("data").digest("base64"), + }, + }); + expect(good.status).toBe(200); + + const bad = await s3Fetch("bucket/md5.txt", { + method: "PUT", + body: "data", + headers: { + "Content-MD5": crypto.createHash("md5").update("other").digest("base64"), + }, + }); + await expectError(bad, 400, "BadDigest", expect); + + const invalid = await s3Fetch("bucket/md5.txt", { + method: "PUT", + body: "data", + headers: { "Content-MD5": "not-base64!!!" }, + }); + await expectError(invalid, 400, "InvalidDigest", expect); +}); + +test("validates x-amz-storage-class", async ({ expect }) => { + const ok = await s3Fetch("bucket/sc.txt", { + method: "PUT", + body: "x", + headers: { "x-amz-storage-class": "STANDARD" }, + }); + expect(ok.status).toBe(200); + + const ia = await s3Fetch("bucket/sc.txt", { + method: "PUT", + body: "x", + headers: { "x-amz-storage-class": "STANDARD_IA" }, + }); + await expectError(ia, 501, "NotImplemented", expect); + + const bad = await s3Fetch("bucket/sc.txt", { + method: "PUT", + body: "x", + headers: { "x-amz-storage-class": "GLACIER" }, + }); + await expectError(bad, 400, "InvalidStorageClass", expect); +}); + +test("screens unsupported headers per operation", async ({ expect }) => { + // x-amz-tagging is rejected on PutObject... + const put = await s3Fetch("bucket/screen.txt", { + method: "PUT", + body: "x", + headers: { "x-amz-tagging": "a=b" }, + }); + expect(put.status).toBe(501); + expect(await put.text()).toContain( + "Header 'x-amz-tagging' with value 'a=b' not implemented" + ); + + // ...but ignored on GetObject + const r2 = await bucket(); + await r2.put("screen.txt", "x"); + const get = await s3Fetch("bucket/screen.txt", { + headers: { "x-amz-tagging": "a=b" }, + }); + expect(get.status).toBe(200); + + // x-amz-mfa is rejected on DeleteObject only + const del = await s3Fetch("bucket/screen.txt", { + method: "DELETE", + headers: { "x-amz-mfa": "device 123456" }, + }); + expect(del.status).toBe(501); + const putMfa = await s3Fetch("bucket/screen.txt", { + method: "PUT", + body: "x", + headers: { "x-amz-mfa": "device 123456" }, + }); + expect(putMfa.status).toBe(200); +}); + +test("validates x-amz-acl and x-amz-server-side-encryption values", async ({ + expect, +}) => { + const cannedAcl = await s3Fetch("bucket/acl.txt", { + method: "PUT", + body: "x", + headers: { "x-amz-acl": "public-read" }, + }); + expect(cannedAcl.status).toBe(200); + + const badAcl = await s3Fetch("bucket/acl.txt", { + method: "PUT", + body: "x", + headers: { "x-amz-acl": "lol-no" }, + }); + expect(badAcl.status).toBe(501); + + const aes = await s3Fetch("bucket/sse.txt", { + method: "PUT", + body: "x", + headers: { "x-amz-server-side-encryption": "AES256" }, + }); + expect(aes.status).toBe(200); + + const kms = await s3Fetch("bucket/sse.txt", { + method: "PUT", + body: "x", + headers: { "x-amz-server-side-encryption": "aws:kms" }, + }); + expect(kms.status).toBe(501); +}); + +test("ignores unrecognized x-amz-* headers", async ({ expect }) => { + const res = await s3Fetch("bucket/unknown-header.txt", { + method: "PUT", + body: "x", + headers: { "x-amz-foobar": "whatever" }, + }); + expect(res.status).toBe(200); +}); + +test("SSE-C reads get InvalidRequest, writes get NotImplemented", async ({ + expect, +}) => { + const r2 = await bucket(); + await r2.put("ssec.txt", "x"); + const key = Buffer.alloc(32, 7).toString("base64"); + const keyMd5 = crypto + .createHash("md5") + .update(Buffer.alloc(32, 7)) + .digest("base64"); + const fullSet = { + "x-amz-server-side-encryption-customer-algorithm": "AES256", + "x-amz-server-side-encryption-customer-key": key, + "x-amz-server-side-encryption-customer-key-MD5": keyMd5, + }; + + // Incomplete header triples are rejected before anything else + const noKey = await s3Fetch("bucket/ssec.txt", { + headers: { "x-amz-server-side-encryption-customer-algorithm": "AES256" }, + }); + expect(noKey.status).toBe(400); + expect(await noKey.text()).toContain( + "must provide an appropriate secret key." + ); + + const noMd5 = await s3Fetch("bucket/ssec.txt", { + headers: { + "x-amz-server-side-encryption-customer-algorithm": "AES256", + "x-amz-server-side-encryption-customer-key": key, + }, + }); + expect(noMd5.status).toBe(400); + expect(await noMd5.text()).toContain( + "must provide the client calculated MD5 of the secret key." + ); + + const noAlgorithm = await s3Fetch("bucket/ssec.txt", { + headers: { + "x-amz-server-side-encryption-customer-key": key, + "x-amz-server-side-encryption-customer-key-MD5": keyMd5, + }, + }); + expect(noAlgorithm.status).toBe(400); + expect(await noAlgorithm.text()).toContain( + "must provide a valid encryption algorithm." + ); + + // A complete triple with a bad algorithm value is rejected next, on + // reads and writes alike (probed against R2) + const badAlgorithm = await s3Fetch("bucket/ssec.txt", { + headers: { + ...fullSet, + "x-amz-server-side-encryption-customer-algorithm": "AES128", + }, + }); + expect(badAlgorithm.status).toBe(400); + expect(await badAlgorithm.text()).toContain( + "InvalidEncryptionAlgorithmErrorThe encryption request that you specified is not valid. The valid value is AES256." + ); + + const get = await s3Fetch("bucket/ssec.txt", { headers: fullSet }); + expect(get.status).toBe(400); + expect(await get.text()).toContain( + "The encryption parameters are not applicable to this object." + ); + + const put = await s3Fetch("bucket/ssec.txt", { + method: "PUT", + body: "x", + headers: fullSet, + }); + expect(put.status).toBe(501); + expect(await put.text()).toContain( + "x-amz-server-side-encryption-customer-algorithm" + ); +}); + +async function seedListKeys() { + const r2 = await bucket(); + await r2.put("ls/a.txt", "aaa"); + await r2.put("ls/b.txt", "bbbb"); + await r2.put("ls/sub/c.txt", "cc"); +} + +test("ListObjectsV2 lists with prefix and KeyCount", async ({ expect }) => { + await seedListKeys(); + const res = await s3().send( + new ListObjectsV2Command({ Bucket: "bucket", Prefix: "ls/" }) + ); + expect(res.Name).toBe("bucket"); + expect(res.KeyCount).toBe(3); + expect(res.IsTruncated).toBe(false); + expect(res.Contents?.map((object) => object.Key)).toEqual([ + "ls/a.txt", + "ls/b.txt", + "ls/sub/c.txt", + ]); + expect(res.Contents?.[0]?.Size).toBe(3); + expect(res.Contents?.[0]?.StorageClass).toBe("STANDARD"); +}); + +test("ListObjectsV2 groups keys with a delimiter", async ({ expect }) => { + await seedListKeys(); + const res = await s3().send( + new ListObjectsV2Command({ + Bucket: "bucket", + Prefix: "ls/", + Delimiter: "/", + }) + ); + expect(res.Contents?.map((object) => object.Key)).toEqual([ + "ls/a.txt", + "ls/b.txt", + ]); + expect(res.CommonPrefixes).toEqual([{ Prefix: "ls/sub/" }]); +}); + +test("ListObjectsV2 paginates with continuation tokens", async ({ expect }) => { + await seedListKeys(); + const client = s3(); + const first = await client.send( + new ListObjectsV2Command({ Bucket: "bucket", Prefix: "ls/", MaxKeys: 2 }) + ); + expect(first.IsTruncated).toBe(true); + assert(first.NextContinuationToken !== undefined); + + const second = await client.send( + new ListObjectsV2Command({ + Bucket: "bucket", + Prefix: "ls/", + MaxKeys: 2, + ContinuationToken: first.NextContinuationToken, + }) + ); + expect(second.Contents?.map((object) => object.Key)).toEqual([ + "ls/sub/c.txt", + ]); + expect(second.IsTruncated).toBe(false); +}); + +test("ListObjectsV2 honors start-after", async ({ expect }) => { + await seedListKeys(); + const res = await s3().send( + new ListObjectsV2Command({ + Bucket: "bucket", + Prefix: "ls/", + StartAfter: "ls/b.txt", + }) + ); + expect(res.Contents?.map((object) => object.Key)).toEqual(["ls/sub/c.txt"]); +}); + +test("ListObjects (V1) supports Marker", async ({ expect }) => { + await seedListKeys(); + const res = await s3().send( + new ListObjectsCommand({ + Bucket: "bucket", + Prefix: "ls/", + Marker: "ls/a.txt", + }) + ); + expect(res.Marker).toBe("ls/a.txt"); + expect(res.Contents?.map((object) => object.Key)).toEqual([ + "ls/b.txt", + "ls/sub/c.txt", + ]); +}); + +test("ListObjects (V1) NextMarker can be a CommonPrefix", async ({ + expect, +}) => { + const r2 = await bucket(); + await r2.put("nm/a.txt", "a"); + await r2.put("nm/sub/c.txt", "c"); + await r2.put("nm/z.txt", "z"); + + const res = await s3().send( + new ListObjectsCommand({ + Bucket: "bucket", + Prefix: "nm/", + Delimiter: "/", + MaxKeys: 2, + }) + ); + expect(res.IsTruncated).toBe(true); + expect(res.Contents?.map((object) => object.Key)).toEqual(["nm/a.txt"]); + expect(res.CommonPrefixes?.map((p) => p.Prefix)).toEqual(["nm/sub/"]); + expect(res.NextMarker).toBe("nm/sub/"); +}); + +test("encoding-type=url encodes keys", async ({ expect }) => { + await seedListKeys(); + const res = await s3Fetch( + "bucket?list-type=2&prefix=ls/sub&encoding-type=url" + ); + const text = await res.text(); + expect(text).toContain("ls%2Fsub%2Fc.txt"); + expect(text).toContain("ls%2Fsub"); + expect(text).toContain("url"); +}); + +test("DeleteObjects deletes keys, reporting missing keys as Deleted", async ({ + expect, +}) => { + const r2 = await bucket(); + await r2.put("batch/a.txt", "x"); + await r2.put("batch/b.txt", "x"); + const res = await s3().send( + new DeleteObjectsCommand({ + Bucket: "bucket", + Delete: { + Objects: [ + { Key: "batch/a.txt" }, + { Key: "batch/b.txt" }, + { Key: "batch/missing.txt" }, + ], + }, + }) + ); + expect(res.Deleted?.map((deleted) => deleted.Key)).toEqual([ + "batch/a.txt", + "batch/b.txt", + "batch/missing.txt", + ]); + expect(await r2.head("batch/a.txt")).toBe(null); + expect(await r2.head("batch/b.txt")).toBe(null); +}); + +test("DeleteObjects validates Quiet but ignores it", async ({ expect }) => { + const r2 = await bucket(); + await r2.put("batch/q.txt", "x"); + // Unlike AWS S3, R2 returns the Deleted list even in quiet mode + const res = await s3().send( + new DeleteObjectsCommand({ + Bucket: "bucket", + Delete: { Quiet: true, Objects: [{ Key: "batch/q.txt" }] }, + }) + ); + expect(res.Deleted?.map((deleted) => deleted.Key)).toEqual(["batch/q.txt"]); + expect(await r2.head("batch/q.txt")).toBe(null); + + // Only literal true/false are accepted as Quiet values + const invalid = await s3Fetch("bucket?delete", { + method: "POST", + body: "TRUEx", + }); + await expectError(invalid, 400, "MalformedXML", expect); +}); + +test("DeleteObjects rejects malformed XML", async ({ expect }) => { + const res = await s3Fetch("bucket?delete", { + method: "POST", + body: "", + }); + await expectError(res, 400, "MalformedXML", expect); + + // An empty object list is malformed too + const empty = await s3Fetch("bucket?delete", { + method: "POST", + body: "", + }); + await expectError(empty, 400, "MalformedXML", expect); + + const missingKey = await s3Fetch("bucket?delete", { + method: "POST", + body: "x", + }); + await expectError(missingKey, 400, "MalformedXML", expect); + + // At most 1000 keys per request + const tooMany = await s3Fetch("bucket?delete", { + method: "POST", + body: `${Array.from( + { length: 1001 }, + (_, i) => `k${i}` + ).join("")}`, + }); + await expectError(tooMany, 400, "MalformedXML", expect); +}); + +test("CopyObject copies an object, preserving metadata", async ({ expect }) => { + const r2 = await bucket(); + await r2.put("copy/src.txt", "copy me", { + httpMetadata: { contentType: "text/csv" }, + customMetadata: { original: "yes" }, + }); + const client = s3(); + const res = await client.send( + new CopyObjectCommand({ + Bucket: "bucket", + Key: "copy/dst.txt", + CopySource: "/bucket/copy/src.txt", + }) + ); + expect(res.CopyObjectResult?.ETag).toBeDefined(); + expect(res.CopyObjectResult?.LastModified).toBeDefined(); + + const get = await client.send( + new GetObjectCommand({ Bucket: "bucket", Key: "copy/dst.txt" }) + ); + assert(get.Body !== undefined); + expect(await get.Body.transformToString()).toBe("copy me"); + expect(get.ContentType).toBe("text/csv"); + expect(get.Metadata).toEqual({ original: "yes" }); +}); + +test("CopyObject REPLACE directive uses request metadata", async ({ + expect, +}) => { + const r2 = await bucket(); + await r2.put("copy/src2.txt", "data", { + httpMetadata: { contentType: "text/csv" }, + }); + const client = s3(); + await client.send( + new CopyObjectCommand({ + Bucket: "bucket", + Key: "copy/dst2.txt", + CopySource: "/bucket/copy/src2.txt", + MetadataDirective: "REPLACE", + ContentType: "application/json", + Metadata: { new: "1" }, + }) + ); + const get = await client.send( + new GetObjectCommand({ Bucket: "bucket", Key: "copy/dst2.txt" }) + ); + expect(get.ContentType).toBe("application/json"); + expect(get.Metadata).toEqual({ new: "1" }); +}); + +test("CopyObject works across buckets", async ({ expect }) => { + const other = await ctx.mf.getR2Bucket("OTHER"); + await other.put("cross.txt", "from the other bucket"); + const client = s3(); + await client.send( + new CopyObjectCommand({ + Bucket: "bucket", + Key: "cross-copied.txt", + CopySource: "/other-bucket/cross.txt", + }) + ); + const get = await client.send( + new GetObjectCommand({ Bucket: "bucket", Key: "cross-copied.txt" }) + ); + assert(get.Body !== undefined); + expect(await get.Body.transformToString()).toBe("from the other bucket"); +}); + +test("CopyObject error cases", async ({ expect }) => { + const r2 = await bucket(); + await r2.put("copy/src3.txt", "x"); + const client = s3(); + + await expectSdkError( + client.send( + new CopyObjectCommand({ + Bucket: "bucket", + Key: "copy/dst3.txt", + CopySource: "/bucket/copy/nope.txt", + }) + ), + 404, + "NoSuchKey", + expect + ); + + await expectSdkError( + client.send( + new CopyObjectCommand({ + Bucket: "bucket", + Key: "copy/dst3.txt", + CopySource: "/no-such-bucket/x.txt", + }) + ), + 404, + "NoSuchBucket", + expect + ); + + // third-bucket is configured with different credentials; the verified + // pair must also grant access to the copy source + await expectSdkError( + client.send( + new CopyObjectCommand({ + Bucket: "bucket", + Key: "copy/dst3.txt", + CopySource: "/third-bucket/x.txt", + }) + ), + 401, + "Unauthorized", + expect + ); + + await expectSdkError( + client.send( + new CopyObjectCommand({ + Bucket: "bucket", + Key: "copy/dst3.txt", + CopySource: "/bucket/copy/src3.txt", + CopySourceIfMatch: '"0123456789abcdef0123456789abcdef"', + }) + ), + 412, + "PreconditionFailed", + expect + ); + + const badDirective = await s3Fetch("bucket/copy/dst3.txt", { + method: "PUT", + headers: { + "x-amz-copy-source": "/bucket/copy/src3.txt", + "x-amz-metadata-directive": "WAT", + }, + }); + expect(badDirective.status).toBe(400); + expect(await badDirective.text()).toContain("metadata directive WAT."); + + const badSource = await s3Fetch("bucket/copy/dst3.txt", { + method: "PUT", + headers: { "x-amz-copy-source": "no-slash" }, + }); + expect(badSource.status).toBe(400); + expect(await badSource.text()).toContain("copy source bucket name"); +}); + +test("CopyObject decodes the copy source and allows self-copies", async ({ + expect, +}) => { + const r2 = await bucket(); + await r2.put("copy/sp ace.txt", "spaced"); + + // Copy sources arrive percent-encoded + const encoded = await s3Fetch("bucket/copy/dst-encoded.txt", { + method: "PUT", + headers: { "x-amz-copy-source": "/bucket/copy/sp%20ace.txt" }, + }); + expect(encoded.status).toBe(200); + const copied = await r2.get("copy/dst-encoded.txt"); + expect(await copied?.text()).toBe("spaced"); + + // Copying an object onto itself is allowed + const self = await s3Fetch("bucket/copy/sp%20ace.txt", { + method: "PUT", + headers: { "x-amz-copy-source": "/bucket/copy/sp%20ace.txt" }, + }); + expect(self.status).toBe(200); + expect(await self.text()).toContain(" { + const client = s3(); + const create = await client.send( + new CreateMultipartUploadCommand({ + Bucket: "bucket", + Key: "mp/obj.bin", + ContentType: "application/x-thing", + Metadata: { mp: "1" }, + }) + ); + expect(create.Bucket).toBe("bucket"); + expect(create.Key).toBe("mp/obj.bin"); + assert(create.UploadId !== undefined); + + const part = await client.send( + new UploadPartCommand({ + Bucket: "bucket", + Key: "mp/obj.bin", + UploadId: create.UploadId, + PartNumber: 1, + Body: "part-one-data", + }) + ); + assert(part.ETag !== undefined); + + const complete = await client.send( + new CompleteMultipartUploadCommand({ + Bucket: "bucket", + Key: "mp/obj.bin", + UploadId: create.UploadId, + MultipartUpload: { Parts: [{ PartNumber: 1, ETag: part.ETag }] }, + }) + ); + expect(complete.Key).toBe("mp/obj.bin"); + expect(complete.Location).toContain("mp%2Fobj.bin"); + // Multipart ETags carry a part-count suffix + expect(complete.ETag).toMatch(/^"[0-9a-f]{32}-1"$/); + + const get = await client.send( + new GetObjectCommand({ Bucket: "bucket", Key: "mp/obj.bin" }) + ); + assert(get.Body !== undefined); + expect(await get.Body.transformToString()).toBe("part-one-data"); + expect(get.ContentType).toBe("application/x-thing"); + expect(get.Metadata).toEqual({ mp: "1" }); + + // The upload is gone after completion + await expectSdkError( + client.send( + new CompleteMultipartUploadCommand({ + Bucket: "bucket", + Key: "mp/obj.bin", + UploadId: create.UploadId, + MultipartUpload: { Parts: [{ PartNumber: 1, ETag: part.ETag }] }, + }) + ), + 404, + "NoSuchUpload", + expect + ); +}); + +test("multipart error cases", async ({ expect }) => { + const client = s3(); + await expectSdkError( + client.send( + new UploadPartCommand({ + Bucket: "bucket", + Key: "mp/err.bin", + UploadId: "bogus", + PartNumber: 1, + Body: "x", + }) + ), + 404, + "NoSuchUpload", + expect + ); + + await expectSdkError( + client.send( + new AbortMultipartUploadCommand({ + Bucket: "bucket", + Key: "mp/err.bin", + UploadId: "bogus", + }) + ), + 404, + "NoSuchUpload", + expect + ); + + const create = await client.send( + new CreateMultipartUploadCommand({ Bucket: "bucket", Key: "mp/err.bin" }) + ); + assert(create.UploadId !== undefined); + + const badNumber = await expectSdkError( + client.send( + new UploadPartCommand({ + Bucket: "bucket", + Key: "mp/err.bin", + UploadId: create.UploadId, + PartNumber: 0, + Body: "x", + }) + ), + 400, + "InvalidArgument", + expect + ); + expect(badNumber.message).toContain( + "Part number must be an integer between 1 and 10000, inclusive." + ); + + const duplicate = await expectSdkError( + client.send( + new CompleteMultipartUploadCommand({ + Bucket: "bucket", + Key: "mp/err.bin", + UploadId: create.UploadId, + MultipartUpload: { + Parts: [ + { PartNumber: 1, ETag: '"0123456789abcdef0123456789abcdef"' }, + { PartNumber: 1, ETag: '"0123456789abcdef0123456789abcdef"' }, + ], + }, + }) + ), + 400, + "InvalidPart", + expect + ); + expect(duplicate.message).toBe( + "There was a problem with the multipart upload." + ); + + await client.send( + new UploadPartCommand({ + Bucket: "bucket", + Key: "mp/err.bin", + UploadId: create.UploadId, + PartNumber: 1, + Body: "data", + }) + ); + await expectSdkError( + client.send( + new CompleteMultipartUploadCommand({ + Bucket: "bucket", + Key: "mp/err.bin", + UploadId: create.UploadId, + MultipartUpload: { + Parts: [ + { + PartNumber: 1, + ETag: '"0123456789abcdef0123456789abcdef"', + }, + ], + }, + }) + ), + 400, + "InvalidPart", + expect + ); + + const malformed = await s3Fetch( + `bucket/mp/err.bin?uploadId=${encodeURIComponent(create.UploadId)}`, + { method: "POST", body: "" } + ); + await expectError(malformed, 400, "MalformedXML", expect); + + // R2's part routes only match integer-shaped partNumber values; padded + // and zero-prefixed integers are accepted, anything else is RouteNotFound + const uid = encodeURIComponent(create.UploadId); + const padded = await s3Fetch( + `bucket/mp/err.bin?uploadId=${uid}&partNumber=01`, + { + method: "PUT", + body: "x", + } + ); + expect(padded.status).toBe(200); + const fractional = await s3Fetch( + `bucket/mp/err.bin?uploadId=${uid}&partNumber=1.0`, + { method: "PUT", body: "x" } + ); + await expectError(fractional, 404, "RouteNotFound", expect); + const empty = await s3Fetch(`bucket/mp/err.bin?uploadId=${uid}&partNumber=`, { + method: "PUT", + body: "x", + }); + await expectError(empty, 400, "InvalidArgument", expect); + + // Inside the Complete XML (unlike the ?partNumber= query param), real R2 + // reports non-integer part numbers as MalformedXML and out-of-range + // integers as InvalidPart (a part that does not exist) + const uploadId = create.UploadId; + const completeWith = (partNumber: string) => + s3Fetch(`bucket/mp/err.bin?uploadId=${encodeURIComponent(uploadId)}`, { + method: "POST", + body: `${partNumber}"0123456789abcdef0123456789abcdef"`, + }); + await expectError(await completeWith("abc"), 400, "MalformedXML", expect); + await expectError(await completeWith("1.5"), 400, "MalformedXML", expect); + await expectError(await completeWith("0"), 400, "InvalidPart", expect); + await expectError(await completeWith("10001"), 400, "InvalidPart", expect); + + // An empty part list is malformed too + const emptyComplete = await s3Fetch( + `bucket/mp/err.bin?uploadId=${encodeURIComponent(uploadId)}`, + { + method: "POST", + body: "", + } + ); + await expectError(emptyComplete, 400, "MalformedXML", expect); + + await client.send( + new AbortMultipartUploadCommand({ + Bucket: "bucket", + Key: "mp/err.bin", + UploadId: create.UploadId, + }) + ); + + await expectSdkError( + client.send( + new UploadPartCommand({ + Bucket: "bucket", + Key: "mp/err.bin", + UploadId: create.UploadId, + PartNumber: 1, + Body: "x", + }) + ), + 404, + "NoSuchUpload", + expect + ); +}); + +test("PUT with uploadId but no partNumber is a plain PutObject", async ({ + expect, +}) => { + const client = s3(); + const create = await client.send( + new CreateMultipartUploadCommand({ Bucket: "bucket", Key: "mp/plain.bin" }) + ); + assert(create.UploadId !== undefined); + + const res = await s3Fetch( + `bucket/mp/plain.bin?uploadId=${encodeURIComponent(create.UploadId)}`, + { method: "PUT", body: "not-a-part" } + ); + expect(res.status).toBe(200); + + // The object was written directly; the multipart upload saw no parts + const r2 = await bucket(); + const object = await r2.get("mp/plain.bin"); + expect(await object?.text()).toBe("not-a-part"); + await client.send( + new AbortMultipartUploadCommand({ + Bucket: "bucket", + Key: "mp/plain.bin", + UploadId: create.UploadId, + }) + ); +}); + +test("complete rejects non-final parts below the minimum size", async ({ + expect, +}) => { + const client = s3(); + const create = await client.send( + new CreateMultipartUploadCommand({ Bucket: "bucket", Key: "mp/small.bin" }) + ); + assert(create.UploadId !== undefined); + + const parts = []; + for (const partNumber of [1, 2]) { + const part = await client.send( + new UploadPartCommand({ + Bucket: "bucket", + Key: "mp/small.bin", + UploadId: create.UploadId, + PartNumber: partNumber, + Body: `tiny${partNumber}`, + }) + ); + parts.push({ PartNumber: partNumber, ETag: part.ETag }); + } + await expectSdkError( + client.send( + new CompleteMultipartUploadCommand({ + Bucket: "bucket", + Key: "mp/small.bin", + UploadId: create.UploadId, + MultipartUpload: { Parts: parts }, + }) + ), + 400, + "EntityTooSmall", + expect + ); + await client.send( + new AbortMultipartUploadCommand({ + Bucket: "bucket", + Key: "mp/small.bin", + UploadId: create.UploadId, + }) + ); +}); + +test("UploadPartCopy copies a part, optionally with a range", async ({ + expect, +}) => { + const r2 = await bucket(); + await r2.put("mp/copy-src.txt", "0123456789"); + const client = s3(); + + const create = await client.send( + new CreateMultipartUploadCommand({ Bucket: "bucket", Key: "mp/copied.bin" }) + ); + assert(create.UploadId !== undefined); + + const copied = await client.send( + new UploadPartCopyCommand({ + Bucket: "bucket", + Key: "mp/copied.bin", + UploadId: create.UploadId, + PartNumber: 1, + CopySource: "/bucket/mp/copy-src.txt", + CopySourceRange: "bytes=0-4", + }) + ); + assert(copied.CopyPartResult?.ETag !== undefined); + + // Only `bytes=start-end` is accepted for x-amz-copy-source-range + const badRange = await s3Fetch( + `bucket/mp/copied.bin?uploadId=${encodeURIComponent(create.UploadId)}&partNumber=2`, + { + method: "PUT", + headers: { + "x-amz-copy-source": "/bucket/mp/copy-src.txt", + "x-amz-copy-source-range": "bytes=0-", + }, + } + ); + expect(badRange.status).toBe(400); + expect(await badRange.text()).toContain( + "Invalid x-amz-copy-source-range: bytes=0-" + ); + + // Inverted ranges are rejected (out-of-bounds ends are clamped instead) + const invertedRange = await s3Fetch( + `bucket/mp/copied.bin?uploadId=${encodeURIComponent(create.UploadId)}&partNumber=2`, + { + method: "PUT", + headers: { + "x-amz-copy-source": "/bucket/mp/copy-src.txt", + "x-amz-copy-source-range": "bytes=5-2", + }, + } + ); + expect(invertedRange.status).toBe(400); + expect(await invertedRange.text()).toContain( + "x-amz-copy-source-range must be positive." + ); + + await client.send( + new CompleteMultipartUploadCommand({ + Bucket: "bucket", + Key: "mp/copied.bin", + UploadId: create.UploadId, + MultipartUpload: { + Parts: [{ PartNumber: 1, ETag: copied.CopyPartResult.ETag }], + }, + }) + ); + const get = await client.send( + new GetObjectCommand({ Bucket: "bucket", Key: "mp/copied.bin" }) + ); + assert(get.Body !== undefined); + expect(await get.Body.transformToString()).toBe("01234"); +}); + +test("unimplemented multipart surfaces respond with NotImplemented", async ({ + expect, +}) => { + const client = s3(); + await expectSdkError( + client.send(new ListMultipartUploadsCommand({ Bucket: "bucket" })), + 501, + "NotImplemented", + expect + ); + + await expectSdkError( + client.send( + new ListPartsCommand({ + Bucket: "bucket", + Key: "mp/x.bin", + UploadId: "any", + }) + ), + 501, + "NotImplemented", + expect + ); + + const r2 = await bucket(); + await r2.put("mp/pn.txt", "x"); + await expectSdkError( + client.send( + new GetObjectCommand({ + Bucket: "bucket", + Key: "mp/pn.txt", + PartNumber: 1, + }) + ), + 501, + "NotImplemented", + expect + ); +}); + +// ## Recognized-but-unimplemented surfaces (messages match real R2) + +test("object subresource operations return R2's templated errors", async ({ + expect, +}) => { + const r2 = await bucket(); + await r2.put("sub/x.txt", "x"); + + const getTagging = await s3Fetch("bucket/sub/x.txt?tagging"); + expect(getTagging.status).toBe(501); + expect(await getTagging.text()).toContain( + "GetObjectTagging not implemented" + ); + + const putAcl = await s3Fetch("bucket/sub/x.txt?acl", { + method: "PUT", + body: "x", + }); + expect(putAcl.status).toBe(501); + expect(await putAcl.text()).toContain( + "PutObjectAcl not implemented" + ); + + // DELETE ignores subresource parameters on R2 and just deletes the object + const del = await s3Fetch("bucket/sub/x.txt?tagging", { method: "DELETE" }); + expect(del.status).toBe(204); + expect(await r2.head("sub/x.txt")).toBe(null); +}); + +test("bucket subresource GETs return R2's templated errors", async ({ + expect, +}) => { + const policy = await s3Fetch("bucket?policy"); + expect(policy.status).toBe(501); + expect(await policy.text()).toContain( + "GetBucketPolicy not implemented" + ); + + const versions = await s3Fetch("bucket?versions"); + expect(versions.status).toBe(501); + expect(await versions.text()).toContain( + "ListObjectVersions not implemented" + ); + + const tiering = await s3Fetch("bucket?intelligent-tiering"); + expect(tiering.status).toBe(501); + expect(await tiering.text()).toContain( + "GetBucketIntelligentTieringConfiguration not implemented" + ); + + const policyStatus = await s3Fetch("bucket?policyStatus"); + expect(policyStatus.status).toBe(501); + expect(await policyStatus.text()).toContain( + "GetGetBucketPolicyStatus not implemented" + ); +}); + +test("lists reject unknown search parameters like R2", async ({ expect }) => { + const v1 = await s3Fetch("bucket?foobar=1"); + expect(v1.status).toBe(501); + expect(await v1.text()).toContain( + "ListObjectsV1 search parameter foobar not implemented" + ); + + const v2 = await s3Fetch("bucket?list-type=2&foobar=1"); + expect(v2.status).toBe(501); + expect(await v2.text()).toContain( + "ListObjectsV2 search parameter foobar not implemented" + ); + + const v3 = await s3Fetch("bucket?list-type=3"); + expect(v3.status).toBe(501); + expect(await v3.text()).toContain( + "ListObjectsV3 not implemented" + ); +}); + +test("HeadBucket and GetBucketLocation work", async ({ expect }) => { + const client = s3(); + const head = await client.send(new HeadBucketCommand({ Bucket: "bucket" })); + expect(head.$metadata.httpStatusCode).toBe(200); + + const location = await client.send( + new GetBucketLocationCommand({ Bucket: "bucket" }) + ); + expect(location.LocationConstraint).toBe("auto"); + + const extraParam = await s3Fetch("bucket?location&foobar=1"); + expect(extraParam.status).toBe(400); + expect(await extraParam.text()).toContain( + "Search param foobar is unsupported for bucket location" + ); +}); + +test("unroutable requests match R2's responses", async ({ expect }) => { + const patch = await s3Fetch("bucket/x.txt", { method: "PATCH" }); + expect(patch.status).toBe(404); + expect(await patch.text()).toContain( + "RouteNotFoundNo route matches this url." + ); + + // R2 responds 200 to a plain *signed* bucket-level POST + const post = await s3Fetch("bucket", { method: "POST" }); + expect(post.status).toBe(200); + expect(await post.text()).toBe(""); + + // An unsigned bucket-level POST is an attempted browser form upload + // (AWS's POST Object, with auth in the form fields); R2 recognizes the + // shape but does not implement it, with a doubled "not implemented" + // (theirs, verbatim) + const form = new FormData(); + form.set("key", "form.txt"); + const unsignedPost = await fetch(s3Url("bucket"), { + method: "POST", + body: form, + }); + expect(unsignedPost.status).toBe(501); + expect(await unsignedPost.text()).toContain( + "Presigned post requests are not yet implemented not implemented" + ); + + // CreateBucket is meaningless locally (buckets come from config) + const put = await s3Fetch("bucket", { method: "PUT" }); + expect(put.status).toBe(501); + expect(await put.text()).toContain( + "CreateBucket not implemented" + ); +}); + +test("bucket-level PUT/DELETE subresources match real R2's routing", async ({ + expect, +}) => { + // A recognized subresource wins, in any position + const putTagging = await s3Fetch("bucket?junk&tagging", { method: "PUT" }); + expect(putTagging.status).toBe(501); + expect(await putTagging.text()).toContain( + "PutBucketTagging not implemented" + ); + + const deletePolicy = await s3Fetch("bucket?policy", { method: "DELETE" }); + expect(deletePolicy.status).toBe(501); + expect(await deletePolicy.text()).toContain( + "DeleteBucketPolicy not implemented" + ); + + // Subresources for other methods and unknown params are all rejected + // together with R2's bucket-route error + const deleteVersioning = await s3Fetch("bucket?versioning&junk", { + method: "DELETE", + }); + expect(deleteVersioning.status).toBe(400); + expect(await deleteVersioning.text()).toContain( + "Unsupported search param(s) "versioning", "junk" on a DELETE bucket route" + ); + + const putJunk = await s3Fetch("bucket?junk", { method: "PUT" }); + expect(putJunk.status).toBe(400); + expect(await putJunk.text()).toContain( + "Unsupported search param(s) "junk" on a PUT bucket route" + ); +}); + +test("static bucket-configuration reads match real R2", async ({ expect }) => { + const client = s3(); + const encryption = await client.send( + new GetBucketEncryptionCommand({ Bucket: "bucket" }) + ); + const rule = encryption.ServerSideEncryptionConfiguration?.Rules?.[0]; + expect(rule?.ApplyServerSideEncryptionByDefault?.SSEAlgorithm).toBe("AES256"); + expect(rule?.BucketKeyEnabled).toBe(true); + + // R2 has no bucket versioning; the configuration is always empty + const versioning = await client.send( + new GetBucketVersioningCommand({ Bucket: "bucket" }) + ); + expect(versioning.Status).toBeUndefined(); + + const tagging = await s3Fetch("bucket?tagging"); + await expectError(tagging, 404, "NoSuchTagSet", expect); + + const objectLock = await s3Fetch("bucket?object-lock"); + await expectError( + objectLock, + 404, + "ObjectLockConfigurationNotFoundError", + expect + ); + + const replication = await s3Fetch("bucket?replication"); + await expectError( + replication, + 404, + "ReplicationConfigurationNotFoundError", + expect + ); + + // Stateful bucket configuration cannot be simulated locally + const cors = await s3Fetch("bucket?cors"); + expect(cors.status).toBe(501); + expect(await cors.text()).toContain("GetBucketCors not implemented"); +}); + +test("ListBuckets lists buckets for the presented credentials", async ({ + expect, +}) => { + const res = await s3().send(new ListBucketsCommand({})); + expect(res.Buckets?.map((bucket) => bucket.Name)).toEqual([ + "bucket", + "other-bucket", + ]); + + const third = await s3({ credentials: THIRD_CREDENTIALS }).send( + new ListBucketsCommand({}) + ); + expect(third.Buckets?.map((bucket) => bucket.Name)).toEqual(["third-bucket"]); + + await expectSdkError( + s3({ + credentials: { ...CREDENTIALS, secretAccessKey: "wrong" }, + }).send(new ListBucketsCommand({})), + 403, + "SignatureDoesNotMatch", + expect + ); + + const unknownParam = await s3Fetch("?foobar=1"); + expect(unknownParam.status).toBe(501); + expect(await unknownParam.text()).toContain( + "ListBuckets search parameter foobar not implemented" + ); + + // Only GET is routable at the account level + const post = await s3Fetch("", { method: "POST" }); + await expectError(post, 404, "RouteNotFound", expect); +}); + +test("V1 lists reject continuation-token with R2's bespoke error", async ({ + expect, +}) => { + const res = await s3Fetch("bucket?continuation-token=x"); + expect(res.status).toBe(400); + expect(await res.text()).toContain( + "continuation-token not supported in ListObjects" + ); +}); + +test("list edge cases match real R2", async ({ expect }) => { + await seedListKeys(); + + // max-keys=0 reports IsTruncated based on whether matching keys exist + const zero = await s3Fetch("bucket?prefix=ls/&max-keys=0"); + const zeroText = await zero.text(); + expect(zeroText).toContain("true"); + expect(zeroText).toContain("0"); + expect(zeroText).not.toContain(""); + + // fractional values are floored + const fractional = await s3Fetch("bucket?prefix=ls/&max-keys=1.5"); + expect(await fractional.text()).toContain("1"); + + const emptyMaxKeys = await s3Fetch("bucket?max-keys="); + await expectError(emptyMaxKeys, 400, "InvalidMaxKeys", expect); + + const nonFinite = await s3Fetch("bucket?max-keys=Infinity"); + await expectError(nonFinite, 400, "InvalidMaxKeys", expect); + + const invalid = await s3Fetch("bucket?max-keys=abc"); + expect(invalid.status).toBe(400); + expect(await invalid.text()).toContain( + "InvalidMaxKeysMaxKeys params must be positive integer <= 1000." + ); + + const encoding = await s3Fetch("bucket?encoding-type=weird"); + expect(encoding.status).toBe(501); + expect(await encoding.text()).toContain( + "Unrecognized encoding-type "weird" not implemented" + ); +}); + +test("Complete with unknown uploadId returns NoSuchUpload", async ({ + expect, +}) => { + await expectSdkError( + s3().send( + new CompleteMultipartUploadCommand({ + Bucket: "bucket", + Key: "mp/nope.bin", + UploadId: "bogus", + MultipartUpload: { + Parts: [ + { PartNumber: 1, ETag: '"0123456789abcdef0123456789abcdef"' }, + ], + }, + }) + ), + 404, + "NoSuchUpload", + expect + ); +}); + +// Unlike real R2 (which only answers preflights according to the bucket's +// CORS configuration), the local endpoint always allows cross-origin use so +// browser requests (e.g. presigned uploads from a frontend dev server) work + +test("answers CORS preflights, including for presigned browser uploads", async ({ + expect, +}) => { + const res = await fetch(s3Url("bucket/upload.bin"), { + method: "OPTIONS", + headers: { + Origin: "http://localhost:3000", + "Access-Control-Request-Method": "PUT", + "Access-Control-Request-Headers": "authorization,x-amz-date,content-type", + }, + }); + expect(res.status).toBe(204); + expect(res.headers.get("Access-Control-Allow-Origin")).toBe("*"); + expect(res.headers.get("Access-Control-Allow-Methods")).toContain("PUT"); + expect(res.headers.get("Access-Control-Allow-Headers")).toBe( + "authorization,x-amz-date,content-type" + ); +}); + +test("sets CORS headers on cross-origin responses", async ({ expect }) => { + const r2 = await bucket(); + await r2.put("cors.txt", "x"); + const res = await s3Fetch("bucket/cors.txt", { + headers: { Origin: "http://localhost:3000" }, + }); + expect(res.status).toBe(200); + expect(res.headers.get("Access-Control-Allow-Origin")).toBe("*"); + expect(res.headers.get("Access-Control-Expose-Headers")).toBe("*"); +}); + +test("rejects different s3Credentials for the same bucket", async ({ + expect, +}) => { + const mf = new Miniflare({ + workers: [ + { + name: "a", + modules: true, + script: "export default {};", + r2Buckets: { BUCKET: { id: "shared", s3Credentials: CREDENTIALS } }, + }, + { + name: "b", + modules: true, + script: "export default {};", + r2Buckets: { + BUCKET: { + id: "shared", + s3Credentials: { + accessKeyId: "B".repeat(32), + secretAccessKey: "other-secret", + }, + }, + }, + }, + ], + }); + await expect(mf.ready).rejects.toThrow( + 'Bucket "shared" is bound by multiple Workers with different S3 credentials' + ); + // dispose() would re-await the failed init and rethrow + await mf.dispose().catch(() => {}); +}); + +test("verifies signatures against the original host when `upstream` is set", async ({ + expect, +}) => { + // With `upstream` configured, the entry worker rewrites the request URL + // and Host header before dispatching to the S3 service; signatures must + // still be verified against the host the client signed + const mf = new Miniflare({ + modules: true, + script: + "export default { fetch: () => new Response(null, { status: 404 }) };", + r2Buckets: { BUCKET: { id: "bucket", s3Credentials: CREDENTIALS } }, + upstream: "https://example.com/", + }); + onTestFinished(() => mf.dispose()); + const url = await mf.ready; + const r2 = await mf.getR2Bucket("BUCKET"); + await r2.put("up.txt", "upstream"); + + const client = new S3Client({ + region: "auto", + endpoint: new URL("/cdn-cgi/local/r2/s3/", url).href, + credentials: CREDENTIALS, + forcePathStyle: true, + }); + client.middlewareStack.remove("addExpectContinueMiddleware"); + onTestFinished(() => client.destroy()); + + const res = await client.send( + new GetObjectCommand({ Bucket: "bucket", Key: "up.txt" }) + ); + assert(res.Body !== undefined); + expect(await res.Body.transformToString()).toBe("upstream"); +}); diff --git a/packages/workers-utils/src/config/environment.ts b/packages/workers-utils/src/config/environment.ts index f0ef4e8aa2..4a028c6290 100644 --- a/packages/workers-utils/src/config/environment.ts +++ b/packages/workers-utils/src/config/environment.ts @@ -18,6 +18,12 @@ export interface Environment extends EnvironmentInheritable, EnvironmentNonInheritable {} type SimpleRoute = string; +/** AWS SigV4 credentials for miniflare's local S3-compatible endpoint */ +export interface LocalS3Credentials { + accessKeyId: string; + secretAccessKey: string; +} + export type ZoneIdRoute = { pattern: string; zone_id: string; @@ -972,6 +978,13 @@ export interface EnvironmentNonInheritable { jurisdiction?: string; /** Whether the R2 bucket should be remote or not in local development */ remote?: boolean; + /** + * EXPERIMENTAL: AWS SigV4 credentials for the local S3-compatible + * endpoint. When set, the bucket is served at + * `/cdn-cgi/local/r2/s3/` during local development. + * Ignored when the bucket runs remotely. + */ + experimental_local_s3_credentials?: LocalS3Credentials; }[]; /** diff --git a/packages/workers-utils/src/config/validation.ts b/packages/workers-utils/src/config/validation.ts index 83969a0e56..5428847fa2 100644 --- a/packages/workers-utils/src/config/validation.ts +++ b/packages/workers-utils/src/config/validation.ts @@ -4091,12 +4091,35 @@ const validateR2Binding: ValidatorFn = (diagnostics, field, value) => { isValid = false; } + experimental( + diagnostics, + value as { experimental_local_s3_credentials?: unknown }, + "experimental_local_s3_credentials" + ); + if (hasProperty(value, "experimental_local_s3_credentials")) { + const credentials = value.experimental_local_s3_credentials; + if ( + typeof credentials !== "object" || + credentials === null || + !isRequiredProperty(credentials, "accessKeyId", "string") || + !isRequiredProperty(credentials, "secretAccessKey", "string") + ) { + diagnostics.errors.push( + `"${field}" bindings should, optionally, have an "experimental_local_s3_credentials" field with string "accessKeyId" and "secretAccessKey" fields, but got ${JSON.stringify( + value + )}.` + ); + isValid = false; + } + } + validateAdditionalProperties(diagnostics, field, Object.keys(value), [ "binding", "bucket_name", "preview_bucket_name", "jurisdiction", "remote", + "experimental_local_s3_credentials", ]); return isValid; diff --git a/packages/workers-utils/src/worker.ts b/packages/workers-utils/src/worker.ts index d5ab168da3..af6d7eb2af 100644 --- a/packages/workers-utils/src/worker.ts +++ b/packages/workers-utils/src/worker.ts @@ -1,4 +1,9 @@ -import type { CacheOptions, Observability, Route } from "./config/environment"; +import type { + CacheOptions, + LocalS3Credentials, + Observability, + Route, +} from "./config/environment"; import type { INHERIT_SYMBOL } from "./constants"; import type { Json, WorkerMetadata } from "./types"; import type { AssetConfig, RouterConfig } from "@cloudflare/workers-shared"; @@ -209,6 +214,8 @@ export interface CfR2Bucket { jurisdiction?: string; remote?: boolean; raw?: boolean; + /** EXPERIMENTAL: credentials for the local S3-compatible endpoint */ + experimental_local_s3_credentials?: LocalS3Credentials; } // TODO: figure out if this is duplicated in packages/wrangler/src/config/environment.ts diff --git a/packages/workers-utils/tests/config/validation/normalize-and-validate-config.test.ts b/packages/workers-utils/tests/config/validation/normalize-and-validate-config.test.ts index d28b2e426c..345f7b8baa 100644 --- a/packages/workers-utils/tests/config/validation/normalize-and-validate-config.test.ts +++ b/packages/workers-utils/tests/config/validation/normalize-and-validate-config.test.ts @@ -4107,6 +4107,58 @@ describe("normalizeAndValidateConfig()", () => { expect(diagnostics.hasWarnings()).toBe(false); expect(diagnostics.hasErrors()).toBe(false); }); + + it("should accept experimental_local_s3_credentials, with an experimental warning", ({ + expect, + }) => { + const { diagnostics } = normalizeAndValidateConfig( + { + r2_buckets: [ + { + binding: "R2_BINDING", + bucket_name: "my-bucket", + experimental_local_s3_credentials: { + accessKeyId: "key-id", + secretAccessKey: "secret", + }, + }, + ], + } as unknown as RawConfig, + undefined, + undefined, + { env: undefined } + ); + + expect(diagnostics.hasErrors()).toBe(false); + expect(diagnostics.renderWarnings()).toMatchInlineSnapshot(` + "Processing wrangler configuration: + - "experimental_local_s3_credentials" fields are experimental and may change or break at any time." + `); + }); + + it("should error if experimental_local_s3_credentials is not valid", ({ + expect, + }) => { + const { diagnostics } = normalizeAndValidateConfig( + { + r2_buckets: [ + { + binding: "R2_BINDING", + bucket_name: "my-bucket", + experimental_local_s3_credentials: { accessKeyId: "key-id" }, + }, + ], + } as unknown as RawConfig, + undefined, + undefined, + { env: undefined } + ); + + expect(diagnostics.renderErrors()).toMatchInlineSnapshot(` + "Processing wrangler configuration: + - "r2_buckets[0]" bindings should, optionally, have an "experimental_local_s3_credentials" field with string "accessKeyId" and "secretAccessKey" fields, but got {"binding":"R2_BINDING","bucket_name":"my-bucket","experimental_local_s3_credentials":{"accessKeyId":"key-id"}}." + `); + }); }); describe("[services]", () => { diff --git a/packages/wrangler/src/dev/miniflare/index.ts b/packages/wrangler/src/dev/miniflare/index.ts index 230c223621..78e9a73fe2 100644 --- a/packages/wrangler/src/dev/miniflare/index.ts +++ b/packages/wrangler/src/dev/miniflare/index.ts @@ -43,6 +43,7 @@ import type { DOContainerOptions, Json, MiniflareOptions, + R2S3Credentials, RemoteProxyConnectionString, SourceOptions, WorkerdStructuredLog, @@ -239,15 +240,24 @@ function kvNamespaceEntry( return [binding, { id, remoteProxyConnectionString }]; } function r2BucketEntry( - { binding, bucket_name, remote }: CfR2Bucket, + { + binding, + bucket_name, + remote, + experimental_local_s3_credentials, + }: CfR2Bucket, remoteProxyConnectionString?: RemoteProxyConnectionString ): [ string, - { id: string; remoteProxyConnectionString?: RemoteProxyConnectionString }, + { + id: string; + remoteProxyConnectionString?: RemoteProxyConnectionString; + s3Credentials?: R2S3Credentials; + }, ] { const id = getRemoteId(bucket_name) ?? binding; if (!remoteProxyConnectionString || !remote) { - return [binding, { id }]; + return [binding, { id, s3Credentials: experimental_local_s3_credentials }]; } return [binding, { id, remoteProxyConnectionString }]; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 74af328264..6e665855b2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2164,6 +2164,15 @@ importers: specifier: 4.1.0-beta.10 version: 4.1.0-beta.10(patch_hash=3e73fc48581841f22ea1d2cf5a3560bde280fdba04940253073fb121a70a9269) devDependencies: + '@aws-crypto/sha256-js': + specifier: ^5.2.0 + version: 5.2.0 + '@aws-sdk/client-s3': + specifier: ^3.721.0 + version: 3.721.0 + '@aws-sdk/s3-request-presigner': + specifier: 3.721.0 + version: 3.721.0 '@cloudflare/cli-shared-helpers': specifier: workspace:* version: link:../cli @@ -2197,6 +2206,9 @@ importers: '@puppeteer/browsers': specifier: ^2.10.6 version: 2.10.6 + '@smithy/signature-v4': + specifier: ^4.2.4 + version: 4.2.4 '@types/debug': specifier: ^4.1.7 version: 4.1.7 @@ -2260,6 +2272,9 @@ importers: expect-type: specifier: ^0.15.0 version: 0.15.0 + fast-xml-parser: + specifier: ^4.4.1 + version: 4.4.1 get-port: specifier: ^7.1.0 version: 7.1.0 @@ -4632,6 +4647,10 @@ packages: resolution: {integrity: sha512-HJzsQxgMOAzZrbf/YIqEx30or4tZK1oNAk6Wm6xecUQx+23JXIaePRu1YFUOLBBERQ4QBPpISFurZWBMZ5ibAw==} engines: {node: '>=16.0.0'} + '@aws-sdk/s3-request-presigner@3.721.0': + resolution: {integrity: sha512-2ibKGssj2TAQyfthNihhBqWdwowlol9bDpKybIi2T6D8l2L9g0ENGLNE50MYzSFAQ3LcjzcvLQ/GByRPiuK+pQ==} + engines: {node: '>=16.0.0'} + '@aws-sdk/signature-v4-multi-region@3.716.0': resolution: {integrity: sha512-k0goWotZKKz+kV6Ln0qeAMSeSVi4NipuIIz5R8A0uCF2zBK4CXWdZR7KeaIoLBhJwQnHj1UU7E+2MK74KIUBzA==} engines: {node: '>=16.0.0'} @@ -4654,6 +4673,10 @@ packages: resolution: {integrity: sha512-Xv+Z2lhe7w7ZZRsgBwBMZgGTVmS+dkkj2S13uNHAx9lhB5ovM8PhK5G/j28xYf6vIibeuHkRAbb7/ozdZIGR+A==} engines: {node: '>=16.0.0'} + '@aws-sdk/util-format-url@3.714.0': + resolution: {integrity: sha512-PA/ES6BeKmYzFOsZ3az/8MqSLf6uzXAS7GsYONZMF6YASn4ewd/AspuvQMp6+x9VreAPCq7PecF+XL9KXejtPg==} + engines: {node: '>=16.0.0'} + '@aws-sdk/util-locate-window@3.693.0': resolution: {integrity: sha512-ttrag6haJLWABhLqtg1Uf+4LgHWIMOVSYL+VYZmAp2v4PUGOwWmWQH0Zk8RM7YuQcLfH/EoR72/Yxz6A4FKcuw==} engines: {node: '>=16.0.0'} @@ -15788,6 +15811,17 @@ snapshots: '@smithy/util-middleware': 3.0.11 tslib: 2.8.1 + '@aws-sdk/s3-request-presigner@3.721.0': + dependencies: + '@aws-sdk/signature-v4-multi-region': 3.716.0 + '@aws-sdk/types': 3.714.0 + '@aws-sdk/util-format-url': 3.714.0 + '@smithy/middleware-endpoint': 3.2.8 + '@smithy/protocol-http': 4.1.8 + '@smithy/smithy-client': 3.7.0 + '@smithy/types': 3.7.2 + tslib: 2.8.1 + '@aws-sdk/signature-v4-multi-region@3.716.0': dependencies: '@aws-sdk/middleware-sdk-s3': 3.716.0 @@ -15822,6 +15856,13 @@ snapshots: '@smithy/util-endpoints': 2.1.7 tslib: 2.8.1 + '@aws-sdk/util-format-url@3.714.0': + dependencies: + '@aws-sdk/types': 3.714.0 + '@smithy/querystring-builder': 3.0.11 + '@smithy/types': 3.7.2 + tslib: 2.8.1 + '@aws-sdk/util-locate-window@3.693.0': dependencies: tslib: 2.8.1