The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,113 lines 44.3 KB
Raw
1 import * as AWS from "@alchemy.run/aws";
2 import { describe, expect } from "@alchemy.run/test";
3 import * as Test from "@alchemy.run/test/Alchemy";
4 import * as Core from "@alchemy.run/test/Core";
5 import * as IAM from "@distilled.cloud/aws/iam";
6 import * as Lambda from "@distilled.cloud/aws/lambda";
7 import * as S3 from "@distilled.cloud/aws/s3";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as HttpClient from "effect/http/HttpClient";
11 import * as HttpClientRequest from "effect/http/HttpClientRequest";
12 import * as Schedule from "effect/Schedule";
13 import * as Schema from "effect/Schema";
14 import * as Stream from "effect/Stream";
15 import S3VersionedMultipartFunctionLive, {
16 S3VersionedMultipartFunction,
17 } from "./fixtures/versioned-multipart-handler.ts";
18
19 const options = { providers: AWS.providers(), dev: false };
20 const { test, beforeAll, afterAll } = Test.make(options);
21 const stack = Core.scratchStack(
22 options,
23 "S3VersionedMultipart",
24 "test/AWS/S3/VersionedMultipart.test.ts",
25 );
26 const actions = {
27 CreateMultipartUpload: ["s3:PutObject"],
28 UploadPart: ["s3:PutObject"],
29 UploadPartCopy: ["s3:PutObject", "s3:GetObject", "s3:GetObjectVersion"],
30 UploadPartCopySource: ["s3:GetObject", "s3:GetObjectVersion"],
31 UploadPartCopyUnboundSource: [],
32 ListParts: ["s3:ListMultipartUploadParts"],
33 ListMultipartUploads: ["s3:ListBucketMultipartUploads"],
34 CompleteMultipartUpload: ["s3:PutObject"],
35 AbortMultipartUpload: ["s3:AbortMultipartUpload"],
36 } as const;
37 type Binding = keyof typeof actions;
38 const bucketInfo = Schema.Array(
39 Schema.Struct({
40 binding: Schema.Literals([
41 "CreateMultipartUpload",
42 "UploadPart",
43 "UploadPartCopy",
44 "UploadPartCopySource",
45 "UploadPartCopyUnboundSource",
46 "ListParts",
47 "ListMultipartUploads",
48 "CompleteMultipartUpload",
49 "AbortMultipartUpload",
50 ]),
51 bucketName: Schema.String,
52 bucketArn: Schema.String,
53 }),
54 );
55 let buckets: typeof bucketInfo.Type = [];
56 let baseUrl: string;
57 let roleName: string;
58 let functionName: string;
59
60 class FixtureNotReady extends Data.TaggedError("FixtureNotReady") {}
61 class UnexpectedFixtureStatus extends Data.TaggedError("UnexpectedFixtureStatus")<{
62 readonly status: number;
63 }> {}
64 class BucketStillExists extends Data.TaggedError("BucketStillExists") {}
65
66 const bucketFor = (binding: Binding) => {
67 const bucket = buckets.find((bucket) => bucket.binding === binding);
68 if (!bucket) throw new Error(`Missing fixture bucket for ${binding}`);
69 return bucket.bucketName;
70 };
71
72 const post = <A, I>(path: string, body: object, schema: Schema.Codec<A, I>, expectedStatus = 200) =>
73 Effect.gen(function* () {
74 const response = yield* HttpClient.execute(
75 HttpClientRequest.post(`${baseUrl}${path}`).pipe(HttpClientRequest.bodyJsonUnsafe(body)),
76 );
77 if (response.status !== expectedStatus) {
78 return yield* Effect.fail(
79 new Error(`${path}: HTTP ${response.status}: ${yield* response.text}`),
80 );
81 }
82 return yield* Schema.decodeUnknownEffect(schema)(yield* response.json);
83 });
84
85 const policyDocument = Schema.fromJsonString(
86 Schema.Struct({
87 Statement: Schema.Array(
88 Schema.Struct({
89 Effect: Schema.String,
90 Action: Schema.Union([Schema.String, Schema.Array(Schema.String)]),
91 Resource: Schema.Union([Schema.String, Schema.Array(Schema.String)]),
92 }),
93 ),
94 }),
95 );
96
97 const assertPermissions = Effect.gen(function* () {
98 const configuration = yield* Lambda.getFunctionConfiguration({ FunctionName: functionName });
99 expect(configuration.FunctionName).toBe(functionName);
100 expect(configuration.Role?.endsWith(`/${roleName}`)).toBe(true);
101 const attached = yield* IAM.listAttachedRolePolicies
102 .pages({ RoleName: roleName })
103 .pipe(Stream.runCollect);
104 expect(
105 attached.flatMap((page) => page.AttachedPolicies ?? []).map((policy) => policy.PolicyArn),
106 ).toEqual(["arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"]);
107 const pages = yield* IAM.listRolePolicies.pages({ RoleName: roleName }).pipe(Stream.runCollect);
108 const policies = yield* Effect.forEach(
109 // Skip the function's log-group guard; only binding grants count.
110 pages
111 .flatMap((page) => page.PolicyNames)
112 .filter((name) => name !== "AlchemyDenyLogGroupCreate"),
113 (PolicyName) =>
114 Effect.gen(function* () {
115 const policy = yield* IAM.getRolePolicy({ RoleName: roleName, PolicyName });
116 const decoded = yield* Effect.try(() => decodeURIComponent(policy.PolicyDocument));
117 return yield* Schema.decodeUnknownEffect(policyDocument)(decoded);
118 }),
119 );
120 const actual = policies
121 .flatMap((policy) =>
122 policy.Statement.flatMap((statement) => {
123 const actionList =
124 typeof statement.Action === "string" ? [statement.Action] : statement.Action;
125 const resources =
126 typeof statement.Resource === "string" ? [statement.Resource] : statement.Resource;
127 return actionList.flatMap((action) =>
128 resources.map((resource) => `${statement.Effect} ${action} ${resource}`),
129 );
130 }),
131 )
132 .sort();
133 const expected = buckets
134 .flatMap((bucket) => [
135 ...actions[bucket.binding].map(
136 (action) =>
137 `Allow ${action} ${bucket.bucketArn}${bucket.binding === "ListMultipartUploads" ? "" : "/*"}`,
138 ),
139 ...(bucket.binding === "UploadPartCopySource"
140 ? [`Allow s3:ListBucket ${bucket.bucketArn}`]
141 : []),
142 ])
143 .sort();
144 expect(actual).toEqual(expected);
145 for (const bucket of buckets) {
146 expect((yield* S3.getBucketVersioning({ Bucket: bucket.bucketName })).Status).toBe("Enabled");
147 }
148 });
149
150 const assertBucketDeleted = (Bucket: string) =>
151 S3.headBucket({ Bucket }).pipe(
152 Effect.flatMap(() => Effect.fail(new BucketStillExists())),
153 Effect.retry({
154 while: (error) => error._tag === "BucketStillExists",
155 schedule: Schedule.spaced("1 second"),
156 times: 10,
157 }),
158 Effect.catchTag("NotFound", () => Effect.void),
159 );
160
161 const abortPending = (Bucket: string, Key: string, UploadId: string) =>
162 S3.abortMultipartUpload({ Bucket, Key, UploadId }).pipe(
163 Effect.catchTag("NoSuchUpload", () => Effect.void),
164 Effect.orDie,
165 );
166
167 const withUpload = <A, E, R>(
168 Bucket: string,
169 Key: string,
170 run: (UploadId: string) => Effect.Effect<A, E, R>,
171 ) =>
172 Effect.gen(function* () {
173 const created = yield* S3.createMultipartUpload({ Bucket, Key });
174 expect(created.UploadId).toBeTruthy();
175 return yield* run(created.UploadId!).pipe(
176 Effect.ensuring(abortPending(Bucket, Key, created.UploadId!)),
177 );
178 });
179
180 const seedVersions = Effect.fn(function* (Bucket: string, Key: string) {
181 const oldBody = "original version";
182 const currentBody = "current version with a different payload";
183 const old = yield* S3.putObject({ Bucket, Key, Body: oldBody });
184 const current = yield* S3.putObject({ Bucket, Key, Body: currentBody });
185 expect(old.VersionId).toBeTruthy();
186 expect(current.VersionId).toBeTruthy();
187 expect(old.VersionId).not.toBe("null");
188 expect(current.VersionId).not.toBe(old.VersionId);
189 return { old: old.VersionId!, current: current.VersionId!, oldBody, currentBody };
190 });
191
192 const assertBody = Effect.fn(function* (
193 Bucket: string,
194 Key: string,
195 VersionId: string | undefined,
196 body: string,
197 ) {
198 const object = yield* S3.getObject({ Bucket, Key, VersionId });
199 if (VersionId !== undefined) expect(object.VersionId).toBe(VersionId);
200 expect(object.Body).toBeDefined();
201 expect(yield* object.Body!.pipe(Stream.decodeText, Stream.mkString)).toBe(body);
202 return object.VersionId;
203 });
204
205 const assertVersions = Effect.fn(function* (
206 Bucket: string,
207 Key: string,
208 versions: { old: string; current: string; oldBody: string; currentBody: string },
209 ) {
210 expect(yield* assertBody(Bucket, Key, undefined, versions.currentBody)).toBe(versions.current);
211 yield* assertBody(Bucket, Key, versions.old, versions.oldBody);
212 const listed = yield* S3.listObjectVersions({ Bucket, Prefix: Key });
213 expect(
214 listed.Versions?.filter((version) => version.Key === Key)
215 .map((version) => ({ id: version.VersionId, latest: version.IsLatest }))
216 .sort((a, b) => a.id!.localeCompare(b.id!)),
217 ).toEqual(
218 [
219 { id: versions.old, latest: false },
220 { id: versions.current, latest: true },
221 ].sort((a, b) => a.id.localeCompare(b.id)),
222 );
223 });
224
225 const minimumPartSize = 5 * 1024 * 1024;
226
227 const assertCompletedVersion = Effect.fn(function* (
228 Bucket: string,
229 Key: string,
230 versions: { old: string; current: string; oldBody: string; currentBody: string },
231 VersionId: string,
232 Body: string,
233 ) {
234 expect(VersionId).toBeTruthy();
235 expect(VersionId).not.toBe("null");
236 expect(VersionId).not.toBe(versions.old);
237 expect(VersionId).not.toBe(versions.current);
238 expect(yield* assertBody(Bucket, Key, undefined, Body)).toBe(VersionId);
239 yield* assertBody(Bucket, Key, VersionId, Body);
240 yield* assertBody(Bucket, Key, versions.old, versions.oldBody);
241 yield* assertBody(Bucket, Key, versions.current, versions.currentBody);
242 const listed = yield* S3.listObjectVersions({ Bucket, Prefix: Key });
243 expect(
244 listed.Versions?.filter((version) => version.Key === Key)
245 .map((version) => ({ id: version.VersionId, latest: version.IsLatest }))
246 .sort((a, b) => a.id!.localeCompare(b.id!)),
247 ).toEqual(
248 [
249 { id: versions.old, latest: false },
250 { id: versions.current, latest: false },
251 { id: VersionId, latest: true },
252 ].sort((a, b) => a.id.localeCompare(b.id)),
253 );
254 expect((yield* S3.listMultipartUploads({ Bucket, Prefix: Key })).Uploads ?? []).toEqual([]);
255 });
256
257 const uploadedPart = Schema.Struct({ ETag: Schema.String });
258 const completedUpload = Schema.Struct({ VersionId: Schema.String, ETag: Schema.String });
259 const noSuchUpload = Schema.Struct({ tag: Schema.Literal("NoSuchUpload") });
260
261 beforeAll(
262 Effect.gen(function* () {
263 yield* stack.destroy();
264 const deployed = yield* stack.deploy(
265 S3VersionedMultipartFunction.pipe(Effect.provide(S3VersionedMultipartFunctionLive)),
266 );
267 expect(deployed.functionUrl).toBeTruthy();
268 baseUrl = deployed.functionUrl!.replace(/\/+$/, "");
269 roleName = deployed.roleName;
270 functionName = deployed.functionName;
271 buckets = yield* HttpClient.get(`${baseUrl}/info`).pipe(
272 Effect.flatMap((response) =>
273 Effect.gen(function* () {
274 if (response.status === 503 || response.status === 502) {
275 return yield* Effect.fail(new FixtureNotReady());
276 }
277 if (response.status === 200) return yield* response.json;
278 return yield* Effect.fail(new UnexpectedFixtureStatus({ status: response.status }));
279 }),
280 ),
281 Effect.flatMap(Schema.decodeUnknownEffect(bucketInfo)),
282 Effect.retry({
283 while: (error) =>
284 error._tag === "FixtureNotReady" ||
285 (error._tag === "HttpClientError" && error.reason._tag === "TransportError"),
286 schedule: Schedule.spaced("2 seconds"),
287 times: 10,
288 }),
289 );
290 expect(buckets.map((bucket) => bucket.binding).sort()).toEqual(Object.keys(actions).sort());
291 expect(new Set(buckets.map((bucket) => bucket.bucketName)).size).toBe(9);
292 yield* Core.withProviders(assertPermissions, options, "S3VersionedMultipart");
293 }),
294 { timeout: 120_000, retry: 0 },
295 );
296
297 afterAll(
298 Effect.gen(function* () {
299 yield* stack.destroy();
300 yield* Core.withProviders(
301 Effect.forEach(buckets, (bucket) => assertBucketDeleted(bucket.bucketName), {
302 discard: true,
303 }),
304 options,
305 "S3VersionedMultipart",
306 );
307 }),
308 { timeout: 120_000, retry: 0 },
309 );
310
311 describe(
312 "CreateMultipartUpload",
313 { tags: ["provider:aws", "provider:aws:iam", "provider:aws:lambda", "provider:aws:s3", "live"] },
314 () => {
315 test.provider(
316 "creates a pending upload without replacing either existing version",
317 () =>
318 Effect.gen(function* () {
319 const Bucket = bucketFor("CreateMultipartUpload");
320 const Key = "versioned-multipart/create.txt";
321 const versions = yield* seedVersions(Bucket, Key);
322 const created = yield* post(
323 "/create",
324 { Key, ContentType: "text/markdown" },
325 Schema.Struct({ Bucket: Schema.String, Key: Schema.String, UploadId: Schema.String }),
326 );
327 yield* Effect.gen(function* () {
328 expect(created.Bucket).toBe(Bucket);
329 expect(created.Key).toBe(Key);
330 expect(created.UploadId).toBeTruthy();
331 const pending = yield* S3.listMultipartUploads({ Bucket, Prefix: Key });
332 expect(pending.Uploads?.map((upload) => upload.UploadId)).toContain(created.UploadId);
333 yield* assertVersions(Bucket, Key, versions);
334 const part = yield* S3.uploadPart({
335 Bucket,
336 Key,
337 UploadId: created.UploadId,
338 PartNumber: 1,
339 Body: "created through Lambda",
340 });
341 const completed = yield* S3.completeMultipartUpload({
342 Bucket,
343 Key,
344 UploadId: created.UploadId,
345 MultipartUpload: { Parts: [{ PartNumber: 1, ETag: part.ETag! }] },
346 });
347 expect(completed.VersionId).toBeTruthy();
348 const head = yield* S3.headObject({ Bucket, Key, VersionId: completed.VersionId });
349 expect(head.ContentType).toBe("text/markdown");
350 yield* assertBody(Bucket, Key, versions.old, versions.oldBody);
351 yield* assertBody(Bucket, Key, versions.current, versions.currentBody);
352 }).pipe(Effect.ensuring(abortPending(Bucket, Key, created.UploadId)));
353 }),
354 { timeout: 120_000, retry: 0 },
355 );
356 },
357 );
358
359 describe(
360 "UploadPart",
361 { tags: ["provider:aws", "provider:aws:iam", "provider:aws:lambda", "provider:aws:s3", "live"] },
362 () => {
363 test.provider(
364 "rejects an out-of-band aborted upload with typed NoSuchUpload",
365 () =>
366 Effect.gen(function* () {
367 const Bucket = bucketFor("UploadPart");
368 const Key = "versioned-multipart/upload-aborted.txt";
369 const versions = yield* seedVersions(Bucket, Key);
370 yield* withUpload(Bucket, Key, (UploadId) =>
371 Effect.gen(function* () {
372 yield* S3.abortMultipartUpload({ Bucket, Key, UploadId });
373 expect(
374 yield* post(
375 "/upload",
376 { Key, UploadId, PartNumber: 1, Body: "rejected part" },
377 noSuchUpload,
378 404,
379 ),
380 ).toEqual({ tag: "NoSuchUpload" });
381 yield* assertVersions(Bucket, Key, versions);
382 expect(
383 (yield* S3.listMultipartUploads({ Bucket, Prefix: Key })).Uploads ?? [],
384 ).toEqual([]);
385 }),
386 );
387 }),
388 { timeout: 120_000, retry: 0 },
389 );
390
391 test.provider(
392 "uploads a part through its isolated binding without changing existing versions",
393 () =>
394 Effect.gen(function* () {
395 const Bucket = bucketFor("UploadPart");
396 const Key = "versioned-multipart/upload.txt";
397 const versions = yield* seedVersions(Bucket, Key);
398 yield* withUpload(Bucket, Key, (UploadId) =>
399 Effect.gen(function* () {
400 const part = yield* post(
401 "/upload",
402 { Key, UploadId, PartNumber: 1, Body: "a", Repeat: minimumPartSize },
403 uploadedPart,
404 );
405 const initial = yield* S3.listParts({ Bucket, Key, UploadId });
406 expect(
407 initial.Parts?.map((entry) => ({
408 part: entry.PartNumber,
409 size: entry.Size,
410 etag: entry.ETag,
411 })),
412 ).toEqual([{ part: 1, size: minimumPartSize, etag: part.ETag }]);
413 yield* assertVersions(Bucket, Key, versions);
414 const replacement = yield* post(
415 "/upload",
416 { Key, UploadId, PartNumber: 1, Body: "b", Repeat: minimumPartSize },
417 uploadedPart,
418 );
419 expect(replacement.ETag).not.toBe(part.ETag);
420 const tail = "final part uploaded through Lambda";
421 const final = yield* post(
422 "/upload",
423 { Key, UploadId, PartNumber: 2, Body: tail },
424 uploadedPart,
425 );
426 const listed = yield* S3.listParts({ Bucket, Key, UploadId });
427 expect(
428 listed.Parts?.map((entry) => ({
429 part: entry.PartNumber,
430 size: entry.Size,
431 etag: entry.ETag,
432 })),
433 ).toEqual([
434 { part: 1, size: minimumPartSize, etag: replacement.ETag },
435 { part: 2, size: tail.length, etag: final.ETag },
436 ]);
437 yield* assertVersions(Bucket, Key, versions);
438 const completed = yield* S3.completeMultipartUpload({
439 Bucket,
440 Key,
441 UploadId,
442 MultipartUpload: {
443 Parts: [
444 { PartNumber: 1, ETag: replacement.ETag },
445 { PartNumber: 2, ETag: final.ETag },
446 ],
447 },
448 });
449 const Body = yield* Effect.sync(() => "b".repeat(minimumPartSize) + tail);
450 yield* assertCompletedVersion(Bucket, Key, versions, completed.VersionId!, Body);
451 }),
452 );
453 }),
454 { timeout: 120_000, retry: 0 },
455 );
456 },
457 );
458
459 describe(
460 "UploadPartCopy",
461 { tags: ["provider:aws", "provider:aws:iam", "provider:aws:lambda", "provider:aws:s3", "live"] },
462 () => {
463 test.provider(
464 "rejects an out-of-band aborted upload with typed NoSuchUpload",
465 () =>
466 Effect.gen(function* () {
467 const Bucket = bucketFor("UploadPartCopy");
468 const Key = "versioned-multipart/copy-aborted/destination.txt";
469 const sourceBucket = bucketFor("UploadPartCopySource");
470 const sourceKey = "versioned-multipart/copy-aborted/source.txt";
471 const sourceVersions = yield* seedVersions(sourceBucket, sourceKey);
472 const destinationVersions = yield* seedVersions(Bucket, Key);
473 const CopySource = yield* Effect.sync(
474 () =>
475 `${sourceBucket}/${sourceKey}?versionId=${encodeURIComponent(sourceVersions.old)}`,
476 );
477 yield* withUpload(Bucket, Key, (UploadId) =>
478 Effect.gen(function* () {
479 yield* S3.abortMultipartUpload({ Bucket, Key, UploadId });
480 expect(
481 yield* post(
482 "/copy",
483 { Key, UploadId, PartNumber: 1, CopySource },
484 noSuchUpload,
485 404,
486 ),
487 ).toEqual({ tag: "NoSuchUpload" });
488 yield* assertVersions(sourceBucket, sourceKey, sourceVersions);
489 yield* assertVersions(Bucket, Key, destinationVersions);
490 expect(
491 (yield* S3.listMultipartUploads({ Bucket, Prefix: Key })).Uploads ?? [],
492 ).toEqual([]);
493 }),
494 );
495 }),
496 { timeout: 120_000, retry: 0 },
497 );
498
499 test.provider(
500 "rejects a deleted source version with typed NoSuchVersion",
501 () =>
502 Effect.gen(function* () {
503 const Bucket = bucketFor("UploadPartCopy");
504 const Key = "versioned-multipart/copy-missing-version/destination.txt";
505 const sourceBucket = bucketFor("UploadPartCopySource");
506 const sourceKey = "versioned-multipart/copy-missing-version/source.txt";
507 const sourceVersions = yield* seedVersions(sourceBucket, sourceKey);
508 const destinationVersions = yield* seedVersions(Bucket, Key);
509 const removed = yield* S3.putObject({
510 Bucket: sourceBucket,
511 Key: sourceKey,
512 Body: "permanently deleted source version",
513 });
514 expect(removed.VersionId).toBeTruthy();
515 yield* S3.deleteObject({
516 Bucket: sourceBucket,
517 Key: sourceKey,
518 VersionId: removed.VersionId!,
519 });
520 const CopySource = yield* Effect.sync(
521 () =>
522 `${sourceBucket}/${sourceKey}?versionId=${encodeURIComponent(removed.VersionId!)}`,
523 );
524 yield* withUpload(Bucket, Key, (UploadId) =>
525 Effect.gen(function* () {
526 expect(
527 yield* post(
528 "/copy",
529 { Key, UploadId, PartNumber: 1, CopySource },
530 Schema.Struct({ tag: Schema.Literal("NoSuchVersion") }),
531 404,
532 ),
533 ).toEqual({ tag: "NoSuchVersion" });
534 expect((yield* S3.listParts({ Bucket, Key, UploadId })).Parts ?? []).toEqual([]);
535 yield* assertVersions(sourceBucket, sourceKey, sourceVersions);
536 yield* assertVersions(Bucket, Key, destinationVersions);
537 }),
538 );
539 expect((yield* S3.listMultipartUploads({ Bucket, Prefix: Key })).Uploads ?? []).toEqual(
540 [],
541 );
542 }),
543 { timeout: 120_000, retry: 0 },
544 );
545
546 test.provider(
547 "rejects an explicitly selected source delete marker with typed InvalidRequest",
548 () =>
549 Effect.gen(function* () {
550 const Bucket = bucketFor("UploadPartCopy");
551 const Key = "versioned-multipart/copy-delete-marker/destination.txt";
552 const sourceBucket = bucketFor("UploadPartCopySource");
553 const sourceKey = "versioned-multipart/copy-delete-marker/source.txt";
554 const sourceVersions = yield* seedVersions(sourceBucket, sourceKey);
555 const destinationVersions = yield* seedVersions(Bucket, Key);
556 const marker = yield* S3.deleteObject({ Bucket: sourceBucket, Key: sourceKey });
557 expect(marker.DeleteMarker).toBe(true);
558 expect(marker.VersionId).toBeTruthy();
559 const sourceState = yield* S3.listObjectVersions({
560 Bucket: sourceBucket,
561 Prefix: sourceKey,
562 });
563 expect(
564 sourceState.DeleteMarkers?.map((entry) => ({
565 id: entry.VersionId,
566 latest: entry.IsLatest,
567 })),
568 ).toEqual([{ id: marker.VersionId, latest: true }]);
569 const CopySource = yield* Effect.sync(
570 () => `${sourceBucket}/${sourceKey}?versionId=${encodeURIComponent(marker.VersionId!)}`,
571 );
572 yield* withUpload(Bucket, Key, (UploadId) =>
573 Effect.gen(function* () {
574 expect(
575 yield* post(
576 "/copy",
577 { Key, UploadId, PartNumber: 1, CopySource },
578 Schema.Struct({ tag: Schema.Literal("InvalidRequest") }),
579 400,
580 ),
581 ).toEqual({ tag: "InvalidRequest" });
582 expect((yield* S3.listParts({ Bucket, Key, UploadId })).Parts ?? []).toEqual([]);
583 yield* assertBody(
584 sourceBucket,
585 sourceKey,
586 sourceVersions.old,
587 sourceVersions.oldBody,
588 );
589 yield* assertBody(
590 sourceBucket,
591 sourceKey,
592 sourceVersions.current,
593 sourceVersions.currentBody,
594 );
595 const after = yield* S3.listObjectVersions({
596 Bucket: sourceBucket,
597 Prefix: sourceKey,
598 });
599 expect(after.Versions).toEqual(sourceState.Versions);
600 expect(after.DeleteMarkers).toEqual(sourceState.DeleteMarkers);
601 yield* assertVersions(Bucket, Key, destinationVersions);
602 }),
603 );
604 expect((yield* S3.listMultipartUploads({ Bucket, Prefix: Key })).Uploads ?? []).toEqual(
605 [],
606 );
607 }),
608 { timeout: 120_000, retry: 0 },
609 );
610
611 test.provider(
612 "copies an encoded source key's old version rather than its differently sized latest version",
613 () =>
614 Effect.gen(function* () {
615 const Bucket = bucketFor("UploadPartCopy");
616 const sourceKey = "versioned-multipart/source space/+plus%percent/雪?#.txt";
617 const Key = "versioned-multipart/destination +%/copy.txt";
618 const versions = yield* seedVersions(Bucket, sourceKey);
619 const CopySource = yield* Effect.sync(
620 () =>
621 `${Bucket}/${sourceKey.split("/").map(encodeURIComponent).join("/")}?versionId=${encodeURIComponent(versions.old)}`,
622 );
623 yield* withUpload(Bucket, Key, (UploadId) =>
624 Effect.gen(function* () {
625 const copied = yield* post(
626 "/copy",
627 { Key, UploadId, PartNumber: 1, CopySource },
628 Schema.Struct({ CopySourceVersionId: Schema.String, CopyPartResult: uploadedPart }),
629 );
630 expect(copied.CopySourceVersionId).toBe(versions.old);
631 expect(copied.CopySourceVersionId).not.toBe(versions.current);
632 const listed = yield* S3.listParts({ Bucket, Key, UploadId });
633 expect(listed.Parts?.map((part) => ({ size: part.Size, etag: part.ETag }))).toEqual([
634 { size: versions.oldBody.length, etag: copied.CopyPartResult.ETag },
635 ]);
636 const completed = yield* S3.completeMultipartUpload({
637 Bucket,
638 Key,
639 UploadId,
640 MultipartUpload: { Parts: [{ PartNumber: 1, ETag: copied.CopyPartResult.ETag }] },
641 });
642 expect(completed.VersionId).toBeTruthy();
643 yield* assertBody(Bucket, Key, completed.VersionId!, versions.oldBody);
644 yield* assertVersions(Bucket, sourceKey, versions);
645 }),
646 );
647 }),
648 { timeout: 120_000, retry: 0 },
649 );
650
651 test.provider(
652 "copies an old version from a separate GetObject-only source bucket",
653 () =>
654 Effect.gen(function* () {
655 const Bucket = bucketFor("UploadPartCopy");
656 const sourceBucket = bucketFor("UploadPartCopySource");
657 const sourceKey = "versioned-multipart/cross-bucket/source +%/雪?#.txt";
658 const Key = "versioned-multipart/cross-bucket/destination.txt";
659 expect(sourceBucket).not.toBe(Bucket);
660 const versions = yield* seedVersions(sourceBucket, sourceKey);
661 const CopySource = yield* Effect.sync(
662 () =>
663 `${sourceBucket}/${sourceKey.split("/").map(encodeURIComponent).join("/")}?versionId=${encodeURIComponent(versions.old)}`,
664 );
665 yield* withUpload(Bucket, Key, (UploadId) =>
666 Effect.gen(function* () {
667 const copied = yield* post(
668 "/copy",
669 { Key, UploadId, PartNumber: 1, CopySource },
670 Schema.Struct({ CopySourceVersionId: Schema.String, CopyPartResult: uploadedPart }),
671 );
672 expect(copied.CopySourceVersionId).toBe(versions.old);
673 expect(copied.CopySourceVersionId).not.toBe(versions.current);
674 const completed = yield* S3.completeMultipartUpload({
675 Bucket,
676 Key,
677 UploadId,
678 MultipartUpload: { Parts: [{ PartNumber: 1, ETag: copied.CopyPartResult.ETag }] },
679 });
680 expect(completed.VersionId).toBeTruthy();
681 expect(completed.VersionId).not.toBe("null");
682 yield* assertBody(Bucket, Key, completed.VersionId!, versions.oldBody);
683 expect(yield* assertBody(Bucket, Key, undefined, versions.oldBody)).toBe(
684 completed.VersionId,
685 );
686 const head = yield* S3.headObject({ Bucket, Key, VersionId: completed.VersionId! });
687 expect(head.VersionId).toBe(completed.VersionId);
688 expect(head.ETag).toBe(completed.ETag);
689 expect(head.ContentLength).toBe(versions.oldBody.length);
690 yield* assertVersions(sourceBucket, sourceKey, versions);
691 }),
692 );
693 }),
694 { timeout: 120_000, retry: 0 },
695 );
696
697 test.provider(
698 "rejects an old version in an unbound source bucket with typed AccessDeniedException",
699 () =>
700 Effect.gen(function* () {
701 const Bucket = bucketFor("UploadPartCopy");
702 const sourceBucket = bucketFor("UploadPartCopyUnboundSource");
703 const sourceKey = "versioned-multipart/unbound/source +%.txt";
704 const Key = "versioned-multipart/unbound/destination.txt";
705 expect(sourceBucket).not.toBe(Bucket);
706 expect(sourceBucket).not.toBe(bucketFor("UploadPartCopySource"));
707 const sourceVersions = yield* seedVersions(sourceBucket, sourceKey);
708 const destinationVersions = yield* seedVersions(Bucket, Key);
709 const CopySource = yield* Effect.sync(
710 () =>
711 `${sourceBucket}/${sourceKey.split("/").map(encodeURIComponent).join("/")}?versionId=${encodeURIComponent(sourceVersions.old)}`,
712 );
713 yield* withUpload(Bucket, Key, (UploadId) =>
714 Effect.gen(function* () {
715 expect(
716 yield* post(
717 "/copy",
718 { Key, UploadId, PartNumber: 1, CopySource },
719 Schema.Struct({ tag: Schema.Literal("AccessDeniedException") }),
720 403,
721 ),
722 ).toEqual({ tag: "AccessDeniedException" });
723 expect((yield* S3.listParts({ Bucket, Key, UploadId })).Parts ?? []).toEqual([]);
724 yield* assertVersions(sourceBucket, sourceKey, sourceVersions);
725 yield* assertVersions(Bucket, Key, destinationVersions);
726 }),
727 );
728 }),
729 { timeout: 120_000, retry: 0 },
730 );
731 },
732 );
733
734 describe(
735 "ListParts",
736 { tags: ["provider:aws", "provider:aws:iam", "provider:aws:lambda", "provider:aws:s3", "live"] },
737 () => {
738 test.provider(
739 "rejects an out-of-band aborted upload with typed NoSuchUpload",
740 () =>
741 Effect.gen(function* () {
742 const Bucket = bucketFor("ListParts");
743 const Key = "versioned-multipart/parts-aborted.txt";
744 const versions = yield* seedVersions(Bucket, Key);
745 yield* withUpload(Bucket, Key, (UploadId) =>
746 Effect.gen(function* () {
747 yield* S3.uploadPart({ Bucket, Key, UploadId, PartNumber: 1, Body: "aborted part" });
748 yield* S3.abortMultipartUpload({ Bucket, Key, UploadId });
749 expect(yield* post("/parts", { Key, UploadId }, noSuchUpload, 404)).toEqual({
750 tag: "NoSuchUpload",
751 });
752 yield* assertVersions(Bucket, Key, versions);
753 expect(
754 (yield* S3.listMultipartUploads({ Bucket, Prefix: Key })).Uploads ?? [],
755 ).toEqual([]);
756 }),
757 );
758 }),
759 { timeout: 120_000, retry: 0 },
760 );
761
762 test.provider(
763 "lists and paginates SDK-seeded parts without touching object versions",
764 () =>
765 Effect.gen(function* () {
766 const Bucket = bucketFor("ListParts");
767 const Key = "versioned-multipart/parts.txt";
768 const versions = yield* seedVersions(Bucket, Key);
769 yield* withUpload(Bucket, Key, (UploadId) =>
770 Effect.gen(function* () {
771 const first = yield* S3.uploadPart({
772 Bucket,
773 Key,
774 UploadId,
775 PartNumber: 1,
776 Body: "first",
777 });
778 const second = yield* S3.uploadPart({
779 Bucket,
780 Key,
781 UploadId,
782 PartNumber: 2,
783 Body: "second part",
784 });
785 const page = Schema.Struct({
786 IsTruncated: Schema.Boolean,
787 NextPartNumberMarker: Schema.optional(Schema.String),
788 Parts: Schema.Array(
789 Schema.Struct({
790 PartNumber: Schema.Number,
791 ETag: Schema.String,
792 Size: Schema.Number,
793 }),
794 ),
795 });
796 const firstPage = yield* post("/parts", { Key, UploadId, MaxParts: 1 }, page);
797 expect(firstPage.IsTruncated).toBe(true);
798 expect(firstPage.Parts).toEqual([{ PartNumber: 1, ETag: first.ETag, Size: 5 }]);
799 expect(firstPage.NextPartNumberMarker).toBeTruthy();
800 const secondPage = yield* post(
801 "/parts",
802 { Key, UploadId, MaxParts: 1, PartNumberMarker: firstPage.NextPartNumberMarker },
803 page,
804 );
805 expect(secondPage.IsTruncated).toBe(false);
806 expect(secondPage.Parts).toEqual([{ PartNumber: 2, ETag: second.ETag, Size: 11 }]);
807 yield* assertVersions(Bucket, Key, versions);
808 }),
809 );
810 }),
811 { timeout: 120_000, retry: 0 },
812 );
813 },
814 );
815
816 describe(
817 "ListMultipartUploads",
818 { tags: ["provider:aws", "provider:aws:iam", "provider:aws:lambda", "provider:aws:s3", "live"] },
819 () => {
820 test.provider(
821 "lists only matching pending uploads, not committed versions or other prefixes",
822 () =>
823 Effect.gen(function* () {
824 const Bucket = bucketFor("ListMultipartUploads");
825 const Key = "versioned-multipart/list/matching.txt";
826 const versions = yield* seedVersions(Bucket, Key);
827 const Prefix = "versioned-multipart/list/";
828 const nextKey = `${Prefix}next.txt`;
829 const excludedKey = "versioned-multipart/other/excluded.txt";
830 const page = Schema.Struct({
831 IsTruncated: Schema.Boolean,
832 NextKeyMarker: Schema.optional(Schema.String),
833 NextUploadIdMarker: Schema.optional(Schema.String),
834 Uploads: Schema.Array(Schema.Struct({ Key: Schema.String, UploadId: Schema.String })),
835 });
836 yield* withUpload(Bucket, Key, (UploadId) =>
837 withUpload(Bucket, Key, (siblingId) =>
838 withUpload(Bucket, nextKey, (nextId) =>
839 withUpload(Bucket, excludedKey, () =>
840 Effect.gen(function* () {
841 const listed = yield* post("/uploads", { Prefix }, page);
842 const expected = [
843 { Key, UploadId },
844 { Key, UploadId: siblingId },
845 { Key: nextKey, UploadId: nextId },
846 ];
847 expect(
848 [...listed.Uploads].sort((a, b) => a.UploadId.localeCompare(b.UploadId)),
849 ).toEqual([...expected].sort((a, b) => a.UploadId.localeCompare(b.UploadId)));
850 const first = yield* post("/uploads", { Prefix, MaxUploads: 1 }, page);
851 expect(first.IsTruncated).toBe(true);
852 expect(first.Uploads).toHaveLength(1);
853 expect(first.Uploads[0].Key).toBe(Key);
854 expect(first.NextKeyMarker).toBe(Key);
855 expect(first.NextUploadIdMarker).toBe(first.Uploads[0].UploadId);
856 const second = yield* post(
857 "/uploads",
858 {
859 Prefix,
860 MaxUploads: 1,
861 KeyMarker: first.NextKeyMarker,
862 UploadIdMarker: first.NextUploadIdMarker,
863 },
864 page,
865 );
866 expect(second.IsTruncated).toBe(true);
867 expect(second.Uploads).toHaveLength(1);
868 expect(second.Uploads[0].Key).toBe(Key);
869 expect(second.NextKeyMarker).toBe(Key);
870 expect(second.NextUploadIdMarker).toBe(second.Uploads[0].UploadId);
871 expect([first.Uploads[0].UploadId, second.Uploads[0].UploadId].sort()).toEqual(
872 [UploadId, siblingId].sort(),
873 );
874 const third = yield* post(
875 "/uploads",
876 {
877 Prefix,
878 MaxUploads: 1,
879 KeyMarker: second.NextKeyMarker,
880 UploadIdMarker: second.NextUploadIdMarker,
881 },
882 page,
883 );
884 expect(third.IsTruncated).toBe(false);
885 expect(third.Uploads).toEqual([{ Key: nextKey, UploadId: nextId }]);
886 yield* assertVersions(Bucket, Key, versions);
887 }),
888 ),
889 ),
890 ),
891 );
892 expect((yield* S3.listMultipartUploads({ Bucket })).Uploads ?? []).toEqual([]);
893 }),
894 { timeout: 120_000, retry: 0 },
895 );
896 },
897 );
898
899 describe(
900 "CompleteMultipartUpload",
901 { tags: ["provider:aws", "provider:aws:iam", "provider:aws:lambda", "provider:aws:s3", "live"] },
902 () => {
903 test.provider(
904 "rejects an out-of-band aborted upload with typed NoSuchUpload",
905 () =>
906 Effect.gen(function* () {
907 const Bucket = bucketFor("CompleteMultipartUpload");
908 const Key = "versioned-multipart/complete-aborted.txt";
909 const versions = yield* seedVersions(Bucket, Key);
910 yield* withUpload(Bucket, Key, (UploadId) =>
911 Effect.gen(function* () {
912 const part = yield* S3.uploadPart({
913 Bucket,
914 Key,
915 UploadId,
916 PartNumber: 1,
917 Body: "aborted part",
918 });
919 expect(part.ETag).toBeTruthy();
920 yield* S3.abortMultipartUpload({ Bucket, Key, UploadId });
921 expect(
922 yield* post(
923 "/complete",
924 {
925 Key,
926 UploadId,
927 MultipartUpload: { Parts: [{ PartNumber: 1, ETag: part.ETag! }] },
928 },
929 noSuchUpload,
930 404,
931 ),
932 ).toEqual({ tag: "NoSuchUpload" });
933 yield* assertVersions(Bucket, Key, versions);
934 expect(
935 (yield* S3.listMultipartUploads({ Bucket, Prefix: Key })).Uploads ?? [],
936 ).toEqual([]);
937 }),
938 );
939 }),
940 { timeout: 120_000, retry: 0 },
941 );
942
943 test.provider(
944 "returns a new version while preserving both previous versions",
945 () =>
946 Effect.gen(function* () {
947 const Bucket = bucketFor("CompleteMultipartUpload");
948 const Key = "versioned-multipart/complete.txt";
949 const versions = yield* seedVersions(Bucket, Key);
950 yield* withUpload(Bucket, Key, (UploadId) =>
951 Effect.gen(function* () {
952 const firstBody = yield* Effect.sync(() => "c".repeat(minimumPartSize));
953 const tail = "new completed multipart version";
954 const Body = firstBody + tail;
955 const part = yield* S3.uploadPart({
956 Bucket,
957 Key,
958 UploadId,
959 PartNumber: 1,
960 Body: firstBody,
961 });
962 const final = yield* S3.uploadPart({
963 Bucket,
964 Key,
965 UploadId,
966 PartNumber: 2,
967 Body: tail,
968 });
969 const completed = yield* post(
970 "/complete",
971 {
972 Key,
973 UploadId,
974 MultipartUpload: {
975 Parts: [
976 { PartNumber: 1, ETag: part.ETag! },
977 { PartNumber: 2, ETag: final.ETag! },
978 ],
979 },
980 },
981 completedUpload,
982 );
983 yield* assertCompletedVersion(Bucket, Key, versions, completed.VersionId, Body);
984 }),
985 );
986 }),
987 { timeout: 120_000, retry: 0 },
988 );
989 },
990 );
991
992 describe(
993 "AbortMultipartUpload",
994 { tags: ["provider:aws", "provider:aws:iam", "provider:aws:lambda", "provider:aws:s3", "live"] },
995 () => {
996 test.provider(
997 "leaves versions and unrelated uploads intact when aborting inactive upload IDs",
998 () =>
999 Effect.gen(function* () {
1000 const Bucket = bucketFor("AbortMultipartUpload");
1001 const Key = "versioned-multipart/invalid-upload-ids.txt";
1002 const sourceKey = "versioned-multipart/invalid-upload-id-source.txt";
1003 const versions = yield* seedVersions(Bucket, Key);
1004 const assertInactiveAbort = (UploadId: string) =>
1005 Effect.gen(function* () {
1006 const response = yield* HttpClient.execute(
1007 HttpClientRequest.post(`${baseUrl}/abort`).pipe(
1008 HttpClientRequest.bodyJsonUnsafe({ Key, UploadId }),
1009 ),
1010 );
1011 expect([200, 404]).toContain(response.status);
1012 expect(yield* response.json).toEqual(
1013 response.status === 200 ? { aborted: true } : { tag: "NoSuchUpload" },
1014 );
1015 });
1016 // Use an issued ID under a different key, not a malformed upload ID.
1017 yield* withUpload(Bucket, sourceKey, (UploadId) =>
1018 Effect.gen(function* () {
1019 yield* assertInactiveAbort(UploadId);
1020 const pending = yield* S3.listMultipartUploads({ Bucket, Prefix: sourceKey });
1021 expect(pending.Uploads?.map((upload) => upload.UploadId)).toEqual([UploadId]);
1022 yield* assertVersions(Bucket, Key, versions);
1023 }),
1024 );
1025 yield* withUpload(Bucket, Key, (UploadId) =>
1026 Effect.gen(function* () {
1027 yield* S3.uploadPart({
1028 Bucket,
1029 Key,
1030 UploadId,
1031 PartNumber: 1,
1032 Body: "aborted upload bytes",
1033 });
1034 yield* post(
1035 "/abort",
1036 { Key, UploadId },
1037 Schema.Struct({ aborted: Schema.Literal(true) }),
1038 );
1039 yield* assertInactiveAbort(UploadId);
1040 yield* assertVersions(Bucket, Key, versions);
1041 }),
1042 );
1043 yield* withUpload(Bucket, Key, (UploadId) =>
1044 Effect.gen(function* () {
1045 const Body = "completed upload survives a rejected abort";
1046 const part = yield* S3.uploadPart({ Bucket, Key, UploadId, PartNumber: 1, Body });
1047 const completed = yield* S3.completeMultipartUpload({
1048 Bucket,
1049 Key,
1050 UploadId,
1051 MultipartUpload: { Parts: [{ PartNumber: 1, ETag: part.ETag! }] },
1052 });
1053 yield* assertInactiveAbort(UploadId);
1054 yield* assertCompletedVersion(Bucket, Key, versions, completed.VersionId!, Body);
1055 }),
1056 );
1057 expect((yield* S3.listMultipartUploads({ Bucket, Prefix: Key })).Uploads ?? []).toEqual(
1058 [],
1059 );
1060 expect(
1061 (yield* S3.listMultipartUploads({ Bucket, Prefix: sourceKey })).Uploads ?? [],
1062 ).toEqual([]);
1063 }),
1064 { timeout: 120_000, retry: 0 },
1065 );
1066
1067 test.provider(
1068 "removes only the selected pending upload, preserving sibling upload and object versions",
1069 () =>
1070 Effect.gen(function* () {
1071 const Bucket = bucketFor("AbortMultipartUpload");
1072 const Key = "versioned-multipart/abort.txt";
1073 const versions = yield* seedVersions(Bucket, Key);
1074 yield* withUpload(Bucket, Key, (UploadId) =>
1075 withUpload(Bucket, Key, (siblingId) =>
1076 Effect.gen(function* () {
1077 yield* S3.uploadPart({
1078 Bucket,
1079 Key,
1080 UploadId,
1081 PartNumber: 1,
1082 Body: "discard this part",
1083 });
1084 const sibling = yield* S3.uploadPart({
1085 Bucket,
1086 Key,
1087 UploadId: siblingId,
1088 PartNumber: 1,
1089 Body: "keep this part",
1090 });
1091 expect(
1092 yield* post(
1093 "/abort",
1094 { Key, UploadId },
1095 Schema.Struct({ aborted: Schema.Boolean }),
1096 ),
1097 ).toEqual({ aborted: true });
1098 const pending = yield* S3.listMultipartUploads({ Bucket, Prefix: Key });
1099 expect(pending.Uploads?.map((upload) => upload.UploadId)).toEqual([siblingId]);
1100 expect(
1101 (yield* S3.listParts({ Bucket, Key, UploadId: siblingId })).Parts?.map(
1102 (part) => part.ETag,
1103 ),
1104 ).toEqual([sibling.ETag]);
1105 yield* assertVersions(Bucket, Key, versions);
1106 }),
1107 ),
1108 );
1109 }),
1110 { timeout: 120_000, retry: 0 },
1111 );
1112 },
1113 );