Multipart uploads

A multipart upload splits a large object into parts that upload independently, then assembles them server-side into one object. Use it for big files (so a failed part retries without re-sending the whole object), for resumable uploads, and for uploading parts in parallel.

The lifecycle is always the same:

1. Create the upload and get an uploadId.
2. Upload each part (1-based partNumber, 1 to 10000), keeping the ETag each part returns.
3. Complete the upload with the ordered list of part numbers and ETags.
4. If anything goes wrong, abort the upload to release the parts.

Every part except the last must be at least 5 MiB; the last part can be any size. An upload that is never completed or aborted is swept by the lifecycle job, so a crashed client does not leak parts forever.

Every part except the last must be at least 5 MiB. A too-small non-final part is rejected at completion, so

size your chunks before you start uploading. :::

A complete large-file upload

The native client mirrors the lifecycle over JSON. Parts stream with no buffering, and the bearer token is refreshed proactively before each streaming part (a streaming body cannot be replayed). This reads a file in fixed-size chunks, uploads each as a part, then completes. On any failure it aborts so no orphaned parts remain.

Node

import { createReadStream } from "node:fs";

const bucket = "reports";
const key = "big.bin";

const mpu = await native.createMultipartUpload(bucket, key, {
  contentType: "application/octet-stream",
});

try {
  const parts = [];
  let partNumber = 1;
  for (const chunk of chunks) {
    const p = await native.uploadPart(bucket, key, mpu.uploadId, partNumber, chunk);
    parts.push({ partNumber, etag: p.etag });
    partNumber++;
  }
  const done = await native.completeMultipartUpload(bucket, key, mpu.uploadId, parts);
  console.log(done.etag, done.versionId);
} catch (err) {
  await native.abortMultipartUpload(bucket, key, mpu.uploadId);
  throw err;
}

Go

mpu, err := native.CreateMultipartUpload(ctx, "reports", "big.bin")
if err != nil {
    return err
}

for n, chunk := range chunks {
    _, err := native.UploadPart(ctx, lockwellnative.UploadPartInput{
        Bucket: "reports", Key: "big.bin", UploadID: mpu.UploadID,
        PartNumber: n + 1, Body: bytes.NewReader(chunk),
    })
    if err != nil {
        _ = native.AbortMultipartUpload(ctx, "reports", "big.bin", mpu.UploadID)
        return err
    }
}

done, err := native.CompleteMultipartUpload(ctx, lockwellnative.CompleteMultipartInput{
    Bucket: "reports", Key: "big.bin", UploadID: mpu.UploadID,
})
if err != nil {
    return err
}
fmt.Println(done.ETag, done.VersionID)

Java

var mpu = native.createMultipartUpload("reports", "big.bin", "application/octet-stream");

try {
    for (int n = 0; n < chunks.size(); n++) {
        native.uploadPart("reports", "big.bin", mpu.uploadId(), n + 1, chunks.get(n));
    }
    var done = native.completeMultipartUpload("reports", "big.bin", mpu.uploadId());
    System.out.println(done.etag());
} catch (Exception e) {
    native.abortMultipartUpload("reports", "big.bin", mpu.uploadId());
    throw e;
}

::::

The Go and Java native CompleteMultipartUpload assemble from the parts already uploaded server-side, so they take no part list. The Node native completeMultipartUpload accepts the {partNumber, etag} list (and uses it to order the assembly).

The S3 client follows the same shape; the body goes up as an aws-chunked stream and completeMultipartUpload takes the ordered ETag list:

Node

import { createReadStream } from "node:fs";

const bucket = "reports";
const key = "big.bin";
const PART = 8 * 1024 * 1024; // 8 MiB

const mpu = await s3.createMultipartUpload(bucket, key, {
  contentType: "application/octet-stream",
});

try {
  const parts = [];
  let partNumber = 1;
  for await (const chunk of chunked(createReadStream(key), PART)) {
    const p = await s3.uploadPart(bucket, key, mpu.uploadId, partNumber, chunk);
    parts.push({ partNumber, etag: p.etag });
    partNumber++;
  }
  const done = await s3.completeMultipartUpload(bucket, key, mpu.uploadId, parts);
  console.log(done.etag, done.versionId);
} catch (err) {
  await s3.abortMultipartUpload(bucket, key, mpu.uploadId);
  throw err;
}

Go

const part = 8 << 20 // 8 MiB

f, err := os.Open("big.bin")
if err != nil {
    return err
}
defer f.Close()

mpu, err := s3.CreateMultipartUpload(ctx, "reports", "big.bin",
    lockwellsdk.WithContentType("application/octet-stream"))
if err != nil {
    return err
}

var parts []lockwellsdk.CompletedPart
buf := make([]byte, part)
for n := 1; ; n++ {
    read, rerr := io.ReadFull(f, buf)
    if read > 0 {
        p, err := s3.UploadPart(ctx, "reports", "big.bin", mpu.UploadID, n, buf[:read])
        if err != nil {
            _ = s3.AbortMultipartUpload(ctx, "reports", "big.bin", mpu.UploadID)
            return err
        }
        parts = append(parts, lockwellsdk.CompletedPart{PartNumber: n, ETag: p.ETag})
    }
    if rerr == io.EOF || rerr == io.ErrUnexpectedEOF {
        break
    }
    if rerr != nil {
        _ = s3.AbortMultipartUpload(ctx, "reports", "big.bin", mpu.UploadID)
        return rerr
    }
}

done, err := s3.CompleteMultipartUpload(ctx, "reports", "big.bin", mpu.UploadID, parts)
if err != nil {
    return err
}
fmt.Println(done.ETag, done.VersionID)

Java

int part = 8 * 1024 * 1024; // 8 MiB
var mpu = s3.createMultipartUpload("reports", "big.bin", "application/octet-stream");

try (var in = new java.io.FileInputStream("big.bin")) {
    var etags = new java.util.ArrayList<String>();
    byte[] buf = new byte[part];
    int read, n = 1;
    while ((read = in.readNBytes(buf, 0, buf.length)) > 0) {
        byte[] chunk = read == buf.length ? buf : java.util.Arrays.copyOf(buf, read);
        etags.add(s3.uploadPart("reports", "big.bin", mpu.uploadId(), n++, chunk));
    }
    var done = s3.completeMultipartUpload("reports", "big.bin", mpu.uploadId(), null, etags);
    System.out.println(done.etag());
} catch (Exception e) {
    s3.abortMultipartUpload("reports", "big.bin", mpu.uploadId());
    throw e;
}

The Java completeMultipartUpload(bucket, key, uploadId, null, etags) infers part numbers from the ETag list order (part 1 is the first ETag). The variant that returns a composite checksum takes explicit part numbers.

Conditional completion (native)

The native completeMultipartUpload accepts If-None-Match: * / If-Match: "<etag>" to gate the assembled object atomically at the commit. This is the multipart equivalent of a conditional create or overwrite, applied at the moment the parts become one object. The S3 client does not gate completion this way.

done, err := native.CompleteMultipartUpload(ctx, lockwellnative.CompleteMultipartInput{
    Bucket: "reports", Key: "big.bin", UploadID: mpu.UploadID,
    IfNoneMatch: "*", // assemble only if the key is still absent (412 otherwise)
})

Per-part checksums

Supply a per-part digest on every uploadPart and read the server-computed composite off the assembled object. Each part is then verified end to end and folded into the composite. This is covered in full on the checksums page; in short, on the native path:

import { computeChecksumBase64 } from "@kelphect/sdk";

const mpu = await native.createMultipartUpload("reports", "big.bin");
const p = await native.uploadPart("reports", "big.bin", mpu.uploadId, 1, chunk, {
  checksums: { crc32c: computeChecksumBase64("CRC32C", chunk) },
});
await native.completeMultipartUpload("reports", "big.bin", mpu.uploadId, [{ partNumber: 1, etag: p.etag }]);
const head = await native.headObject("reports", "big.bin");
console.log(head.checksums.crc32c); // composite over all parts

List in-progress uploads

listMultipartUploads shows every upload that has been created but not yet completed or aborted in a bucket. Use it to find and clean up stale uploads.

Node

const page = await native.listMultipartUploads("reports", { prefix: "big" });
for (const u of page.uploads) console.log(u.key, u.uploadId, u.initiated);

// Or every upload across pages:
for await (const page of native.paginateMultipartUploads("reports")) {
  for (const u of page.uploads) console.log(u.key, u.uploadId);
}

Go

page, err := native.ListMultipartUploads(ctx, "reports")
for _, u := range page.Uploads {
    fmt.Println(u.Key, u.UploadID, u.Initiated)
}

Java

var page = native.listMultipartUploads("reports");
page.uploads().forEach(u -> System.out.println(u.key() + " " + u.uploadId()));

The S3 ListMultipartUploads pages by a (keyMarker, uploadIdMarker) pair:

Node

const page = await s3.listMultipartUploads("reports", { prefix: "big", maxUploads: 100 });
for (const u of page.uploads) console.log(u.key, u.uploadId, u.initiated);

// Or every upload across pages:
for await (const page of s3.paginateMultipartUploads("reports")) {
  for (const u of page.uploads) console.log(u.key, u.uploadId);
}

Go

page, err := s3.ListMultipartUploads(ctx, "reports",
    lockwellsdk.WithUploadsPrefix("big"),
    lockwellsdk.WithMaxUploads(100))
for _, u := range page.Uploads {
    fmt.Println(u.Key, u.UploadID, u.Initiated)
}

Java

import com.lockwell.sdk.LockwellClient.ListUploadsOptions;

var pager = s3.listMultipartUploadsPaginator("reports",
    new ListUploadsOptions().prefix("big"));
while (pager.hasMorePages()) {
    pager.nextPage().uploads().forEach(u ->
        System.out.println(u.key() + " " + u.uploadId()));
}

List the parts of an upload

listParts returns the parts uploaded so far for one in-progress upload, with each part's number, ETag, and size. Use it to resume an interrupted upload (skip the parts already present) or to rebuild the completion list.

Node

const page = await native.listParts("reports", "big.bin", mpu.uploadId);
for (const p of page.parts) console.log(p.partNumber, p.etag, p.size);

Go

page, err := native.ListParts(ctx, "reports", "big.bin", mpu.UploadID)
for _, p := range page.Parts {
    fmt.Println(p.PartNumber, p.ETag, p.Size)
}

Java

var page = native.listParts("reports", "big.bin", mpu.uploadId());
page.parts().forEach(p -> System.out.println(p.partNumber() + " " + p.etag()));

The S3 ListParts pages by partNumberMarker:

Node

const page = await s3.listParts("reports", "big.bin", mpu.uploadId, { maxParts: 100 });
for (const p of page.parts) console.log(p.partNumber, p.etag, p.size);

// Every part across pages:
for await (const page of s3.paginateParts("reports", "big.bin", mpu.uploadId)) {
  for (const p of page.parts) console.log(p.partNumber, p.etag);
}

Go

page, err := s3.ListParts(ctx, "reports", "big.bin", mpu.UploadID,
    lockwellsdk.WithMaxParts(100))
for _, p := range page.Parts {
    fmt.Println(p.PartNumber, p.ETag, p.Size)
}

Java

import com.lockwell.sdk.LockwellClient.ListPartsOptions;

var pager = s3.listPartsPaginator("reports", "big.bin", mpu.uploadId(),
    new ListPartsOptions().maxParts(100));
while (pager.hasMorePages()) {
    pager.nextPage().parts().forEach(p ->
        System.out.println(p.partNumber() + " " + p.etag()));
}

UploadPartCopy: build a part from an existing object

UploadPartCopy fills a part by server-side-copying bytes from an existing object instead of re-uploading them. Pass a byte range to copy a slice; pass an empty range to copy the whole source object. This is how you stitch or re-chunk objects without moving data through the client. Supply a source version id to copy from a specific version. This is an S3-client operation; on the native path, use copyObject for server-side copies.

Node

const cp = await s3.uploadPartCopy(
  "reports",
  "source.bin",
  "", // src bucket, key, version (''=current)
  "reports",
  "assembled.bin",
  mpu.uploadId,
  1, // destination part number
  "bytes=0-5242879", // copy the first 5 MiB ('' = whole object)
);
parts.push({ partNumber: 1, etag: cp.etag });

Go

cp, err := s3.UploadPartCopy(ctx,
    "reports", "source.bin", "", // src bucket, key, version
    "reports", "assembled.bin", mpu.UploadID,
    1,                  // destination part number
    "bytes=0-5242879")  // "" copies the whole object
parts = append(parts, lockwellsdk.CompletedPart{PartNumber: 1, ETag: cp.ETag})

Java

String etag = s3.uploadPartCopy("reports", "assembled.bin", mpu.uploadId(), 1,
    "reports", "source.bin", null, "bytes=0-5242879");

Complete or abort

completeMultipartUpload assembles the parts in part-number order into the final object and returns its ETag and version id. abortMultipartUpload discards the upload and frees its parts. Always abort on failure; an upload left dangling holds its parts until the lifecycle sweep expires it.

The native completion accepts an idempotency key so a retried completion replays the original result instead of failing or double-assembling (Node passes it as the idempotencyKey option).

const done = await native.completeMultipartUpload("reports", "big.bin", mpu.uploadId, parts, {
  idempotencyKey: "assemble-big-bin-2026-001",
});

// Or give up on the upload entirely:
await native.abortMultipartUpload("reports", "big.bin", mpu.uploadId);

const done = await s3.completeMultipartUpload("reports", "big.bin", mpu.uploadId, parts, {
  idempotencyKey: "assemble-big-bin-2026-001",
});

await s3.abortMultipartUpload("reports", "big.bin", mpu.uploadId);

Related

• Checksums for per-part and composite verification.
• Conditional writes for the idempotency key and If-None-Match at completion.
• Errors and retries for which calls retry automatically.
• S3 operations reference for the full operation matrix.