feat(emulator): adds resumable upload API support for multiple chunk and cancelling

CHANGELOG.md

@@ -1 +1,3 @@
 - Fixed an issue where some MCP tools would error with "Invalid input: expected record, received array". (#10437)
+- feat(emulator): resumable upload API now supports multiple chunk uploads
+- feat(emulator): resumable upload API now supports cancelling an upload

src/emulator/storage/apis/gcloud.ts

@@ -13,7 +13,7 @@ import { EmulatorLogger } from "../../emulatorLogger";
 import { GetObjectResponse, ListObjectsResponse } from "../files";
 import type { Request, Response } from "express";
 import { parseObjectUploadMultipartRequest } from "../multipart";
-import { Upload, UploadNotActiveError } from "../upload";
+import { NotCancellableError, Upload, UploadNotActiveError, UploadStatus } from "../upload";
 import { ForbiddenError, NotFoundError } from "../errors";
 import { reqBodyToBuffer } from "../../shared/request";
 import type { Query } from "express-serve-static-core";
@@ -183,34 +183,172 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
 
   gcloudStorageAPI.put("/upload/storage/v1/b/:bucketId/o", async (req, res) => {
     if (!req.query.upload_id) {
-      res.sendStatus(400);
-      return;
+      return res.sendStatus(404);
+    }
+
+    async function handleStatusCheck(upload: Upload, contentRange: string) {
+      // Extract the numeric total from bytes */total or bytes start-end/total forms.
+      const totalMatch = contentRange && /\/(\d+)$/.exec(contentRange);
+      const declaredTotal = totalMatch ? parseInt(totalMatch[1], 10) : undefined;
+
+      if (upload.status === UploadStatus.FINISHED) {
+        let getObjectResponse: GetObjectResponse;
+        try {
+          getObjectResponse = await adminStorageLayer.getObject({
+            bucketId: upload.bucketId,
+            decodedObjectId: upload.objectId,
+          });
+        } catch (err) {
+          if (err instanceof NotFoundError) {
+            return res.sendStatus(404);
+          }
+          if (err instanceof ForbiddenError) {
+            return res.sendStatus(403);
+          }
+          throw err;
+        }
+        return res.status(200).json(new CloudStorageObjectMetadata(getObjectResponse.metadata));
+      }
+
+      if (upload.status !== UploadStatus.ACTIVE) {
+        return res.sendStatus(400);
+      }
+
+      // if the declared total matches the bytes received so far, the client is signalling that
+      // the upload is complete (i.e. streaming upload where total size wasn't known upfront).
+      if (declaredTotal !== undefined && upload.size === declaredTotal) {
+        const metadata = await finalizeUpload(upload.id);
+        return res.status(200).json(new CloudStorageObjectMetadata(metadata));
+      }
+
+      if (upload.size === 0) {
+        return res.status(308).send();
+      }
+
+      res.setHeader("Range", `bytes=0-${upload.size - 1}`);
+      return res.status(308).send();
+    }
+
+    async function handleChunkedUpload(upload: Upload, contentRange: string, data: Buffer) {
+      const rangeMatch = /^bytes (\d+)-(\d+)(?:\/(\d+|\*))?$/.exec(contentRange);
+      if (!rangeMatch) {
+        return res.status(400).send("Failed to parse Content-Range header.");
+      }
+
+      const rangeStart = parseInt(rangeMatch[1], 10);
+      const rangeEnd = parseInt(rangeMatch[2], 10);
+      const rangeTotal =
+        rangeMatch[3] && rangeMatch[3] !== "*" ? parseInt(rangeMatch[3], 10) : undefined;
+
+      if (rangeEnd < rangeStart || (rangeTotal !== undefined && rangeTotal < rangeEnd + 1)) {
+        return res.status(400).send("Failed to parse Content-Range header.");
+      }
+
+      if (rangeStart !== upload.size) {
+        return res
+          .status(400)
+          .send(`Invalid chunk position. The next chunk should start at offset ${upload.size}.`);
+      }
+
+      const expectedChunkSize = rangeEnd - rangeStart + 1;
+
+      if (data.byteLength !== expectedChunkSize) {
+        const message = `Invalid request.  There were ${data.byteLength} byte(s) in the request body.  There should have been ${expectedChunkSize} byte(s) (starting at offset ${rangeStart} and ending at offset ${rangeEnd}) according to the Content-Range header.`;
+        return res.status(400).send(message);
+      }
+
+      /**
+       * When the client uses an unknown total (`*` or omitted), we treat a chunk whose
+       * size is not a multiple of 256 KiB as the final chunk. Objects whose size is
+       * an exact multiple of 256 KiB must end with an explicit total in Content-Range.
+       * @see https://docs.cloud.google.com/storage/docs/performing-resumable-uploads#chunked-upload
+       */
+      const isComplete =
+        typeof rangeTotal === "number"
+          ? rangeEnd + 1 === rangeTotal
+          : data.byteLength % (256 * 1024) !== 0;
+
+      if (!isComplete && data.byteLength % (256 * 1024) !== 0) {
+        return res
+          .status(400)
+          .send("Chunk size must be a multiple of 256 KiB unless it is the last chunk.");
+      }
+
+      const updatedUpload = uploadService.continueResumableUpload(upload.id, data);
+
+      if (isComplete) {
+        const metadata = await finalizeUpload(upload.id);
+        return res.status(200).json(new CloudStorageObjectMetadata(metadata));
+      }
+
+      res.setHeader("Range", `bytes=0-${updatedUpload.size - 1}`);
+      return res.status(308).send();
+    }
+
+    async function finalizeUpload(uploadId: string) {
+      const finalizedUpload = uploadService.finalizeResumableUpload(uploadId);
+      return await adminStorageLayer.uploadObject(finalizedUpload);
     }
 
-    const uploadId = req.query.upload_id.toString();
     let upload: Upload;
     try {
-      uploadService.continueResumableUpload(uploadId, await reqBodyToBuffer(req));
-      upload = uploadService.finalizeResumableUpload(uploadId);
+      upload = uploadService.getResumableUpload(req.query.upload_id.toString());
+
+      if (upload.status === UploadStatus.CANCELLED) {
+        return res.sendStatus(499);
+      }
+
+      const contentLength = req.headers["content-length"];
+      const contentRange = req.headers["content-range"] ?? "";
+
+      // Status check request
+      // @see https://docs.cloud.google.com/storage/docs/performing-resumable-uploads#status-check
+      if (contentLength === "0") {
+        return await handleStatusCheck(upload, contentRange);
+      }
+
+      if (contentLength && contentLength !== "0") {
+        const data = await reqBodyToBuffer(req);
+
+        // Multiple chunk upload
+        if (contentRange) {
+          return await handleChunkedUpload(upload, contentRange, data);
+        } else {
+          // Single chunk upload
+          uploadService.continueResumableUpload(upload.id, data);
+          const metadata = await finalizeUpload(upload.id);
+          return res.status(200).json(new CloudStorageObjectMetadata(metadata));
+        }
+      }
+      return res.status(400).send();
     } catch (err) {
       if (err instanceof NotFoundError) {
         return res.sendStatus(404);
+      } else if (err instanceof ForbiddenError) {
+        return res.sendStatus(403);
       } else if (err instanceof UploadNotActiveError) {
         return res.sendStatus(400);
       }
       throw err;
     }
+  });
+
+  gcloudStorageAPI.delete("/upload/storage/v1/b/:bucketId/o", async (req, res) => {
+    if (!req.query.upload_id) {
+      return res.sendStatus(405);
+    }
 
-    let metadata: StoredFileMetadata;
     try {
-      metadata = await adminStorageLayer.uploadObject(upload);
+      uploadService.cancelResumableUpload(req.query.upload_id.toString());
+      return res.sendStatus(499);
     } catch (err) {
-      if (err instanceof ForbiddenError) {
-        return res.sendStatus(403);
+      if (err instanceof NotFoundError) {
+        return res.sendStatus(404);
+      } else if (err instanceof NotCancellableError) {
+        return res.sendStatus(405);
       }
       throw err;
     }
-    return res.json(new CloudStorageObjectMetadata(metadata));
   });
 
   gcloudStorageAPI.post("/b/:bucketId/o/:objectId/acl", async (req, res) => {