Content validation/upload stress tests

sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml

@@ -323,3 +323,246 @@ matrix:
       uploadFaults: true
       durationMin: 60
       imageBuildDir: "../../.."
+
+    # --- Content validation (ContentValidation*; default CRC64). Scenario keys are short: K8s label
+    #     testInstance = "{Scenario}-{BaseName}-{revision}" must be <= 63 chars (see stress-test-job.yaml).
+
+    cv-appendblock-sm:
+      testScenario: contentvalidationappendblock
+      sync: true
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-appendblock-lg:
+      testScenario: contentvalidationappendblock
+      sync: true
+      sizeBytes: "26214400"
+      uploadFaults: true
+      durationMin: 30
+      imageBuildDir: "../../.."
+
+    cv-appendblock-async-sm:
+      testScenario: contentvalidationappendblock
+      sync: false
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-appendblock-async-lg:
+      testScenario: contentvalidationappendblock
+      sync: false
+      sizeBytes: "26214400"
+      uploadFaults: true
+      durationMin: 30
+      imageBuildDir: "../../.."
+
+    cv-appendbloboutputstream-sm:
+      testScenario: contentvalidationappendbloboutputstream
+      sync: true
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-appendbloboutputstream-lg:
+      testScenario: contentvalidationappendbloboutputstream
+      sync: true
+      sizeBytes: "10240"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    cv-blockblobupload-sm:
+      testScenario: contentvalidationblockblobupload
+      sync: true
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-blockblobupload-lg:
+      testScenario: contentvalidationblockblobupload
+      sync: true
+      sizeBytes: "26214400"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    cv-blockbloboutputstream-sm:
+      testScenario: contentvalidationblockbloboutputstream
+      sync: true
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-blockbloboutputstream-lg:
+      testScenario: contentvalidationblockbloboutputstream
+      sync: true
+      sizeBytes: "26214400"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    cv-seekablebytechannelwrite-sm:
+      testScenario: contentvalidationseekablebytechannelwrite
+      sync: true
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 10
+      imageBuildDir: "../../.."
+
+    cv-seekablebytechannelwrite-lg:
+      testScenario: contentvalidationseekablebytechannelwrite
+      sync: true
+      sizeBytes: "52428800"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    cv-stageblock-sm:
+      testScenario: contentvalidationstageblock
+      sync: true
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-stageblock-lg:
+      testScenario: contentvalidationstageblock
+      sync: true
+      sizeBytes: "26214400"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    cv-stageblock-async-sm:
+      testScenario: contentvalidationstageblock
+      sync: false
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-stageblock-async-lg:
+      testScenario: contentvalidationstageblock
+      sync: false
+      sizeBytes: "26214400"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    cv-pagebloboutputstream-sm:
+      testScenario: contentvalidationpagebloboutputstream
+      sync: true
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-pagebloboutputstream-lg:
+      testScenario: contentvalidationpagebloboutputstream
+      sync: true
+      sizeBytes: "10240"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    cv-uploadpages-sm:
+      testScenario: contentvalidationuploadpages
+      sync: true
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-uploadpages-lg:
+      testScenario: contentvalidationuploadpages
+      sync: true
+      sizeBytes: "4194304"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    cv-uploadpages-async-sm:
+      testScenario: contentvalidationuploadpages
+      sync: false
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-uploadpages-async-lg:
+      testScenario: contentvalidationuploadpages
+      sync: false
+      sizeBytes: "4194304"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    cv-upload-sm:
+      testScenario: contentvalidationupload
+      sync: true
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-upload-lg:
+      testScenario: contentvalidationupload
+      sync: true
+      sizeBytes: "52428800"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    cv-upload-async-sm:
+      testScenario: contentvalidationupload
+      sync: false
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-upload-async-lg:
+      testScenario: contentvalidationupload
+      sync: false
+      sizeBytes: "52428800"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    cv-uploadfromfile-sm:
+      testScenario: contentvalidationuploadfromfile
+      sync: true
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-uploadfromfile-lg:
+      testScenario: contentvalidationuploadfromfile
+      sync: true
+      sizeBytes: "52428800"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."
+
+    cv-uploadfromfile-async-sm:
+      testScenario: contentvalidationuploadfromfile
+      sync: false
+      sizeBytes: 1024
+      uploadFaults: true
+      durationMin: 25
+      imageBuildDir: "../../.."
+
+    cv-uploadfromfile-async-lg:
+      testScenario: contentvalidationuploadfromfile
+      sync: false
+      sizeBytes: "52428800"
+      uploadFaults: true
+      durationMin: 60
+      imageBuildDir: "../../.."

sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java

@@ -15,6 +15,16 @@ public static void main(String[] args) {
             BlockBlobOutputStream.class,
             BlockBlobUpload.class,
             CommitBlockList.class,
+            ContentValidationAppendBlobOutputStream.class,
+            ContentValidationAppendBlock.class,
+            ContentValidationBlockBlobOutputStream.class,
+            ContentValidationBlockBlobUpload.class,
+            ContentValidationPageBlobOutputStream.class,
+            ContentValidationStageBlock.class,
+            ContentValidationSeekableByteChannelWrite.class,
+            ContentValidationUpload.class,
+            ContentValidationUploadFromFile.class,
+            ContentValidationUploadPages.class,
             DownloadToFile.class,
             DownloadStream.class,
             DownloadContent.class,

sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/ContentValidationAppendBlobOutputStream.java

@@ -0,0 +1,80 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.storage.blob.stress;
+
+import com.azure.core.util.Context;
+import com.azure.core.util.logging.ClientLogger;
+import com.azure.storage.blob.BlobAsyncClient;
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.options.AppendBlobOutputStreamOptions;
+import com.azure.storage.blob.specialized.AppendBlobClient;
+import com.azure.storage.blob.specialized.BlobOutputStream;
+import com.azure.storage.blob.stress.utils.OriginalContent;
+import com.azure.storage.stress.CrcInputStream;
+import reactor.core.publisher.Mono;
+
+import java.io.IOException;
+
+import static com.azure.core.util.FluxUtil.monoError;
+
+/**
+ * Append blob output stream with {@link AppendBlobOutputStreamOptions#setRequestChecksumAlgorithm} (sync only).
+ */
+public class ContentValidationAppendBlobOutputStream extends BlobScenarioBase<ContentValidationStressOptions> {
+    private static final ClientLogger LOGGER = new ClientLogger(ContentValidationAppendBlobOutputStream.class);
+    private final OriginalContent originalContent = new OriginalContent();
+    private final BlobClient syncClient;
+    private final BlobAsyncClient asyncNoFaultClient;
+    /** Separate blob used to upload reference content for {@link OriginalContent} checksum (block blob). */
+    private final BlobAsyncClient tempSetupBlobClient;
+
+    public ContentValidationAppendBlobOutputStream(ContentValidationStressOptions options) {
+        super(options);
+        String blobName = generateBlobName();
+        String tempBlobName = generateBlobName();
+
+        this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName);
+        this.syncClient = getSyncContainerClient().getBlobClient(blobName);
+        this.tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName);
+    }
+
+    @Override
+    protected void runInternal(Context span) throws IOException {
+        AppendBlobClient appendBlobClient = syncClient.getAppendBlobClient();
+        AppendBlobOutputStreamOptions streamOptions = new AppendBlobOutputStreamOptions()
+            .setRequestChecksumAlgorithm(options.getRequestChecksumAlgorithm());
+
+        try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize());
+             BlobOutputStream outputStream = appendBlobClient.getBlobOutputStream(streamOptions)) {
+            byte[] buffer = new byte[4096];
+            int bytesRead;
+
+            while ((bytesRead = inputStream.read(buffer)) != -1) {
+                outputStream.write(buffer, 0, bytesRead);
+            }
+
+            outputStream.close();
+            originalContent.checkMatch(inputStream.getContentInfo(), span).block();
+        }
+    }
+
+    @Override
+    protected Mono<Void> runInternalAsync(Context span) {
+        return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client"));
+    }
+
+    @Override
+    public Mono<Void> setupAsync() {
+        return super.setupAsync()
+            .then(asyncNoFaultClient.getAppendBlobAsyncClient().create())
+            .then(originalContent.setupBlob(tempSetupBlobClient, options.getSize()));
+    }
+
+    @Override
+    public Mono<Void> cleanupAsync() {
+        return asyncNoFaultClient.deleteIfExists()
+            .then(tempSetupBlobClient.deleteIfExists())
+            .then(super.cleanupAsync());
+    }
+}