diff --git a/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/implementation/models/InternalBlobChangefeedEventData.java b/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/implementation/models/InternalBlobChangefeedEventData.java index b29c1cf0bea2..2e92d87a1a6d 100644 --- a/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/implementation/models/InternalBlobChangefeedEventData.java +++ b/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/implementation/models/InternalBlobChangefeedEventData.java @@ -8,6 +8,7 @@ import com.azure.storage.internal.avro.implementation.AvroConstants; import com.azure.storage.internal.avro.implementation.schema.AvroSchema; +import java.time.OffsetDateTime; import java.util.Map; import java.util.Objects; @@ -29,6 +30,9 @@ public class InternalBlobChangefeedEventData implements BlobChangefeedEventData private final String blobUrl; private final boolean recursive; private final String sequencer; + private final OffsetDateTime creationTime; + private final OffsetDateTime lastAccessTime; + private final String restoredContainerVersion; /** * Constructs a {@link InternalBlobChangefeedEventData}. @@ -46,10 +50,14 @@ public class InternalBlobChangefeedEventData implements BlobChangefeedEventData * @param blobUrl The blob url. * @param recursive Whether this operation was recursive. * @param sequencer The sequencer. + * @param creationTime The blob creation time. Schema V6. + * @param lastAccessTime The last access time. Schema V7. + * @param restoredContainerVersion The restored container version. Schema V8. */ public InternalBlobChangefeedEventData(String api, String clientRequestId, String requestId, String eTag, String contentType, Long contentLength, BlobType blobType, Long contentOffset, String destinationUrl, - String sourceUrl, String blobUrl, boolean recursive, String sequencer) { + String sourceUrl, String blobUrl, boolean recursive, String sequencer, OffsetDateTime creationTime, + OffsetDateTime lastAccessTime, String restoredContainerVersion) { this.api = api; this.clientRequestId = clientRequestId; this.requestId = requestId; @@ -63,6 +71,9 @@ public InternalBlobChangefeedEventData(String api, String clientRequestId, Strin this.blobUrl = blobUrl; this.recursive = recursive; this.sequencer = sequencer; + this.creationTime = creationTime; + this.lastAccessTime = lastAccessTime; + this.restoredContainerVersion = restoredContainerVersion; } static InternalBlobChangefeedEventData fromRecord(Object d) { @@ -86,6 +97,9 @@ static InternalBlobChangefeedEventData fromRecord(Object d) { Object blobUrl = data.get("url"); Object recursive = data.get("recursive"); Object sequencer = data.get("sequencer"); + Object createTime = data.get("createTime"); + Object lastAccessTime = data.get("lastAccessTime"); + Object restoredContainerVersion = data.get("restoredContainerVersion"); return new InternalBlobChangefeedEventData(ChangefeedTypeValidator.nullOr("api", api, String.class), ChangefeedTypeValidator.nullOr("clientRequestId", clientRequestId, String.class), @@ -101,7 +115,14 @@ static InternalBlobChangefeedEventData fromRecord(Object d) { ChangefeedTypeValidator.nullOr("sourceUrl", sourceUrl, String.class), ChangefeedTypeValidator.nullOr("url", blobUrl, String.class), Boolean.TRUE.equals(ChangefeedTypeValidator.nullOr("recursive", recursive, Boolean.class)), - ChangefeedTypeValidator.nullOr("sequencer", sequencer, String.class)); + ChangefeedTypeValidator.nullOr("sequencer", sequencer, String.class), + ChangefeedTypeValidator.isNull(createTime) + ? null + : OffsetDateTime.parse(ChangefeedTypeValidator.nullOr("createTime", createTime, String.class)), + ChangefeedTypeValidator.isNull(lastAccessTime) + ? null + : OffsetDateTime.parse(ChangefeedTypeValidator.nullOr("lastAccessTime", lastAccessTime, String.class)), + ChangefeedTypeValidator.nullOr("restoredContainerVersion", restoredContainerVersion, String.class)); } @Override @@ -169,6 +190,21 @@ public String getSequencer() { return sequencer; } + @Override + public OffsetDateTime getCreationTime() { + return creationTime; + } + + @Override + public OffsetDateTime getLastAccessTime() { + return lastAccessTime; + } + + @Override + public String getRestoredContainerVersion() { + return restoredContainerVersion; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -190,14 +226,17 @@ && getBlobType() == that.getBlobType() && Objects.equals(getSourceUrl(), that.getSourceUrl()) && Objects.equals(getBlobUrl(), that.getBlobUrl()) && Objects.equals(isRecursive(), that.isRecursive()) - && Objects.equals(getSequencer(), that.getSequencer()); + && Objects.equals(getSequencer(), that.getSequencer()) + && Objects.equals(getCreationTime(), that.getCreationTime()) + && Objects.equals(getLastAccessTime(), that.getLastAccessTime()) + && Objects.equals(getRestoredContainerVersion(), that.getRestoredContainerVersion()); } @Override public int hashCode() { return Objects.hash(getApi(), getClientRequestId(), getRequestId(), getETag(), getContentType(), getContentLength(), getBlobType(), getContentOffset(), getDestinationUrl(), getSourceUrl(), getBlobUrl(), - isRecursive(), getSequencer()); + isRecursive(), getSequencer(), getCreationTime(), getLastAccessTime(), getRestoredContainerVersion()); } @Override @@ -206,6 +245,8 @@ public String toString() { + ", requestId='" + requestId + '\'' + ", eTag='" + eTag + '\'' + ", contentType='" + contentType + '\'' + ", contentLength=" + contentLength + ", blobType=" + blobType + ", contentOffset=" + contentOffset + ", destinationUrl='" + destinationUrl + '\'' + ", sourceUrl='" + sourceUrl + '\'' + ", blobUrl='" - + blobUrl + '\'' + ", recursive=" + recursive + ", sequencer='" + sequencer + '\'' + '}'; + + blobUrl + '\'' + ", recursive=" + recursive + ", sequencer='" + sequencer + '\'' + ", creationTime=" + + creationTime + ", lastAccessTime=" + lastAccessTime + ", restoredContainerVersion='" + + restoredContainerVersion + '\'' + '}'; } } diff --git a/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventData.java b/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventData.java index dc07dfb90e72..b9f917cb8838 100644 --- a/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventData.java +++ b/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventData.java @@ -5,6 +5,8 @@ import com.azure.storage.blob.models.BlobType; +import java.time.OffsetDateTime; + /** * This class contains properties of a BlobChangefeedEventData. */ @@ -101,4 +103,31 @@ public interface BlobChangefeedEventData { */ String getSequencer(); + /** + * Gets the blob creation time. Present in schema V6 and later for AppendBlob data-updated events. + * + * @return The creation time, or null if not present. + */ + default OffsetDateTime getCreationTime() { + return null; + } + + /** + * Gets the last access time of the blob. Present in schema V7 and later. + * + * @return The last access time, or null if not present. + */ + default OffsetDateTime getLastAccessTime() { + return null; + } + + /** + * Gets the restored container version. Present in schema V8 and later for RestoreContainer events. + * + * @return The restored container version, or null if not present. + */ + default String getRestoredContainerVersion() { + return null; + } + } diff --git a/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventType.java b/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventType.java index 358b0cac36ea..38834b1a0920 100644 --- a/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventType.java +++ b/sdk/storage/azure-storage-blob-changefeed/src/main/java/com/azure/storage/blob/changefeed/models/BlobChangefeedEventType.java @@ -22,6 +22,31 @@ public final class BlobChangefeedEventType extends ExpandableStringEnum getMockChangefeedEventDataRecord(BlobChangefe cfEventData.put("url", data.getBlobUrl()); cfEventData.put("sequencer", data.getSequencer()); cfEventData.put("recursive", data.isRecursive()); + cfEventData.put("createTime", data.getCreationTime() != null ? data.getCreationTime().toString() : null); + cfEventData.put("lastAccessTime", + data.getLastAccessTime() != null ? data.getLastAccessTime().toString() : null); + cfEventData.put("restoredContainerVersion", data.getRestoredContainerVersion()); return cfEventData; } } diff --git a/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/MockedChangefeedResources.java b/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/MockedChangefeedResources.java index 39480424de9a..c6de3d53e8c9 100644 --- a/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/MockedChangefeedResources.java +++ b/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/MockedChangefeedResources.java @@ -44,7 +44,8 @@ static BlobChangefeedEvent getMockBlobChangefeedEvent(int index) { static BlobChangefeedEventData getMockBlobChangefeedEventData() { return new InternalBlobChangefeedEventData("PutBlob", "clientRequestId", "requestId", "etag", "application/octet-stream", 100L, BlobType.BLOCK_BLOB, 0L, "destinationUrl", "sourceUrl", "", false, - "sequencer"); + "sequencer", OffsetDateTime.of(2020, 4, 4, 6, 30, 0, 0, ZoneOffset.UTC), + OffsetDateTime.of(2020, 4, 6, 6, 30, 0, 0, ZoneOffset.UTC), "restoredContainerVersion"); } private MockedChangefeedResources() { diff --git a/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/implementation/models/BlobChangefeedEventDeserializationTests.java b/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/implementation/models/BlobChangefeedEventDeserializationTests.java new file mode 100644 index 000000000000..a48d32043550 --- /dev/null +++ b/sdk/storage/azure-storage-blob-changefeed/src/test/java/com/azure/storage/blob/changefeed/implementation/models/BlobChangefeedEventDeserializationTests.java @@ -0,0 +1,463 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.changefeed.implementation.models; + +import com.azure.json.JsonProviders; +import com.azure.json.JsonReader; +import com.azure.json.JsonToken; +import com.azure.storage.blob.changefeed.models.BlobChangefeedEventType; +import com.azure.storage.blob.models.BlobType; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** + * Tests deserialization of BlobChangefeedEvent and BlobChangefeedEventData for schema versions V6, V7, and V8. + */ +public class BlobChangefeedEventDeserializationTests { + + // Values from EventSchemaV6.json / EventSchemaV7.json / EventSchemaV8.json + private static final long CONTENT_OFFSET = 256L; + private static final String CREATE_TIME = "2022-02-17T13:11:52.5901564Z"; + private static final String LAST_ACCESS_TIME = "2022-02-17T13:11:53.5901564Z"; + private static final String RESTORED_CONTAINER_VERSION = "0000000000000002"; + private static final String CLIENT_REQUEST_ID = "clientRequestId"; + private static final String REQUEST_ID = "requestId"; + private static final String ETAG = "0x8D9F2171BE32588"; + private static final String CONTENT_TYPE = "application/octet-stream"; + private static final long CONTENT_LENGTH = 128L; + private static final String SEQUENCER = "00000000000000010000000000000002000000000000001d"; + private static final String DESTINATION_URL = "destinationUrl"; + private static final String SOURCE_URL = "sourceUrl"; + private static final String BLOB_URL = "https://www.myurl.com"; + + // ======================== Schema V6 ======================== + + @Test + public void schemaV6AppendBlobDataUpdatedEventTypeDeserializes() { + assertEquals(BlobChangefeedEventType.APPEND_BLOB_DATA_UPDATED, + BlobChangefeedEventType.fromString("AppendBlobDataUpdated")); + } + + @Test + public void schemaV6ContentOffsetDeserializes() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + r.put("api", "PutBlob"); + r.put("contentOffset", CONTENT_OFFSET); + })); + assertEquals(CONTENT_OFFSET, data.getContentOffset()); + } + + @Test + public void schemaV6CreationTimeDeserializes() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + r.put("api", "PutBlob"); + r.put("createTime", CREATE_TIME); + })); + assertEquals(OffsetDateTime.parse(CREATE_TIME), data.getCreationTime()); + } + + @Test + public void schemaV6ContentOffsetAndCreationTimeNullWhenAbsent() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + })); + assertNull(data.getContentOffset()); + assertNull(data.getCreationTime()); + } + + @Test + public void schemaV6FullEventDeserializes() { + Map eventMap = buildEventRecord(r -> { + r.put("eventType", "BlobCreated"); + r.put("data", buildDataRecord(d -> { + d.put("clientRequestId", CLIENT_REQUEST_ID); + d.put("requestId", REQUEST_ID); + d.put("etag", ETAG); + d.put("contentType", CONTENT_TYPE); + d.put("contentLength", CONTENT_LENGTH); + d.put("blobType", "BlockBlob"); + d.put("contentOffset", CONTENT_OFFSET); + d.put("destinationUrl", DESTINATION_URL); + d.put("sourceUrl", SOURCE_URL); + d.put("url", BLOB_URL); + d.put("recursive", false); + d.put("sequencer", SEQUENCER); + d.put("createTime", CREATE_TIME); + })); + }); + + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + + assertEquals("topic", event.getTopic()); + assertEquals("subject", event.getSubject()); + assertEquals(BlobChangefeedEventType.BLOB_CREATED, event.getEventType()); + assertEquals(OffsetDateTime.of(2022, 2, 17, 13, 12, 11, 0, ZoneOffset.UTC), event.getEventTime()); + assertEquals("62616073-8020-0000-00ff-233467060cc0", event.getId()); + assertEquals(1L, event.getDataVersion()); + assertEquals("1", event.getMetadataVersion()); + assertEquals("PutBlob", event.getData().getApi()); + assertEquals(CLIENT_REQUEST_ID, event.getData().getClientRequestId()); + assertEquals(REQUEST_ID, event.getData().getRequestId()); + assertEquals(ETAG, event.getData().getETag()); + assertEquals(CONTENT_TYPE, event.getData().getContentType()); + assertEquals(CONTENT_LENGTH, event.getData().getContentLength()); + assertEquals(BlobType.BLOCK_BLOB, event.getData().getBlobType()); + assertEquals(CONTENT_OFFSET, event.getData().getContentOffset()); + assertEquals(DESTINATION_URL, event.getData().getDestinationUrl()); + assertEquals(SOURCE_URL, event.getData().getSourceUrl()); + assertEquals(BLOB_URL, event.getData().getBlobUrl()); + assertFalse(event.getData().isRecursive()); + assertEquals(SEQUENCER, event.getData().getSequencer()); + assertEquals(OffsetDateTime.parse(CREATE_TIME), event.getData().getCreationTime()); + assertNull(event.getData().getLastAccessTime()); + assertNull(event.getData().getRestoredContainerVersion()); + } + + // ======================== Schema V7 ======================== + + @Test + public void schemaV7BlobLastAccessTimeUpdatedEventTypeDeserializes() { + assertEquals(BlobChangefeedEventType.BLOB_LAST_ACCESS_TIME_UPDATED, + BlobChangefeedEventType.fromString("BlobLastAccessTimeUpdated")); + } + + @Test + public void schemaV7LastAccessTimeDeserializes() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + r.put("api", "PutBlob"); + r.put("lastAccessTime", LAST_ACCESS_TIME); + })); + assertEquals(OffsetDateTime.parse(LAST_ACCESS_TIME), data.getLastAccessTime()); + } + + @Test + public void schemaV7LastAccessTimeNullWhenAbsent() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + })); + assertNull(data.getLastAccessTime()); + } + + @Test + public void schemaV7FullEventDeserializes() { + Map eventMap = buildEventRecord(r -> { + r.put("eventType", "BlobCreated"); + r.put("data", buildDataRecord(d -> { + d.put("clientRequestId", CLIENT_REQUEST_ID); + d.put("requestId", REQUEST_ID); + d.put("etag", ETAG); + d.put("contentType", CONTENT_TYPE); + d.put("contentLength", CONTENT_LENGTH); + d.put("blobType", "BlockBlob"); + d.put("contentOffset", CONTENT_OFFSET); + d.put("destinationUrl", DESTINATION_URL); + d.put("sourceUrl", SOURCE_URL); + d.put("url", BLOB_URL); + d.put("recursive", false); + d.put("sequencer", SEQUENCER); + d.put("createTime", CREATE_TIME); + d.put("lastAccessTime", LAST_ACCESS_TIME); + })); + }); + + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + + assertEquals("topic", event.getTopic()); + assertEquals("subject", event.getSubject()); + assertEquals(BlobChangefeedEventType.BLOB_CREATED, event.getEventType()); + assertEquals(OffsetDateTime.of(2022, 2, 17, 13, 12, 11, 0, ZoneOffset.UTC), event.getEventTime()); + assertEquals("62616073-8020-0000-00ff-233467060cc0", event.getId()); + assertEquals(1L, event.getDataVersion()); + assertEquals("1", event.getMetadataVersion()); + assertEquals("PutBlob", event.getData().getApi()); + assertEquals(CLIENT_REQUEST_ID, event.getData().getClientRequestId()); + assertEquals(REQUEST_ID, event.getData().getRequestId()); + assertEquals(ETAG, event.getData().getETag()); + assertEquals(CONTENT_TYPE, event.getData().getContentType()); + assertEquals(CONTENT_LENGTH, event.getData().getContentLength()); + assertEquals(BlobType.BLOCK_BLOB, event.getData().getBlobType()); + assertEquals(CONTENT_OFFSET, event.getData().getContentOffset()); + assertEquals(DESTINATION_URL, event.getData().getDestinationUrl()); + assertEquals(SOURCE_URL, event.getData().getSourceUrl()); + assertEquals(BLOB_URL, event.getData().getBlobUrl()); + assertFalse(event.getData().isRecursive()); + assertEquals(SEQUENCER, event.getData().getSequencer()); + assertEquals(OffsetDateTime.parse(CREATE_TIME), event.getData().getCreationTime()); + assertEquals(OffsetDateTime.parse(LAST_ACCESS_TIME), event.getData().getLastAccessTime()); + assertNull(event.getData().getRestoredContainerVersion()); + } + + // ======================== Schema V8 / Container Change Feed ======================== + + @Test + public void schemaV8ContainerCreatedEventTypeDeserializes() { + assertEquals(BlobChangefeedEventType.CONTAINER_CREATED, BlobChangefeedEventType.fromString("ContainerCreated")); + } + + @Test + public void schemaV8ContainerDeletedEventTypeDeserializes() { + assertEquals(BlobChangefeedEventType.CONTAINER_DELETED, BlobChangefeedEventType.fromString("ContainerDeleted")); + } + + @Test + public void schemaV8ContainerPropertiesUpdatedEventTypeDeserializes() { + assertEquals(BlobChangefeedEventType.CONTAINER_PROPERTIES_UPDATED, + BlobChangefeedEventType.fromString("ContainerPropertiesUpdated")); + } + + @Test + public void schemaV8RestoredContainerVersionDeserializes() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + r.put("api", "PutBlob"); + r.put("restoredContainerVersion", RESTORED_CONTAINER_VERSION); + })); + assertEquals(RESTORED_CONTAINER_VERSION, data.getRestoredContainerVersion()); + } + + @Test + public void schemaV8RestoredContainerVersionNullWhenAbsent() { + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(buildDataRecord(r -> { + })); + assertNull(data.getRestoredContainerVersion()); + } + + @Test + public void schemaV8FullEventDeserializes() { + Map eventMap = buildEventRecord(r -> { + r.put("eventType", "BlobCreated"); + r.put("data", buildDataRecord(d -> { + d.put("clientRequestId", CLIENT_REQUEST_ID); + d.put("requestId", REQUEST_ID); + d.put("etag", ETAG); + d.put("contentType", CONTENT_TYPE); + d.put("contentLength", CONTENT_LENGTH); + d.put("blobType", "BlockBlob"); + d.put("contentOffset", CONTENT_OFFSET); + d.put("destinationUrl", DESTINATION_URL); + d.put("sourceUrl", SOURCE_URL); + d.put("url", BLOB_URL); + d.put("recursive", false); + d.put("sequencer", SEQUENCER); + d.put("createTime", CREATE_TIME); + d.put("lastAccessTime", LAST_ACCESS_TIME); + d.put("restoredContainerVersion", RESTORED_CONTAINER_VERSION); + })); + }); + + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + + assertEquals("topic", event.getTopic()); + assertEquals("subject", event.getSubject()); + assertEquals(BlobChangefeedEventType.BLOB_CREATED, event.getEventType()); + assertEquals(OffsetDateTime.of(2022, 2, 17, 13, 12, 11, 0, ZoneOffset.UTC), event.getEventTime()); + assertEquals("62616073-8020-0000-00ff-233467060cc0", event.getId()); + assertEquals(1L, event.getDataVersion()); + assertEquals("1", event.getMetadataVersion()); + assertEquals("PutBlob", event.getData().getApi()); + assertEquals(CLIENT_REQUEST_ID, event.getData().getClientRequestId()); + assertEquals(REQUEST_ID, event.getData().getRequestId()); + assertEquals(ETAG, event.getData().getETag()); + assertEquals(CONTENT_TYPE, event.getData().getContentType()); + assertEquals(CONTENT_LENGTH, event.getData().getContentLength()); + assertEquals(BlobType.BLOCK_BLOB, event.getData().getBlobType()); + assertEquals(CONTENT_OFFSET, event.getData().getContentOffset()); + assertEquals(DESTINATION_URL, event.getData().getDestinationUrl()); + assertEquals(SOURCE_URL, event.getData().getSourceUrl()); + assertEquals(BLOB_URL, event.getData().getBlobUrl()); + assertFalse(event.getData().isRecursive()); + assertEquals(SEQUENCER, event.getData().getSequencer()); + assertEquals(OffsetDateTime.parse(CREATE_TIME), event.getData().getCreationTime()); + assertEquals(OffsetDateTime.parse(LAST_ACCESS_TIME), event.getData().getLastAccessTime()); + assertEquals(RESTORED_CONTAINER_VERSION, event.getData().getRestoredContainerVersion()); + } + + // ======================== JSON File Loading ======================== + + @Test + public void schemaV6JsonFileDeserializes() throws IOException { + Map eventMap = loadJsonAsAvroMap("EventSchemaV6.json"); + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + assertEquals(CONTENT_OFFSET, event.getData().getContentOffset()); + assertEquals(OffsetDateTime.parse(CREATE_TIME), event.getData().getCreationTime()); + assertNull(event.getData().getLastAccessTime()); + assertNull(event.getData().getRestoredContainerVersion()); + } + + @Test + public void schemaV7JsonFileDeserializes() throws IOException { + Map eventMap = loadJsonAsAvroMap("EventSchemaV7.json"); + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + assertEquals(CONTENT_OFFSET, event.getData().getContentOffset()); + assertEquals(OffsetDateTime.parse(CREATE_TIME), event.getData().getCreationTime()); + assertEquals(OffsetDateTime.parse(LAST_ACCESS_TIME), event.getData().getLastAccessTime()); + assertNull(event.getData().getRestoredContainerVersion()); + } + + @Test + public void schemaV8JsonFileDeserializes() throws IOException { + Map eventMap = loadJsonAsAvroMap("EventSchemaV8.json"); + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + assertEquals(CONTENT_OFFSET, event.getData().getContentOffset()); + assertEquals(OffsetDateTime.parse(CREATE_TIME), event.getData().getCreationTime()); + assertEquals(OffsetDateTime.parse(LAST_ACCESS_TIME), event.getData().getLastAccessTime()); + assertEquals(RESTORED_CONTAINER_VERSION, event.getData().getRestoredContainerVersion()); + } + + // ======================== Regression Tests ======================== + + @Test + public void olderSchemaPayloadDeserializesWithoutNewFields() { + Map eventMap = buildEventRecord(r -> { + r.put("eventType", "BlobCreated"); + r.put("data", buildDataRecord(d -> { + d.put("api", "PutBlob"); + d.put("etag", "0x8D9F2171BE32588"); + d.put("contentType", "application/octet-stream"); + d.put("contentLength", 128L); + d.put("blobType", "BlockBlob"); + d.put("url", "https://www.myurl.com"); + })); + }); + + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + + assertEquals(BlobChangefeedEventType.BLOB_CREATED, event.getEventType()); + assertEquals("PutBlob", event.getData().getApi()); + assertNull(event.getData().getContentOffset()); + assertNull(event.getData().getCreationTime()); + assertNull(event.getData().getLastAccessTime()); + assertNull(event.getData().getRestoredContainerVersion()); + } + + @Test + public void existingBlobEventsUnaffected() { + Map eventMap = buildEventRecord(r -> { + r.put("eventType", "BlobDeleted"); + r.put("data", buildDataRecord(d -> { + d.put("api", "DeleteBlob"); + d.put("sequencer", "00000000000000010000000000000002000000000000001d"); + })); + }); + + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + + assertEquals(BlobChangefeedEventType.BLOB_DELETED, event.getEventType()); + assertEquals("DeleteBlob", event.getData().getApi()); + assertNull(event.getData().getCreationTime()); + assertNull(event.getData().getLastAccessTime()); + assertNull(event.getData().getRestoredContainerVersion()); + } + + @Test + public void unknownOptionalFieldsDoNotFailDeserialization() { + Map dataRecord = buildDataRecord(r -> { + r.put("unknownFutureField", "someValue"); + r.put("anotherUnknownField", 42L); + }); + + InternalBlobChangefeedEventData data = InternalBlobChangefeedEventData.fromRecord(dataRecord); + assertEquals("PutBlob", data.getApi()); + } + + @Test + public void dataVersionFieldUnaffected() { + Map eventMap = buildEventRecord(r -> r.put("dataVersion", 8L)); + + InternalBlobChangefeedEvent event = InternalBlobChangefeedEvent.fromRecord(eventMap); + assertEquals(8L, event.getDataVersion()); + } + + // ======================== Helpers ======================== + + @FunctionalInterface + private interface MapCustomizer { + void customize(Map map); + } + + @SuppressWarnings("unchecked") + private static Map loadJsonAsAvroMap(String resourceName) throws IOException { + try (InputStream is = Objects.requireNonNull( + BlobChangefeedEventDeserializationTests.class.getClassLoader().getResourceAsStream(resourceName), + "Test resource not found: " + resourceName); JsonReader reader = JsonProviders.createReader(is)) { + reader.nextToken(); + Map map = readJsonObject(reader); + map.put("$record", "BlobChangeEvent"); + Map data = (Map) map.get("data"); + if (data != null) { + data.put("$record", "BlobChangeEventData"); + } + return map; + } + } + + private static Map readJsonObject(JsonReader reader) throws IOException { + Map map = new HashMap<>(); + while (reader.nextToken() != JsonToken.END_OBJECT) { + String fieldName = reader.getFieldName(); + reader.nextToken(); + map.put(fieldName, readJsonValue(reader)); + } + return map; + } + + private static Object readJsonValue(JsonReader reader) throws IOException { + switch (reader.currentToken()) { + case NULL: + return null; + + case STRING: + return reader.getString(); + + case NUMBER: + return reader.getLong(); + + case BOOLEAN: + return reader.getBoolean(); + + case START_OBJECT: + return readJsonObject(reader); + + case START_ARRAY: + reader.skipChildren(); + return null; + + default: + return null; + } + } + + private static Map buildEventRecord(MapCustomizer customizer) { + Map record = new HashMap<>(); + record.put("$record", "BlobChangeEvent"); + record.put("schemaVersion", 1); + record.put("topic", "topic"); + record.put("subject", "subject"); + record.put("eventType", "BlobCreated"); + record.put("eventTime", OffsetDateTime.of(2022, 2, 17, 13, 12, 11, 0, ZoneOffset.UTC).toString()); + record.put("id", "62616073-8020-0000-00ff-233467060cc0"); + record.put("dataVersion", 1L); + record.put("metadataVersion", "1"); + record.put("data", buildDataRecord(d -> { + })); + customizer.customize(record); + return record; + } + + private static Map buildDataRecord(MapCustomizer customizer) { + Map record = new HashMap<>(); + record.put("$record", "BlobChangeEventData"); + record.put("api", "PutBlob"); + customizer.customize(record); + return record; + } +} diff --git a/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV6.json b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV6.json new file mode 100644 index 000000000000..d6f2b6e18018 --- /dev/null +++ b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV6.json @@ -0,0 +1,84 @@ +{ + "schemaVersion": 6, + "topic": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/haambaga/providers/Microsoft.Storage/storageAccounts/HAAMBAGA-DEV", + "subject": "/blobServices/default/containers/apitestcontainerver/blobs/20220217_131202494_Blob_oaG6iu7ImEB1cX8M", + "eventType": "BlobCreated", + "eventTime": "2022-02-17T13:12:11.5746587Z", + "id": "62616073-8020-0000-00ff-233467060cc0", + "data": { + "api": "PutBlob", + "clientRequestId": "00000000-0000-0000-0000-000000000000", + "requestId": "62616073-8020-0000-00ff-233467000000", + "etag": "0x8D9F2171BE32588", + "contentType": "application/octet-stream", + "contentLength": 128, + "blobType": "BlockBlob", + "blobVersion": "2022-02-17T16:11:52.5901564Z", + "containerVersion": "0000000000000001", + "blobTier": "Archive", + "url": "https://www.myurl.com", + "sequencer": "00000000000000010000000000000002000000000000001d", + "previousInfo": { + "SoftDeleteSnapshot": "2022-02-17T13:12:11.5726507Z", + "WasBlobSoftDeleted": "true", + "BlobVersion": "2024-02-17T16:11:52.0781797Z", + "LastVersion" : "2022-02-17T16:11:52.0781797Z", + "PreviousTier": "Hot" + }, + "snapshot" : "2022-02-17T16:09:16.7261278Z", + "blobPropertiesUpdated" : { + "ContentLanguage" : { + "current" : "pl-Pl", + "previous" : "nl-NL" + }, + "CacheControl" : { + "current" : "max-age=100", + "previous" : "max-age=99" + }, + "ContentEncoding" : { + "current" : "gzip, identity", + "previous" : "gzip" + }, + "ContentMD5" : { + "current" : "Q2h1Y2sgSW51ZwDIAXR5IQ==", + "previous" : "Q2h1Y2sgSW=" + }, + "ContentDisposition" : { + "current" : "attachment", + "previous" : "" + }, + "ContentType" : { + "current" : "application/json", + "previous" : "application/octet-stream" + } + }, + "asyncOperationInfo": { + "DestinationTier": "Hot", + "WasAsyncOperation": "true", + "CopyId": "copyId" + }, + "blobTagsUpdated": { + "previous": { + "Tag1": "Value1_3", + "Tag2": "Value2_3" + }, + "current": { + "Tag1": "Value1_4", + "Tag2": "Value2_4" + } + }, + "restorePointMarker": { + "rpi": "00000000-0000-0000-0000-000000000000", + "rpp": "00000000-0000-0000-0000-000000000000", + "rpl": "test-restore-label", + "rpt": "2022-02-17T13:56:09.3559772Z" + }, + "contentOffset": 256, + "createTime": "2022-02-17T13:11:52.5901564Z", + "storageDiagnostics": { + "bid": "9d726db1-8006-0000-00ff-233467000000", + "seq": "(2,18446744073709551615,29,29)", + "sid": "00000000-0000-0000-0000-000000000000" + } + } +} diff --git a/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV7.json b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV7.json new file mode 100644 index 000000000000..22f1f3554586 --- /dev/null +++ b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV7.json @@ -0,0 +1,85 @@ +{ + "schemaVersion": 7, + "topic": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/haambaga/providers/Microsoft.Storage/storageAccounts/HAAMBAGA-DEV", + "subject": "/blobServices/default/containers/apitestcontainerver/blobs/20220217_131202494_Blob_oaG6iu7ImEB1cX8M", + "eventType": "BlobCreated", + "eventTime": "2022-02-17T13:12:11.5746587Z", + "id": "62616073-8020-0000-00ff-233467060cc0", + "data": { + "api": "PutBlob", + "clientRequestId": "00000000-0000-0000-0000-000000000000", + "requestId": "62616073-8020-0000-00ff-233467000000", + "etag": "0x8D9F2171BE32588", + "contentType": "application/octet-stream", + "contentLength": 128, + "blobType": "BlockBlob", + "blobVersion": "2022-02-17T16:11:52.5901564Z", + "containerVersion": "0000000000000001", + "blobTier": "Archive", + "url": "https://www.myurl.com", + "sequencer": "00000000000000010000000000000002000000000000001d", + "previousInfo": { + "SoftDeleteSnapshot": "2022-02-17T13:12:11.5726507Z", + "WasBlobSoftDeleted": "true", + "BlobVersion": "2024-02-17T16:11:52.0781797Z", + "LastVersion" : "2022-02-17T16:11:52.0781797Z", + "PreviousTier": "Hot" + }, + "snapshot" : "2022-02-17T16:09:16.7261278Z", + "blobPropertiesUpdated" : { + "ContentLanguage" : { + "current" : "pl-Pl", + "previous" : "nl-NL" + }, + "CacheControl" : { + "current" : "max-age=100", + "previous" : "max-age=99" + }, + "ContentEncoding" : { + "current" : "gzip, identity", + "previous" : "gzip" + }, + "ContentMD5" : { + "current" : "Q2h1Y2sgSW51ZwDIAXR5IQ==", + "previous" : "Q2h1Y2sgSW=" + }, + "ContentDisposition" : { + "current" : "attachment", + "previous" : "" + }, + "ContentType" : { + "current" : "application/json", + "previous" : "application/octet-stream" + } + }, + "asyncOperationInfo": { + "DestinationTier": "Hot", + "WasAsyncOperation": "true", + "CopyId": "copyId" + }, + "blobTagsUpdated": { + "previous": { + "Tag1": "Value1_3", + "Tag2": "Value2_3" + }, + "current": { + "Tag1": "Value1_4", + "Tag2": "Value2_4" + } + }, + "restorePointMarker": { + "rpi": "00000000-0000-0000-0000-000000000000", + "rpp": "00000000-0000-0000-0000-000000000000", + "rpl": "test-restore-label", + "rpt": "2022-02-17T13:56:09.3559772Z" + }, + "contentOffset": 256, + "createTime": "2022-02-17T13:11:52.5901564Z", + "lastAccessTime": "2022-02-17T13:11:53.5901564Z", + "storageDiagnostics": { + "bid": "9d726db1-8006-0000-00ff-233467000000", + "seq": "(2,18446744073709551615,29,29)", + "sid": "00000000-0000-0000-0000-000000000000" + } + } +} diff --git a/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV8.json b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV8.json new file mode 100644 index 000000000000..167d1478ec31 --- /dev/null +++ b/sdk/storage/azure-storage-blob-changefeed/src/test/resources/EventSchemaV8.json @@ -0,0 +1,86 @@ +{ + "schemaVersion": 8, + "topic": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/haambaga/providers/Microsoft.Storage/storageAccounts/HAAMBAGA-DEV", + "subject": "/blobServices/default/containers/apitestcontainerver/blobs/20220217_131202494_Blob_oaG6iu7ImEB1cX8M", + "eventType": "BlobCreated", + "eventTime": "2022-02-17T13:12:11.5746587Z", + "id": "62616073-8020-0000-00ff-233467060cc0", + "data": { + "api": "PutBlob", + "clientRequestId": "00000000-0000-0000-0000-000000000000", + "requestId": "62616073-8020-0000-00ff-233467000000", + "etag": "0x8D9F2171BE32588", + "contentType": "application/octet-stream", + "contentLength": 128, + "blobType": "BlockBlob", + "blobVersion": "2022-02-17T16:11:52.5901564Z", + "containerVersion": "0000000000000001", + "blobTier": "Archive", + "url": "https://www.myurl.com", + "sequencer": "00000000000000010000000000000002000000000000001d", + "previousInfo": { + "SoftDeleteSnapshot": "2022-02-17T13:12:11.5726507Z", + "WasBlobSoftDeleted": "true", + "BlobVersion": "2024-02-17T16:11:52.0781797Z", + "LastVersion" : "2022-02-17T16:11:52.0781797Z", + "PreviousTier": "Hot" + }, + "snapshot" : "2022-02-17T16:09:16.7261278Z", + "blobPropertiesUpdated" : { + "ContentLanguage" : { + "current" : "pl-Pl", + "previous" : "nl-NL" + }, + "CacheControl" : { + "current" : "max-age=100", + "previous" : "max-age=99" + }, + "ContentEncoding" : { + "current" : "gzip, identity", + "previous" : "gzip" + }, + "ContentMD5" : { + "current" : "Q2h1Y2sgSW51ZwDIAXR5IQ==", + "previous" : "Q2h1Y2sgSW=" + }, + "ContentDisposition" : { + "current" : "attachment", + "previous" : "" + }, + "ContentType" : { + "current" : "application/json", + "previous" : "application/octet-stream" + } + }, + "asyncOperationInfo": { + "DestinationTier": "Hot", + "WasAsyncOperation": "true", + "CopyId": "copyId" + }, + "blobTagsUpdated": { + "previous": { + "Tag1": "Value1_3", + "Tag2": "Value2_3" + }, + "current": { + "Tag1": "Value1_4", + "Tag2": "Value2_4" + } + }, + "restorePointMarker": { + "rpi": "00000000-0000-0000-0000-000000000000", + "rpp": "00000000-0000-0000-0000-000000000000", + "rpl": "test-restore-label", + "rpt": "2022-02-17T13:56:09.3559772Z" + }, + "restoredContainerVersion": "0000000000000002", + "contentOffset": 256, + "createTime": "2022-02-17T13:11:52.5901564Z", + "lastAccessTime": "2022-02-17T13:11:53.5901564Z", + "storageDiagnostics": { + "bid": "9d726db1-8006-0000-00ff-233467000000", + "seq": "(2,18446744073709551615,29,29)", + "sid": "00000000-0000-0000-0000-000000000000" + } + } +} diff --git a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml index 3a8920a67b35..4587e20a1bd6 100644 --- a/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml +++ b/sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml @@ -108,6 +108,150 @@ matrix: durationMin: 60 imageBuildDir: "../../.." + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation + crc64downloadstreamsm: + testScenario: downloadstreamwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation and async client + crc64downloadstreamasyncsm: + testScenario: downloadstreamwithcrc64 + sync: false + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation and large payload + crc64downloadstreamlg: + testScenario: downloadstreamwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadStreamOptions with CRC64 validation, async client, and large payload + crc64downloadstreamasynclg: + testScenario: downloadstreamwithcrc64 + sync: false + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation + crc64downloadcontentsm: + testScenario: downloadcontentwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation and async client + crc64downloadcontentasyncsm: + testScenario: downloadcontentwithcrc64 + sync: false + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation and large payload + crc64downloadcontentlg: + testScenario: downloadcontentwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadContentOptions with CRC64 validation, async client, and large payload + crc64downloadcontentasynclg: + testScenario: downloadcontentwithcrc64 + sync: false + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation + crc64downloadfilesm: + testScenario: downloadtofilewithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation and async client + crc64downloadfileasyncsm: + testScenario: downloadtofilewithcrc64 + sync: false + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation and multi-block payload + crc64downloadfilemd: + testScenario: downloadtofilewithcrc64 + sync: true + sizeBytes: "16777216" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobDownloadToFileOptions with CRC64 validation, async client, and multi-block payload + crc64downloadfileasyncmd: + testScenario: downloadtofilewithcrc64 + sync: false + sizeBytes: "16777216" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobInputStreamOptions with CRC64 validation + crc64inputstreamsm: + testScenario: openinputstreamwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobInputStreamOptions with CRC64 validation and large payload + crc64inputstreamlg: + testScenario: openinputstreamwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + # content validation downloads using BlobSeekableByteChannelReadOptions with CRC64 validation + crc64bytechannelreadsm: + testScenario: openseekablebytechannelreadwithcrc64 + sync: true + sizeBytes: 1024 + downloadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + # content validation downloads using BlobSeekableByteChannelReadOptions with CRC64 validation and large payload + crc64bytechannelreadlg: + testScenario: openseekablebytechannelreadwithcrc64 + sync: true + sizeBytes: "52428800" + downloadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + # this test uploads 1KB (1024 bytes) to append blob, no chunking appendblocksmall: testScenario: appendblock @@ -323,3 +467,243 @@ matrix: uploadFaults: true durationMin: 60 imageBuildDir: "../../.." + + crc64appendblock-sm: + testScenario: appendblockwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64appendblock-lg: + testScenario: appendblockwithcrc64 + sync: true + sizeBytes: "26214400" + uploadFaults: true + durationMin: 30 + imageBuildDir: "../../.." + + crc64appendblockasync-sm: + testScenario: appendblockwithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64appendblockasync-lg: + testScenario: appendblockwithcrc64 + sync: false + sizeBytes: "26214400" + uploadFaults: true + durationMin: 30 + imageBuildDir: "../../.." + + crc64appendoutputstream-sm: + testScenario: appendbloboutputstreamwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64appendoutputstream-lg: + testScenario: appendbloboutputstreamwithcrc64 + sync: true + sizeBytes: "10240" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64blockblobupload-sm: + testScenario: blockblobuploadwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64blockblobupload-lg: + testScenario: blockblobuploadwithcrc64 + sync: true + sizeBytes: "26214400" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64blockoutputstream-sm: + testScenario: blockbloboutputstreamwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64blockoutputstream-lg: + testScenario: blockbloboutputstreamwithcrc64 + sync: true + sizeBytes: "26214400" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64bytechannelwrite-sm: + testScenario: seekablebytechannelwritewithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 10 + imageBuildDir: "../../.." + + crc64bytechannelwrite-lg: + testScenario: seekablebytechannelwritewithcrc64 + sync: true + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64stageblock-sm: + testScenario: stageblockwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64stageblock-lg: + testScenario: stageblockwithcrc64 + sync: true + sizeBytes: "26214400" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64stageblockasync-sm: + testScenario: stageblockwithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64stageblockasync-lg: + testScenario: stageblockwithcrc64 + sync: false + sizeBytes: "26214400" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64pageoutputstream-sm: + testScenario: pagebloboutputstreamwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64pageoutputstream-lg: + testScenario: pagebloboutputstreamwithcrc64 + sync: true + sizeBytes: "10240" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadpages-sm: + testScenario: uploadpageswithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadpages-lg: + testScenario: uploadpageswithcrc64 + sync: true + sizeBytes: "4194304" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadpagesasync-sm: + testScenario: uploadpageswithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadpagesasync-lg: + testScenario: uploadpageswithcrc64 + sync: false + sizeBytes: "4194304" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64upload-sm: + testScenario: uploadwithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64upload-lg: + testScenario: uploadwithcrc64 + sync: true + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadasync-sm: + testScenario: uploadwithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadasync-lg: + testScenario: uploadwithcrc64 + sync: false + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadfromfile-sm: + testScenario: uploadfromfilewithcrc64 + sync: true + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadfromfile-lg: + testScenario: uploadfromfilewithcrc64 + sync: true + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." + + crc64uploadfromfileasync-sm: + testScenario: uploadfromfilewithcrc64 + sync: false + sizeBytes: 1024 + uploadFaults: true + durationMin: 25 + imageBuildDir: "../../.." + + crc64uploadfromfileasync-lg: + testScenario: uploadfromfilewithcrc64 + sync: false + sizeBytes: "52428800" + uploadFaults: true + durationMin: 60 + imageBuildDir: "../../.." diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java index e38bd16791ca..f146945db423 100644 --- a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/App.java @@ -15,6 +15,21 @@ public static void main(String[] args) { BlockBlobOutputStream.class, BlockBlobUpload.class, CommitBlockList.class, + DownloadContentWithCRC64.class, + DownloadStreamWithCRC64.class, + DownloadToFileWithCRC64.class, + OpenInputStreamWithCRC64.class, + OpenSeekableByteChannelReadWithCRC64.class, + AppendBlobOutputStreamWithCRC64.class, + AppendBlockWithCRC64.class, + BlockBlobOutputStreamWithCRC64.class, + BlockBlobUploadWithCRC64.class, + PageBlobOutputStreamWithCRC64.class, + StageBlockWithCRC64.class, + SeekableByteChannelWriteWithCRC64.class, + UploadWithCRC64.class, + UploadFromFileWithCRC64.class, + UploadPagesWithCRC64.class, DownloadToFile.class, DownloadStream.class, DownloadContent.class, diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java new file mode 100644 index 000000000000..7ea8a3841f32 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlobOutputStreamWithCRC64.java @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.core.util.Context; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.options.AppendBlobOutputStreamOptions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.specialized.AppendBlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import com.azure.storage.stress.CrcInputStream; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Append blob output stream with CRC64 enabled (sync only). + */ +public class AppendBlobOutputStreamWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(AppendBlobOutputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + /** Separate blob used to upload reference content for {@link OriginalContent} checksum (block blob). */ + private final BlobAsyncClient tempSetupBlobClient; + + public AppendBlobOutputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + AppendBlobClient appendBlobClient = syncClient.getAppendBlobClient(); + // Reset the append blob at the start of each iteration. The boolean overload + // getBlobOutputStream(true) does this implicitly via create(true); the options overload + // does not, so we replicate that behavior here. Without this reset, fault-injection + // sequences that commit a block server-side but drop the response leave the cached + // appendPosition stale, causing subsequent retries to fail with 412 AppendPositionConditionNotMet, + // which combined with non-retriable Crc64Mismatch on truncated-body faults collapses the pass rate. + appendBlobClient.create(true); + + AppendBlobOutputStreamOptions streamOptions = new AppendBlobOutputStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); + BlobOutputStream outputStream = appendBlobClient.getBlobOutputStream(streamOptions)) { + byte[] buffer = new byte[4096]; + int bytesRead; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + outputStream.close(); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(asyncNoFaultClient.getAppendBlobAsyncClient().create()) + .then(originalContent.setupBlob(tempSetupBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(tempSetupBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java new file mode 100644 index 000000000000..360d19995080 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/AppendBlockWithCRC64.java @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; +import com.azure.storage.blob.specialized.AppendBlobAsyncClient; +import com.azure.storage.blob.specialized.AppendBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Append block with CRC64 enabled. + */ +public class AppendBlockWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final BlobAsyncClient tempSetupBlobClient; + + public AppendBlockWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + this.tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + } + + @Override + protected void runInternal(Context span) { + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + AppendBlobClient appendBlobClient = syncClient.getAppendBlobClient(); + appendBlobClient.appendBlockWithResponse( + new AppendBlobAppendBlockOptions(inputStream, options.getSize()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + AppendBlobAsyncClient appendBlobAsyncClient = asyncClient.getAppendBlobAsyncClient(); + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + return appendBlobAsyncClient.appendBlockWithResponse( + new AppendBlobAppendBlockOptions(byteBufferFlux, options.getSize()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(asyncNoFaultClient.getAppendBlobAsyncClient().create()) + .then(originalContent.setupBlob(tempSetupBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.getAppendBlobAsyncClient().deleteIfExists() + .onErrorResume(e -> Mono.empty()) + .then(tempSetupBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java new file mode 100644 index 000000000000..1873ac740d72 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobOutputStreamWithCRC64.java @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlockBlobOutputStreamOptions; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Block blob output stream with CRC64 enabled (sync only). + */ +public class BlockBlobOutputStreamWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(BlockBlobOutputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public BlockBlobOutputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.parallelTransferOptions = new ParallelTransferOptions().setMaxConcurrency(options.getMaxConcurrency()); + } + + @Override + protected void runInternal(Context span) throws IOException { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + BlockBlobOutputStreamOptions streamOptions = new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); + BlobOutputStream outputStream = blockBlobClient.getBlobOutputStream(streamOptions, span)) { + byte[] buffer = new byte[4096]; + int bytesRead; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + outputStream.close(); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists().then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java new file mode 100644 index 000000000000..b88b9b5c808d --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlockBlobUploadWithCRC64.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; +import com.azure.storage.blob.specialized.BlockBlobAsyncClient; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Single-shot block blob upload with request content validation with CRC64 enabled. + */ +public class BlockBlobUploadWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public BlockBlobUploadWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + blockBlobClient.uploadWithResponse( + new BlockBlobSimpleUploadOptions(inputStream, options.getSize()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + BlockBlobAsyncClient blockBlobAsyncClient = asyncClient.getBlockBlobAsyncClient(); + return blockBlobAsyncClient.uploadWithResponse( + new BlockBlobSimpleUploadOptions(byteBufferFlux, options.getSize()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java new file mode 100644 index 000000000000..6c92e1563c60 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadContentWithCRC64.java @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +/** + * Download content with CRC64 Algorithm enabled. + * Verifies the correctness of the download response content via CRC. + */ +public class DownloadContentWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public DownloadContentWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) { + originalContent.checkMatch( + syncClient.downloadContentWithResponse( + new BlobDownloadContentOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span).getValue(), + span).block(); + } + + @Override + protected Mono runInternalAsync(Context span) { + // TODO return downloadContent once it stops buffering. + return asyncClient.downloadStreamWithResponse( + new BlobDownloadStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(response -> { + long contentLength = Long.valueOf(response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + return BinaryData.fromFlux(response.getValue(), contentLength, false); + }) + .flatMap(bd -> originalContent.checkMatch(bd, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java new file mode 100644 index 000000000000..64cccb2c7dd4 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadStreamWithCRC64.java @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcOutputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +/** + * Streaming blob download with CRC64 Algorithm enabled. + * Verifies the correctness of the download response content via CRC. + */ +public class DownloadStreamWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public DownloadStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + try (CrcOutputStream outputStream = new CrcOutputStream()) { + syncClient.downloadStreamWithResponse(outputStream, + new BlobDownloadStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + outputStream.close(); + originalContent.checkMatch(outputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return asyncClient.downloadStreamWithResponse( + new BlobDownloadStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(response -> originalContent.checkMatch(response.getValue(), span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java new file mode 100644 index 000000000000..ceb2faa8b153 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/DownloadToFileWithCRC64.java @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.UUID; + +/** + * Download to file with CRC64 Algorithm enabled. + * Verifies the correctness of the download response content via CRC. + */ +public class DownloadToFileWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(DownloadToFileWithCRC64.class); + private final Path directoryPath; + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public DownloadToFileWithCRC64(StorageStressOptions options) { + super(options); + this.directoryPath = getTempPath("test"); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + this.parallelTransferOptions = new ParallelTransferOptions() + .setMaxConcurrency(options.getMaxConcurrency()); + } + + @Override + protected void runInternal(Context span) { + Path downloadPath = directoryPath.resolve(UUID.randomUUID() + ".txt"); + BlobDownloadToFileOptions blobOptions = new BlobDownloadToFileOptions(downloadPath.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try { + syncClient.downloadToFileWithResponse(blobOptions, Duration.ofSeconds(options.getDuration()), span); + originalContent.checkMatch(BinaryData.fromFile(downloadPath), span).block(); + } finally { + deleteFile(downloadPath); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return Mono.using( + () -> directoryPath.resolve(UUID.randomUUID() + ".txt"), + path -> asyncClient.downloadToFileWithResponse( + new BlobDownloadToFileOptions(path.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(ignored -> originalContent.checkMatch(BinaryData.fromFile(path), span)), + DownloadToFileWithCRC64::deleteFile); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } + + private Path getTempPath(String prefix) { + try { + return Files.createTempDirectory(prefix); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + } + + private static void deleteFile(Path path) { + try { + Files.deleteIfExists(path); + } catch (Throwable e) { + LOGGER.atError() + .addKeyValue("path", path) + .log("failed to delete file", e); + } + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java new file mode 100644 index 000000000000..eaae1570560b --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenInputStreamWithCRC64.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlobInputStreamOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.io.InputStream; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Open input stream with CRC64 Algorithm enabled (sync only). + * Verifies the correctness of the download response content via CRC. + */ +public class OpenInputStreamWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(OpenInputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public OpenInputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + try (InputStream stream = syncClient.openInputStream( + new BlobInputStreamOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + span)) { + try (CrcInputStream crcStream = new CrcInputStream(stream)) { + byte[] buffer = new byte[8192]; + while (crcStream.read(buffer) != -1) { + // do nothing + } + originalContent.checkMatch(crcStream.getContentInfo(), span).block(); + } + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("openInputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java new file mode 100644 index 000000000000..e6376238fe92 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/OpenSeekableByteChannelReadWithCRC64.java @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.BlobSeekableByteChannelReadResult; +import com.azure.storage.blob.options.BlobSeekableByteChannelReadOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.channels.Channels; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Seekable byte channel read with CRC64 Algorithm enabled (sync only). + * Verifies the correctness of the download response content via CRC. + */ +public class OpenSeekableByteChannelReadWithCRC64 + extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(OpenSeekableByteChannelReadWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public OpenSeekableByteChannelReadWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + BlobSeekableByteChannelReadResult result = syncClient.openSeekableByteChannelRead( + new BlobSeekableByteChannelReadOptions() + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + span); + try (CrcInputStream crcStream = new CrcInputStream(Channels.newInputStream(result.getChannel()))) { + byte[] buffer = new byte[8192]; + while (crcStream.read(buffer) != -1) { + // do nothing + } + originalContent.checkMatch(crcStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, + new RuntimeException("openSeekableByteChannelRead() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java new file mode 100644 index 000000000000..f4dda3295b18 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/PageBlobOutputStreamWithCRC64.java @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.options.PageBlobOutputStreamOptions; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.blob.specialized.PageBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static com.azure.core.util.FluxUtil.monoError; + +/** + * Page blob output stream with CRC64 enabled (sync only). + */ +public class PageBlobOutputStreamWithCRC64 extends PageBlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(PageBlobOutputStreamWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + /** Page blob used only to seed {@link OriginalContent} (same pattern as {@link PageBlobOutputStream}). */ + private final PageBlobAsyncClient tempSetupPageBlobClient; + + public PageBlobOutputStreamWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + BlobAsyncClient tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + this.tempSetupPageBlobClient = tempSetupBlobClient.getPageBlobAsyncClient(); + } + + @Override + protected void runInternal(Context span) throws IOException { + PageBlobClient pageBlobClient = syncClient.getPageBlobClient(); + PageBlobOutputStreamOptions streamOptions = new PageBlobOutputStreamOptions( + new PageRange().setStart(0).setEnd(options.getSize() - 1)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()); + BlobOutputStream outputStream = pageBlobClient.getBlobOutputStream(streamOptions)) { + ByteArrayOutputStream bufferStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[512]; + int bytesRead; + + while ((bytesRead = inputStream.read(buffer)) != -1) { + // Always accumulate into bufferStream to avoid dropping or reordering bytes + bufferStream.write(buffer, 0, bytesRead); + // Flush all full 512-byte pages from the accumulator + if (bufferStream.size() >= buffer.length) { + byte[] toWrite = bufferStream.toByteArray(); + int length = toWrite.length - (toWrite.length % buffer.length); + if (length > 0) { + outputStream.write(toWrite, 0, length); + bufferStream.reset(); + // Keep any remaining partial page bytes in the accumulator + bufferStream.write(toWrite, length, toWrite.length - length); + } + } + } + // For page blobs, total content size must be a multiple of 512 bytes. + // Any remaining bytes here indicate misalignment and would result in silent truncation. + if (bufferStream.size() != 0) { + throw new IOException("Remaining bytes in buffer that do not align to 512-byte page size."); + } + + outputStream.close(); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException("getBlobOutputStream() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(asyncNoFaultClient.getPageBlobAsyncClient().create(options.getSize())) + .then(tempSetupPageBlobClient.create(options.getSize())) + .then(originalContent.setupPageBlob(tempSetupPageBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.getPageBlobAsyncClient().deleteIfExists() + .onErrorResume(e -> Mono.empty()) + .then(tempSetupPageBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java new file mode 100644 index 000000000000..5f345bbfd5f1 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/SeekableByteChannelWriteWithCRC64.java @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.StorageSeekableByteChannel; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static com.azure.core.util.FluxUtil.monoError; +import static com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE; + +/** + * Block-blob seekable byte channel write with CRC64 enabled (sync only). + */ +public class SeekableByteChannelWriteWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(SeekableByteChannelWriteWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncNoFaultClient; + + public SeekableByteChannelWriteWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) throws IOException { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + BlockBlobSeekableByteChannelWriteOptions writeOptions = new BlockBlobSeekableByteChannelWriteOptions(OVERWRITE) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + try (CrcInputStream crcInput = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + Flux byteBufferFlux = crcInput.convertStreamToByteBuffer(); + try (StorageSeekableByteChannel channel = (StorageSeekableByteChannel) blockBlobClient.openSeekableByteChannelWrite( + writeOptions)) { + Mono writeOperation = byteBufferFlux + .doOnNext(buffer -> { + try { + while (buffer.hasRemaining()) { + channel.write(buffer); + } + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new RuntimeException(e)); + } + }).then(); + writeOperation.block(); + channel.getWriteBehavior().commit(options.getSize()); + } + originalContent.checkMatch(byteBufferFlux, span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + return monoError(LOGGER, new RuntimeException( + "openSeekableByteChannelWrite() does not exist on the async client")); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists().then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java new file mode 100644 index 000000000000..c06bbe6a44b6 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/StageBlockWithCRC64.java @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.options.BlockBlobCommitBlockListOptions; +import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.blob.specialized.BlockBlobAsyncClient; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Collections; + +/** + * Stage block with CRC64 enabled on the faulted client, then commit via the non-faulted client. + */ +public class StageBlockWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobClient syncNoFaultClient; + private final BlobAsyncClient asyncNoFaultClient; + + public StageBlockWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.syncNoFaultClient = getSyncContainerClientNoFault().getBlobClient(blobName); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + } + + @Override + protected void runInternal(Context span) { + BlockBlobClient blockBlobClient = syncClient.getBlockBlobClient(); + BlockBlobClient blockBlobClientNoFault = syncNoFaultClient.getBlockBlobClient(); + String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString() + .getBytes(StandardCharsets.UTF_8)); + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + BinaryData data = BinaryData.fromStream(inputStream, options.getSize()); + blockBlobClient.stageBlockWithResponse( + new BlockBlobStageBlockOptions(blockId, data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + blockBlobClientNoFault.commitBlockListWithResponse( + new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)), null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + BlockBlobAsyncClient blockBlobAsyncClient = asyncClient.getBlockBlobAsyncClient(); + BlockBlobAsyncClient blockBlobAsyncClientNoFault = asyncNoFaultClient.getBlockBlobAsyncClient(); + String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString() + .getBytes(StandardCharsets.UTF_8)); + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + return BinaryData.fromFlux(byteBufferFlux, options.getSize(), false) + .flatMap(binaryData -> blockBlobAsyncClient.stageBlockWithResponse( + new BlockBlobStageBlockOptions(blockId, binaryData) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))) + .then(blockBlobAsyncClientNoFault.commitBlockListWithResponse( + new BlockBlobCommitBlockListOptions(Collections.singletonList(blockId)))) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java new file mode 100644 index 000000000000..b9594326e6ad --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadFromFileWithCRC64.java @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.CoreUtils; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Mono; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.UUID; + +/** + * Upload from file with CRC64 enabled. + */ +public class UploadFromFileWithCRC64 extends BlobScenarioBase { + private static final ClientLogger LOGGER = new ClientLogger(UploadFromFileWithCRC64.class); + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobClient syncNoFaultClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public UploadFromFileWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncNoFaultClient = getSyncContainerClientNoFault().getBlobClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + this.parallelTransferOptions = new ParallelTransferOptions() + .setMaxConcurrency(options.getMaxConcurrency()) + .setMaxSingleUploadSizeLong(4 * 1024 * 1024L); + } + + @Override + protected void runInternal(Context span) { + Path downloadPath = getTempPath("test"); + Path uploadFilePath = null; + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + uploadFilePath = generateFile(inputStream); + downloadPath = downloadPath.resolve(CoreUtils.randomUuid() + ".txt"); + syncClient.uploadFromFileWithResponse(new BlobUploadFromFileOptions(uploadFilePath.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + syncNoFaultClient.downloadToFileWithResponse( + new BlobDownloadToFileOptions(downloadPath.toString()), null, span); + originalContent.checkMatch(BinaryData.fromFile(downloadPath), span).block(); + } finally { + deleteFile(downloadPath); + deleteFile(uploadFilePath); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + Path downloadPath = getTempPath("test"); + // This is written differently than the other runInternalAsync methods because uploadFromFile requires a file + // path, so we need to generate the temp file. + return Mono.using( + () -> new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()), + inputStream -> uploadAndVerifyAsync(inputStream, downloadPath, span), + CrcInputStream::close); + } + + private Mono uploadAndVerifyAsync(CrcInputStream inputStream, Path downloadDir, Context span) { + Path uploadFilePath = generateFile(inputStream); + Path downloadFilePath = downloadDir.resolve(UUID.randomUUID() + ".txt"); + + return asyncClient.uploadFromFileWithResponse(new BlobUploadFromFileOptions(uploadFilePath.toString()) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(ignored -> asyncNoFaultClient.downloadToFileWithResponse( + new BlobDownloadToFileOptions(downloadFilePath.toString()))) + .flatMap(ignored -> originalContent.checkMatch(BinaryData.fromFile(downloadFilePath), span)) + .doFinally(signal -> { + deleteFile(uploadFilePath); + deleteFile(downloadFilePath); + }); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } + + private Path getTempPath(String prefix) { + try { + return Files.createTempDirectory(prefix); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + } + + private static void deleteFile(Path path) { + try { + Files.deleteIfExists(path); + } catch (Throwable e) { + LOGGER.atError() + .addKeyValue("path", path) + .log("failed to delete file", e); + } + } + + private static Path generateFile(InputStream inputStream) { + try { + File file = Files.createTempFile(CoreUtils.randomUuid().toString(), ".txt").toFile(); + file.deleteOnExit(); + Files.copy(inputStream, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + return file.toPath(); + } catch (IOException e) { + throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); + } + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java new file mode 100644 index 000000000000..3572871bea9f --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadPagesWithCRC64.java @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.blob.specialized.PageBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Page blob upload pages with CRC64 enabled. + */ +public class UploadPagesWithCRC64 extends PageBlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final PageBlobAsyncClient tempSetupPageBlobClient; + + public UploadPagesWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + String tempBlobName = generateBlobName(); + + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + BlobAsyncClient tempSetupBlobClient = getAsyncContainerClientNoFault().getBlobAsyncClient(tempBlobName); + this.tempSetupPageBlobClient = tempSetupBlobClient.getPageBlobAsyncClient(); + } + + @Override + protected void runInternal(Context span) { + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + PageBlobClient pageBlobClient = syncClient.getPageBlobClient(); + PageRange range = new PageRange().setStart(0).setEnd(options.getSize() - 1); + pageBlobClient.uploadPagesWithResponse( + new PageBlobUploadPagesOptions(range, inputStream) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), + null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + PageBlobAsyncClient pageBlobAsyncClient = asyncClient.getPageBlobAsyncClient(); + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + PageRange range = new PageRange().setStart(0).setEnd(options.getSize() - 1); + return pageBlobAsyncClient.uploadPagesWithResponse( + new PageBlobUploadPagesOptions(range, byteBufferFlux) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync() + .then(asyncNoFaultClient.getPageBlobAsyncClient().create(options.getSize())) + .then(tempSetupPageBlobClient.create(options.getSize())) + .then(originalContent.setupPageBlob(tempSetupPageBlobClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.getPageBlobAsyncClient().deleteIfExists() + .onErrorResume(e -> Mono.empty()) + .then(tempSetupPageBlobClient.deleteIfExists()) + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java new file mode 100644 index 000000000000..55340ba93946 --- /dev/null +++ b/sdk/storage/azure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/UploadWithCRC64.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.stress; + +import com.azure.core.util.Context; +import com.azure.storage.blob.BlobAsyncClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlobParallelUploadOptions; +import com.azure.storage.blob.stress.utils.OriginalContent; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.stress.CrcInputStream; +import com.azure.storage.stress.StorageStressOptions; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; + +/** + * Parallel blob upload with CRC64 enabled. + */ +public class UploadWithCRC64 extends BlobScenarioBase { + private final OriginalContent originalContent = new OriginalContent(); + private final BlobClient syncClient; + private final BlobAsyncClient asyncClient; + private final BlobAsyncClient asyncNoFaultClient; + private final ParallelTransferOptions parallelTransferOptions; + + public UploadWithCRC64(StorageStressOptions options) { + super(options); + String blobName = generateBlobName(); + this.asyncNoFaultClient = getAsyncContainerClientNoFault().getBlobAsyncClient(blobName); + this.syncClient = getSyncContainerClient().getBlobClient(blobName); + this.asyncClient = getAsyncContainerClient().getBlobAsyncClient(blobName); + parallelTransferOptions = new ParallelTransferOptions() + .setMaxConcurrency(options.getMaxConcurrency()) + .setMaxSingleUploadSizeLong(4 * 1024 * 1024L); + } + + @Override + protected void runInternal(Context span) { + try (CrcInputStream inputStream = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize())) { + syncClient.uploadWithResponse(new BlobParallelUploadOptions(inputStream) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, span); + originalContent.checkMatch(inputStream.getContentInfo(), span).block(); + } + } + + @Override + protected Mono runInternalAsync(Context span) { + Flux byteBufferFlux = new CrcInputStream(originalContent.getBlobContentHead(), options.getSize()) + .convertStreamToByteBuffer(); + return asyncClient.uploadWithResponse(new BlobParallelUploadOptions(byteBufferFlux) + .setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .then(originalContent.checkMatch(byteBufferFlux, span)); + } + + @Override + public Mono setupAsync() { + return super.setupAsync().then(originalContent.setupBlob(asyncNoFaultClient, options.getSize())); + } + + @Override + public Mono cleanupAsync() { + return asyncNoFaultClient.deleteIfExists() + .then(super.cleanupAsync()); + } +} diff --git a/sdk/storage/azure-storage-blob/assets.json b/sdk/storage/azure-storage-blob/assets.json index 6b2f467873bf..769ef2797967 100644 --- a/sdk/storage/azure-storage-blob/assets.json +++ b/sdk/storage/azure-storage-blob/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "java", "TagPrefix": "java/storage/azure-storage-blob", - "Tag": "java/storage/azure-storage-blob_447cc62a15" + "Tag": "java/storage/azure-storage-blob_494b72fca7" } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java index 5c4b16b3b9e9..6f4a32f70dda 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java @@ -37,10 +37,14 @@ import com.azure.storage.blob.specialized.BlockBlobClient; import com.azure.storage.blob.specialized.PageBlobAsyncClient; import com.azure.storage.blob.specialized.SpecializedBlobClientBuilder; +import com.azure.storage.common.ContentValidationAlgorithm; + import com.azure.storage.common.implementation.BufferStagingArea; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.implementation.UploadUtils; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -695,16 +699,23 @@ public Mono> uploadWithResponse(BlobParallelUploadOption ? new BlobImmutabilityPolicy() : options.getImmutabilityPolicy(); final Boolean legalHold = options.isLegalHold(); + final ContentValidationAlgorithm contentValidationAlgorithm = options.getContentValidationAlgorithm(); + + ContentValidationModeResolver.validateTransactionalChecksumOptions(computeMd5, contentValidationAlgorithm); + ContentValidationModeResolver.validateProgressWithContentValidation( + parallelTransferOptions.getProgressListener(), contentValidationAlgorithm); BlockBlobAsyncClient blockBlobAsyncClient = getBlockBlobAsyncClient(); Function, Mono>> uploadInChunksFunction = (stream) -> uploadInChunks(blockBlobAsyncClient, stream, parallelTransferOptions, headers, metadata, - tags, tier, requestConditions, computeMd5, immutabilityPolicy, legalHold); + tags, tier, requestConditions, computeMd5, immutabilityPolicy, legalHold, + contentValidationAlgorithm); BiFunction, Long, Mono>> uploadFullBlobFunction = (stream, length) -> uploadFullBlob(blockBlobAsyncClient, stream, length, parallelTransferOptions, - headers, metadata, tags, tier, requestConditions, computeMd5, immutabilityPolicy, legalHold); + headers, metadata, tags, tier, requestConditions, computeMd5, immutabilityPolicy, legalHold, + contentValidationAlgorithm); Flux data = options.getDataFlux(); data = UploadUtils.extractByteBuffer(data, options.getOptionalLength(), @@ -721,7 +732,7 @@ private Mono> uploadFullBlob(BlockBlobAsyncClient blockB Flux data, long length, ParallelTransferOptions parallelTransferOptions, BlobHttpHeaders headers, Map metadata, Map tags, AccessTier tier, BlobRequestConditions requestConditions, boolean computeMd5, BlobImmutabilityPolicy immutabilityPolicy, - Boolean legalHold) { + Boolean legalHold, ContentValidationAlgorithm contentValidationAlgorithm) { /* * Note that there is no need to buffer here as the flux returned by the size gate in this case is created @@ -738,7 +749,8 @@ private Mono> uploadFullBlob(BlockBlobAsyncClient blockB .setImmutabilityPolicy(immutabilityPolicy) .setLegalHold(legalHold)) .flatMap(options -> { - Mono> responseMono = blockBlobAsyncClient.uploadWithResponse(options); + Mono> responseMono = ContentValidationModeResolver.addContentValidationMode( + blockBlobAsyncClient.uploadWithResponse(options), contentValidationAlgorithm, length, false); if (parallelTransferOptions.getProgressListener() != null) { ProgressReporter progressReporter = ProgressReporter.withProgressListener(parallelTransferOptions.getProgressListener()); @@ -753,7 +765,7 @@ private Mono> uploadInChunks(BlockBlobAsyncClient blockB Flux data, ParallelTransferOptions parallelTransferOptions, BlobHttpHeaders headers, Map metadata, Map tags, AccessTier tier, BlobRequestConditions requestConditions, boolean computeMd5, BlobImmutabilityPolicy immutabilityPolicy, - Boolean legalHold) { + Boolean legalHold, ContentValidationAlgorithm contentValidationAlgorithm) { // TODO: Sample/api reference ProgressListener progressListener = parallelTransferOptions.getProgressListener(); @@ -777,12 +789,12 @@ private Mono> uploadInChunks(BlockBlobAsyncClient blockB .concatWith(Flux.defer(stagingArea::flush)) .flatMapSequential(bufferAggregator -> { Flux chunkData = bufferAggregator.asFlux(); - String blockId = Base64.getEncoder().encodeToString(CoreUtils.randomUuid().toString().getBytes(UTF_8)); return UploadUtils.computeMd5(chunkData, computeMd5, LOGGER).flatMap(fluxMd5Wrapper -> { - Mono> responseMono - = blockBlobAsyncClient.stageBlockWithResponse(blockId, fluxMd5Wrapper.getData(), - bufferAggregator.length(), fluxMd5Wrapper.getMd5(), requestConditions.getLeaseId()); + Mono> responseMono = ContentValidationModeResolver.addContentValidationMode( + blockBlobAsyncClient.stageBlockWithResponse(blockId, fluxMd5Wrapper.getData(), + bufferAggregator.length(), fluxMd5Wrapper.getMd5(), requestConditions.getLeaseId()), + contentValidationAlgorithm, 0, true); if (progressReporter != null) { responseMono = responseMono.contextWrite(FluxUtil.toReactorContext(Contexts.empty() .setHttpRequestProgressReporter(progressReporter.createChild()) @@ -968,7 +980,12 @@ public Mono> uploadFromFileWithResponse(BlobUploadFromFi : options.getParallelTransferOptions().getBlockSizeLong(); final ParallelTransferOptions finalParallelTransferOptions = ModelHelper.populateAndApplyDefaults(options.getParallelTransferOptions()); + final ContentValidationAlgorithm contentValidationAlgorithm = options.getContentValidationAlgorithm(); + try { + ContentValidationModeResolver.validateProgressWithContentValidation( + finalParallelTransferOptions.getProgressListener(), contentValidationAlgorithm); + Path filePath = Paths.get(options.getFilePath()); BlockBlobAsyncClient blockBlobAsyncClient = getBlockBlobAsyncClient(); // This will retrieve file length but won't read file body. @@ -980,15 +997,17 @@ public Mono> uploadFromFileWithResponse(BlobUploadFromFi if (fileSize > finalParallelTransferOptions.getMaxSingleUploadSizeLong()) { return uploadFileChunks(fileSize, finalParallelTransferOptions, originalBlockSize, options.getHeaders(), options.getMetadata(), options.getTags(), options.getTier(), options.getRequestConditions(), - filePath, blockBlobAsyncClient); + filePath, blockBlobAsyncClient, contentValidationAlgorithm); } else { // Otherwise, we know it can be sent in a single request reducing network overhead. - Mono> responseMono = blockBlobAsyncClient - .uploadWithResponse(new BlockBlobSimpleUploadOptions(fullFileData).setHeaders(options.getHeaders()) - .setMetadata(options.getMetadata()) - .setTags(options.getTags()) - .setTier(options.getTier()) - .setRequestConditions(options.getRequestConditions())); + Mono> responseMono = ContentValidationModeResolver.addContentValidationMode( + blockBlobAsyncClient.uploadWithResponse( + new BlockBlobSimpleUploadOptions(fullFileData).setHeaders(options.getHeaders()) + .setMetadata(options.getMetadata()) + .setTags(options.getTags()) + .setTier(options.getTier()) + .setRequestConditions(options.getRequestConditions())), + contentValidationAlgorithm, fileSize, false); if (finalParallelTransferOptions.getProgressListener() != null) { ProgressReporter progressReporter = ProgressReporter.withProgressListener(finalParallelTransferOptions.getProgressListener()); @@ -1005,7 +1024,8 @@ public Mono> uploadFromFileWithResponse(BlobUploadFromFi private Mono> uploadFileChunks(long fileSize, ParallelTransferOptions parallelTransferOptions, Long originalBlockSize, BlobHttpHeaders headers, Map metadata, Map tags, AccessTier tier, - BlobRequestConditions requestConditions, Path filePath, BlockBlobAsyncClient client) { + BlobRequestConditions requestConditions, Path filePath, BlockBlobAsyncClient client, + ContentValidationAlgorithm contentValidationAlgorithm) { final BlobRequestConditions finalRequestConditions = (requestConditions == null) ? new BlobRequestConditions() : requestConditions; // parallelTransferOptions are finalized in the calling method. @@ -1023,8 +1043,10 @@ private Mono> uploadFileChunks(long fileSize, BinaryData data = BinaryData.fromFile(filePath, chunk.getOffset(), chunk.getCount(), DEFAULT_FILE_READ_CHUNK_SIZE); - Mono> responseMono = client.stageBlockWithResponse( - new BlockBlobStageBlockOptions(blockId, data).setLeaseId(finalRequestConditions.getLeaseId())); + Mono> responseMono = ContentValidationModeResolver.addContentValidationMode( + client.stageBlockWithResponse( + new BlockBlobStageBlockOptions(blockId, data).setLeaseId(finalRequestConditions.getLeaseId())), + contentValidationAlgorithm, 0, true); if (progressReporter != null) { responseMono = responseMono.contextWrite(FluxUtil.toReactorContext( Contexts.empty().setHttpRequestProgressReporter(progressReporter.createChild()).getContext())); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceVersion.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceVersion.java index 75fb74a59a5d..d7a60722a3ec 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceVersion.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobServiceVersion.java @@ -162,7 +162,17 @@ public enum BlobServiceVersion implements ServiceVersion { /** * Service version {@code 2026-06-06}. */ - V2026_06_06("2026-06-06"); + V2026_06_06("2026-06-06"), + + /** + * Service version {@code 2026-10-06}. + */ + V2026_10_06("2026-10-06"), + + /** + * Service version {@code 2026-12-06}. + */ + V2026_12_06("2026-12-06"); private final String version; @@ -184,6 +194,6 @@ public String getVersion() { * @return the latest {@link BlobServiceVersion} */ public static BlobServiceVersion getLatest() { - return V2026_06_06; + return V2026_12_06; } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobsDownloadHeaders.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobsDownloadHeaders.java index 1d409a5a4cdd..7dd424fe666c 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobsDownloadHeaders.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/models/BlobsDownloadHeaders.java @@ -295,6 +295,30 @@ public final class BlobsDownloadHeaders { @Generated private Long xMsStructuredContentLength; + /* + * The x-ms-access-tier property. + */ + @Generated + private String xMsAccessTier; + + /* + * The x-ms-access-tier-inferred property. + */ + @Generated + private Boolean xMsAccessTierInferred; + + /* + * The x-ms-access-tier-change-time property. + */ + @Generated + private DateTimeRfc1123 xMsAccessTierChangeTime; + + /* + * The x-ms-smart-access-tier property. + */ + @Generated + private String xMsSmartAccessTier; + /* * The x-ms-content-crc64 property. */ @@ -367,6 +391,16 @@ public final class BlobsDownloadHeaders { private static final HttpHeaderName X_MS_STRUCTURED_CONTENT_LENGTH = HttpHeaderName.fromString("x-ms-structured-content-length"); + private static final HttpHeaderName X_MS_ACCESS_TIER = HttpHeaderName.fromString("x-ms-access-tier"); + + private static final HttpHeaderName X_MS_ACCESS_TIER_INFERRED + = HttpHeaderName.fromString("x-ms-access-tier-inferred"); + + private static final HttpHeaderName X_MS_ACCESS_TIER_CHANGE_TIME + = HttpHeaderName.fromString("x-ms-access-tier-change-time"); + + private static final HttpHeaderName X_MS_SMART_ACCESS_TIER = HttpHeaderName.fromString("x-ms-smart-access-tier"); + private static final HttpHeaderName X_MS_CONTENT_CRC64 = HttpHeaderName.fromString("x-ms-content-crc64"); // HttpHeaders containing the raw property values. @@ -529,6 +563,20 @@ public BlobsDownloadHeaders(HttpHeaders rawHeaders) { } else { this.xMsStructuredContentLength = null; } + this.xMsAccessTier = rawHeaders.getValue(X_MS_ACCESS_TIER); + String xMsAccessTierInferred = rawHeaders.getValue(X_MS_ACCESS_TIER_INFERRED); + if (xMsAccessTierInferred != null) { + this.xMsAccessTierInferred = Boolean.parseBoolean(xMsAccessTierInferred); + } else { + this.xMsAccessTierInferred = null; + } + String xMsAccessTierChangeTime = rawHeaders.getValue(X_MS_ACCESS_TIER_CHANGE_TIME); + if (xMsAccessTierChangeTime != null) { + this.xMsAccessTierChangeTime = new DateTimeRfc1123(xMsAccessTierChangeTime); + } else { + this.xMsAccessTierChangeTime = null; + } + this.xMsSmartAccessTier = rawHeaders.getValue(X_MS_SMART_ACCESS_TIER); String xMsContentCrc64 = rawHeaders.getValue(X_MS_CONTENT_CRC64); if (xMsContentCrc64 != null) { this.xMsContentCrc64 = Base64.getDecoder().decode(xMsContentCrc64); @@ -1584,6 +1632,101 @@ public BlobsDownloadHeaders setXMsStructuredContentLength(Long xMsStructuredCont return this; } + /** + * Get the xMsAccessTier property: The x-ms-access-tier property. + * + * @return the xMsAccessTier value. + */ + @Generated + public String getXMsAccessTier() { + return this.xMsAccessTier; + } + + /** + * Set the xMsAccessTier property: The x-ms-access-tier property. + * + * @param xMsAccessTier the xMsAccessTier value to set. + * @return the BlobsDownloadHeaders object itself. + */ + @Generated + public BlobsDownloadHeaders setXMsAccessTier(String xMsAccessTier) { + this.xMsAccessTier = xMsAccessTier; + return this; + } + + /** + * Get the xMsAccessTierInferred property: The x-ms-access-tier-inferred property. + * + * @return the xMsAccessTierInferred value. + */ + @Generated + public Boolean isXMsAccessTierInferred() { + return this.xMsAccessTierInferred; + } + + /** + * Set the xMsAccessTierInferred property: The x-ms-access-tier-inferred property. + * + * @param xMsAccessTierInferred the xMsAccessTierInferred value to set. + * @return the BlobsDownloadHeaders object itself. + */ + @Generated + public BlobsDownloadHeaders setXMsAccessTierInferred(Boolean xMsAccessTierInferred) { + this.xMsAccessTierInferred = xMsAccessTierInferred; + return this; + } + + /** + * Get the xMsAccessTierChangeTime property: The x-ms-access-tier-change-time property. + * + * @return the xMsAccessTierChangeTime value. + */ + @Generated + public OffsetDateTime getXMsAccessTierChangeTime() { + if (this.xMsAccessTierChangeTime == null) { + return null; + } + return this.xMsAccessTierChangeTime.getDateTime(); + } + + /** + * Set the xMsAccessTierChangeTime property: The x-ms-access-tier-change-time property. + * + * @param xMsAccessTierChangeTime the xMsAccessTierChangeTime value to set. + * @return the BlobsDownloadHeaders object itself. + */ + @Generated + public BlobsDownloadHeaders setXMsAccessTierChangeTime(OffsetDateTime xMsAccessTierChangeTime) { + if (xMsAccessTierChangeTime == null) { + this.xMsAccessTierChangeTime = null; + } else { + this.xMsAccessTierChangeTime = new DateTimeRfc1123(xMsAccessTierChangeTime); + } + return this; + } + + /** + * Get the xMsSmartAccessTier property: The x-ms-smart-access-tier property. + * + * @return the xMsSmartAccessTier value. + */ + @Generated + public String getXMsSmartAccessTier() { + return this.xMsSmartAccessTier; + } + + /** + * Set the xMsSmartAccessTier property: The x-ms-smart-access-tier property. + * + * @param xMsSmartAccessTier the xMsSmartAccessTier value to set. + * @return the BlobsDownloadHeaders object itself. + */ + @Generated + public BlobsDownloadHeaders setXMsSmartAccessTier(String xMsSmartAccessTier) { + this.xMsSmartAccessTier = xMsSmartAccessTier; + return this; + } + /** * Get the xMsContentCrc64 property: The x-ms-content-crc64 property. * diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java index 0866d310981c..53fcb67447dc 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BuilderHelper.java @@ -39,6 +39,8 @@ import com.azure.storage.common.policy.ResponseValidationPolicyBuilder; import com.azure.storage.common.policy.ScrubEtagPolicy; import com.azure.storage.common.policy.StorageBearerTokenChallengeAuthorizationPolicy; +import com.azure.storage.common.policy.StorageContentValidationDecoderPolicy; +import com.azure.storage.common.policy.StorageContentValidationPolicy; import com.azure.storage.common.policy.StorageSharedKeyCredentialPolicy; import java.net.MalformedURLException; @@ -115,6 +117,9 @@ public static HttpPipeline buildPipeline(StorageSharedKeyCredential storageShare } policies.add(new MetadataValidationPolicy()); + policies.add(new StorageContentValidationPolicy()); + policies.add(new StorageContentValidationDecoderPolicy()); + if (storageSharedKeyCredential != null) { policies.add(new StorageSharedKeyCredentialPolicy(storageSharedKeyCredential)); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadHeaders.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadHeaders.java index dfd93a535fc6..6a372f5b2c74 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadHeaders.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobDownloadHeaders.java @@ -825,6 +825,97 @@ public BlobDownloadHeaders setEncryptionScope(String encryptionScope) { return this; } + /** + * Gets the access tier of the blob. + * + * @return the access tier of the blob. This is only set for Page blobs on a premium storage account or for Block + * blobs on blob storage or general purpose V2 account. + */ + public AccessTier getAccessTier() { + String accessTier = internalHeaders.getXMsAccessTier(); + return accessTier == null ? null : AccessTier.fromString(accessTier); + } + + /** + * Sets the access tier of the blob. + * + * @param accessTier the access tier of the blob. + * @return the BlobDownloadHeaders object itself. + */ + public BlobDownloadHeaders setAccessTier(AccessTier accessTier) { + internalHeaders.setXMsAccessTier(accessTier == null ? null : accessTier.toString()); + return this; + } + + /** + * Gets the status of the tier being inferred for the blob. + * + * @return the status of the tier being inferred for the blob. This is only set for Page blobs on a premium storage + * account or for Block blobs on blob storage or general purpose V2 account. + */ + public Boolean isAccessTierInferred() { + return Boolean.TRUE.equals(internalHeaders.isXMsAccessTierInferred()); + } + + /** + * Sets the status of the tier being inferred for the blob. + * + * @param accessTierInferred the status of the tier being inferred for the blob. + * @return the BlobDownloadHeaders object itself. + */ + public BlobDownloadHeaders setAccessTierInferred(Boolean accessTierInferred) { + internalHeaders.setXMsAccessTierInferred(accessTierInferred); + return this; + } + + /** + * Gets the time when the access tier for the blob was last changed. + * + * @return the time when the access tier for the blob was last changed. + */ + public OffsetDateTime getAccessTierChangeTime() { + return internalHeaders.getXMsAccessTierChangeTime(); + } + + /** + * Sets the time when the access tier for the blob was last changed. + * + * @param accessTierChangeTime the time when the access tier for the blob was last changed. + * @return the BlobDownloadHeaders object itself. + */ + public BlobDownloadHeaders setAccessTierChangeTime(OffsetDateTime accessTierChangeTime) { + internalHeaders.setXMsAccessTierChangeTime(accessTierChangeTime); + return this; + } + + /** + * Gets the underlying access tier of the blob when its access tier is {@link AccessTier#SMART}. + *

+ * This value is only populated when {@link #getAccessTier()} returns {@link AccessTier#SMART}. In that case, it + * represents the concrete access tier (for example {@link AccessTier#HOT} or {@link AccessTier#COOL}) that the + * service has selected for the blob. For all other access tiers, this property is {@code null} and should be + * ignored. + * + * @return the underlying access tier chosen by the service when the blob's access tier is {@link AccessTier#SMART}, + * or {@code null} if the blob is not using the smart access tier. + */ + public AccessTier getSmartAccessTier() { + String smartAccessTier = internalHeaders.getXMsSmartAccessTier(); + return smartAccessTier == null ? null : AccessTier.fromString(smartAccessTier); + } + + /** + * Sets the underlying access tier of the blob when its access tier is {@link AccessTier#SMART}. + * + * @param smartAccessTier the underlying access tier chosen by the service when the blob's access tier is + * {@link AccessTier#SMART}. + * @return the BlobDownloadHeaders object itself. + */ + public BlobDownloadHeaders setSmartAccessTier(AccessTier smartAccessTier) { + internalHeaders.setXMsSmartAccessTier(smartAccessTier == null ? null : smartAccessTier.toString()); + return this; + } + /** * Get the blobContentMD5 property: If the blob has a MD5 hash, and if request contains range header (Range or * x-ms-range), this response header is returned with the value of the whole blob's MD5 value. This value may or may diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobAppendBlockOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobAppendBlockOptions.java new file mode 100644 index 000000000000..7c349b2a088c --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobAppendBlockOptions.java @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.models.AppendBlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.StorageImplUtils; + +import reactor.core.publisher.Flux; + +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * Extended options that may be passed when appending a block to an append blob. + */ +@Fluent +public final class AppendBlobAppendBlockOptions { + private final InputStream dataStream; + private final Flux dataFlux; + private final long length; + private byte[] contentMd5; + private AppendBlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link AppendBlobAppendBlockOptions} for use with the sync client. + * + * @param data The data to write to the blob. Must be markable for retries. + * @param length The exact length of the data. + * @throws NullPointerException If {@code data} is null. + * @throws IllegalArgumentException If {@code length} is negative. + */ + public AppendBlobAppendBlockOptions(InputStream data, long length) { + StorageImplUtils.assertNotNull("data", data); + if (length < 0) { + throw new IllegalArgumentException("'length' must be >= 0"); + } + this.dataStream = data; + this.dataFlux = null; + this.length = length; + } + + /** + * Creates a new instance of {@link AppendBlobAppendBlockOptions} for use with the async client. + * + * @param data The data to write to the blob. Must be replayable if retries are enabled. + * @param length The exact length of the data. + * @throws NullPointerException If {@code data} is null. + * @throws IllegalArgumentException If {@code length} is negative. + */ + public AppendBlobAppendBlockOptions(Flux data, long length) { + StorageImplUtils.assertNotNull("data", data); + if (length < 0) { + throw new IllegalArgumentException("'length' must be >= 0"); + } + this.dataStream = null; + this.dataFlux = data; + this.length = length; + } + + /** + * Gets the body as an InputStream. Null if constructed with {@link Flux}. + * + * @return The body stream, or null. + */ + public InputStream getDataStream() { + return dataStream; + } + + /** + * Gets the body as a Flux. Null if constructed with {@link InputStream}. + * + * @return The body flux, or null. + */ + public Flux getDataFlux() { + return dataFlux; + } + + /** + * Gets the exact length of the block data. + * + * @return The length in bytes. + */ + public long getLength() { + return length; + } + + /** + * Gets the MD5 hash of the block content. + * + * @return An MD5 hash of the content, or null. + */ + public byte[] getContentMd5() { + return CoreUtils.clone(contentMd5); + } + + /** + * Sets the MD5 hash of the block content for transactional verification. + * + * @param contentMd5 An MD5 hash of the block content. + * @return The updated options. + */ + public AppendBlobAppendBlockOptions setContentMd5(byte[] contentMd5) { + this.contentMd5 = CoreUtils.clone(contentMd5); + return this; + } + + /** + * Gets the {@link AppendBlobRequestConditions}. + * + * @return The request conditions. + */ + public AppendBlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link AppendBlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public AppendBlobAppendBlockOptions setRequestConditions(AppendBlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public AppendBlobAppendBlockOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobOutputStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobOutputStreamOptions.java new file mode 100644 index 000000000000..df12b00c32d1 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/AppendBlobOutputStreamOptions.java @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.blob.models.AppendBlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; + +/** + * Extended options that may be passed when opening an output stream to an append blob. + */ +@Fluent +public final class AppendBlobOutputStreamOptions { + private AppendBlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link AppendBlobOutputStreamOptions}. + */ + public AppendBlobOutputStreamOptions() { + } + + /** + * Gets the {@link AppendBlobRequestConditions}. + * + * @return The request conditions. + */ + public AppendBlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link AppendBlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public AppendBlobOutputStreamOptions setRequestConditions(AppendBlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public AppendBlobOutputStreamOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadContentOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadContentOptions.java new file mode 100644 index 000000000000..df63b74238fc --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadContentOptions.java @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.common.ContentValidationAlgorithm; + +/** + * Extended options that may be passed when downloading blob content (full blob or range in memory). + */ +@Fluent +public final class BlobDownloadContentOptions { + private BlobRange range; + private DownloadRetryOptions downloadRetryOptions; + private BlobRequestConditions requestConditions; + private boolean retrieveContentRangeMd5; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link BlobDownloadContentOptions}. + */ + public BlobDownloadContentOptions() { + } + + /** + * Gets the {@link BlobRange}. + * + * @return The blob range. + */ + public BlobRange getRange() { + return range; + } + + /** + * Sets the {@link BlobRange}. + * + * @param range The blob range. + * @return The updated options. + */ + public BlobDownloadContentOptions setRange(BlobRange range) { + this.range = range; + return this; + } + + /** + * Gets the {@link DownloadRetryOptions}. + * + * @return The download retry options. + */ + public DownloadRetryOptions getDownloadRetryOptions() { + return downloadRetryOptions; + } + + /** + * Sets the {@link DownloadRetryOptions}. + * + * @param downloadRetryOptions The download retry options. + * @return The updated options. + */ + public BlobDownloadContentOptions setDownloadRetryOptions(DownloadRetryOptions downloadRetryOptions) { + this.downloadRetryOptions = downloadRetryOptions; + return this; + } + + /** + * Gets the {@link BlobRequestConditions}. + * + * @return The request conditions. + */ + public BlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link BlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public BlobDownloadContentOptions setRequestConditions(BlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets whether the content MD5 for the specified blob range should be returned. + * + * @return Whether to retrieve content range MD5. + */ + public boolean isRetrieveContentRangeMd5() { + return retrieveContentRangeMd5; + } + + /** + * Sets whether the content MD5 for the specified blob range should be returned. + * + * @param retrieveContentRangeMd5 Whether to retrieve content range MD5. + * @return The updated options. + */ + public BlobDownloadContentOptions setRetrieveContentRangeMd5(boolean retrieveContentRangeMd5) { + this.retrieveContentRangeMd5 = retrieveContentRangeMd5; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation. See {@link ContentValidationAlgorithm} for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobDownloadContentOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadStreamOptions.java new file mode 100644 index 000000000000..c8bf93b7ae03 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadStreamOptions.java @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.common.ContentValidationAlgorithm; + +/** + * Extended options that may be passed when downloading a blob range to an output stream. + */ +@Fluent +public final class BlobDownloadStreamOptions { + private BlobRange range; + private DownloadRetryOptions downloadRetryOptions; + private BlobRequestConditions requestConditions; + private boolean retrieveContentRangeMd5; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link BlobDownloadStreamOptions}. + */ + public BlobDownloadStreamOptions() { + } + + /** + * Gets the {@link BlobRange}. + * + * @return The blob range. + */ + public BlobRange getRange() { + return range; + } + + /** + * Sets the {@link BlobRange}. + * + * @param range The blob range. + * @return The updated options. + */ + public BlobDownloadStreamOptions setRange(BlobRange range) { + this.range = range; + return this; + } + + /** + * Gets the {@link DownloadRetryOptions}. + * + * @return The download retry options. + */ + public DownloadRetryOptions getDownloadRetryOptions() { + return downloadRetryOptions; + } + + /** + * Sets the {@link DownloadRetryOptions}. + * + * @param downloadRetryOptions The download retry options. + * @return The updated options. + */ + public BlobDownloadStreamOptions setDownloadRetryOptions(DownloadRetryOptions downloadRetryOptions) { + this.downloadRetryOptions = downloadRetryOptions; + return this; + } + + /** + * Gets the {@link BlobRequestConditions}. + * + * @return The request conditions. + */ + public BlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link BlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public BlobDownloadStreamOptions setRequestConditions(BlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets whether the content MD5 for the specified blob range should be returned. + * + * @return Whether to retrieve content range MD5. + */ + public boolean isRetrieveContentRangeMd5() { + return retrieveContentRangeMd5; + } + + /** + * Sets whether the content MD5 for the specified blob range should be returned. + * + * @param retrieveContentRangeMd5 Whether to retrieve content range MD5. + * @return The updated options. + */ + public BlobDownloadStreamOptions setRetrieveContentRangeMd5(boolean retrieveContentRangeMd5) { + this.retrieveContentRangeMd5 = retrieveContentRangeMd5; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobDownloadStreamOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadToFileOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadToFileOptions.java index 434844645f58..0214ca4b8613 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadToFileOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobDownloadToFileOptions.java @@ -8,6 +8,7 @@ import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.DownloadRetryOptions; import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; import java.nio.file.OpenOption; @@ -25,6 +26,7 @@ public class BlobDownloadToFileOptions { private BlobRequestConditions requestConditions; private boolean retrieveContentRangeMd5; private Set openOptions; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Constructs a {@link BlobDownloadToFileOptions}. @@ -165,4 +167,27 @@ public BlobDownloadToFileOptions setOpenOptions(Set openOptions) { this.openOptions = openOptions; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobDownloadToFileOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobInputStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobInputStreamOptions.java index d98d744f3dd3..bb2e2f240215 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobInputStreamOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobInputStreamOptions.java @@ -7,6 +7,7 @@ import com.azure.storage.blob.models.BlobRange; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ConsistentReadControl; +import com.azure.storage.common.ContentValidationAlgorithm; /** * Extended options that may be passed when opening a blob input stream. @@ -17,6 +18,7 @@ public class BlobInputStreamOptions { private BlobRequestConditions requestConditions; private Integer blockSize; private ConsistentReadControl consistentReadControl; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlobInputStreamOptions}. @@ -111,4 +113,26 @@ public BlobInputStreamOptions setConsistentReadControl(ConsistentReadControl con this.consistentReadControl = consistentReadControl; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobInputStreamOptions setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobParallelUploadOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobParallelUploadOptions.java index 681ff994ae62..e4aac1781f61 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobParallelUploadOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobParallelUploadOptions.java @@ -12,7 +12,9 @@ import com.azure.storage.blob.models.BlobImmutabilityPolicy; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; + import reactor.core.publisher.Flux; import java.io.InputStream; @@ -38,6 +40,7 @@ public class BlobParallelUploadOptions { private Duration timeout; private BlobImmutabilityPolicy immutabilityPolicy; private Boolean legalHold; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Constructs a new {@link BlobParallelUploadOptions}. @@ -366,4 +369,27 @@ public BlobParallelUploadOptions setLegalHold(Boolean legalHold) { this.legalHold = legalHold; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobParallelUploadOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobSeekableByteChannelReadOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobSeekableByteChannelReadOptions.java index ac92a1a022b3..91f61b9b509e 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobSeekableByteChannelReadOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobSeekableByteChannelReadOptions.java @@ -6,6 +6,7 @@ import com.azure.core.annotation.Fluent; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ConsistentReadControl; +import com.azure.storage.common.ContentValidationAlgorithm; import java.nio.channels.SeekableByteChannel; @@ -18,6 +19,7 @@ public final class BlobSeekableByteChannelReadOptions { private BlobRequestConditions requestConditions; private Integer readSizeInBytes; private ConsistentReadControl consistentReadControl; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlobSeekableByteChannelReadOptions}. @@ -108,4 +110,27 @@ public BlobSeekableByteChannelReadOptions setConsistentReadControl(ConsistentRea this.consistentReadControl = consistentReadControl; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the response. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobSeekableByteChannelReadOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobUploadFromFileOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobUploadFromFileOptions.java index 65ca9b0ab052..a9e283466af6 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobUploadFromFileOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlobUploadFromFileOptions.java @@ -8,6 +8,7 @@ import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; import java.util.Map; @@ -24,6 +25,7 @@ public class BlobUploadFromFileOptions { private Map tags; private AccessTier tier; private BlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Constructs a {@link BlobUploadFromFileOptions}. @@ -164,4 +166,27 @@ public BlobUploadFromFileOptions setRequestConditions(BlobRequestConditions requ this.requestConditions = requestConditions; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlobUploadFromFileOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobOutputStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobOutputStreamOptions.java index e6385dfe5207..d02b6fd11681 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobOutputStreamOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobOutputStreamOptions.java @@ -7,6 +7,7 @@ import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import java.util.Map; @@ -20,6 +21,7 @@ public class BlockBlobOutputStreamOptions { private Map tags; private AccessTier tier; private BlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlockBlobOutputStreamOptions}. @@ -146,4 +148,27 @@ public BlockBlobOutputStreamOptions setRequestConditions(BlobRequestConditions r this.requestConditions = requestConditions; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlockBlobOutputStreamOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSeekableByteChannelWriteOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSeekableByteChannelWriteOptions.java index 1234ea04c738..3ab7ff719ead 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSeekableByteChannelWriteOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSeekableByteChannelWriteOptions.java @@ -6,6 +6,7 @@ import com.azure.storage.blob.models.AccessTier; import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; import java.util.Collection; import java.util.Map; @@ -60,6 +61,7 @@ public static Collection values() { private Map tags; private AccessTier tier; private BlobRequestConditions conditions; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Options constructor. @@ -199,4 +201,26 @@ public BlockBlobSeekableByteChannelWriteOptions setRequestConditions(BlobRequest return this; } + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated instance. + */ + public BlockBlobSeekableByteChannelWriteOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSimpleUploadOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSimpleUploadOptions.java index bef1780c2f59..47212d6033d8 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSimpleUploadOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobSimpleUploadOptions.java @@ -9,7 +9,9 @@ import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobImmutabilityPolicy; import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; + import reactor.core.publisher.Flux; import java.io.InputStream; @@ -32,6 +34,7 @@ public class BlockBlobSimpleUploadOptions { private BlobRequestConditions requestConditions; private BlobImmutabilityPolicy immutabilityPolicy; private Boolean legalHold; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlockBlobSimpleUploadOptions}. @@ -293,4 +296,27 @@ public BlockBlobSimpleUploadOptions setLegalHold(Boolean legalHold) { this.legalHold = legalHold; return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlockBlobSimpleUploadOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobStageBlockOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobStageBlockOptions.java index 37445b450a28..9c9868d57ed4 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobStageBlockOptions.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/BlockBlobStageBlockOptions.java @@ -6,6 +6,7 @@ import com.azure.core.annotation.Fluent; import com.azure.core.util.BinaryData; import com.azure.core.util.CoreUtils; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageImplUtils; /** @@ -17,6 +18,7 @@ public final class BlockBlobStageBlockOptions { private final BinaryData data; private String leaseId; private byte[] contentMd5; + private ContentValidationAlgorithm contentValidationAlgorithm; /** * Creates a new instance of {@link BlockBlobStageBlockOptions}. @@ -97,4 +99,27 @@ public BlockBlobStageBlockOptions setContentMd5(byte[] contentMd5) { this.contentMd5 = CoreUtils.clone(contentMd5); return this; } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public BlockBlobStageBlockOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobOutputStreamOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobOutputStreamOptions.java new file mode 100644 index 000000000000..5a9ce4b49af4 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobOutputStreamOptions.java @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.StorageImplUtils; + +/** + * Extended options that may be passed when opening an output stream to a page blob. + */ +@Fluent +public final class PageBlobOutputStreamOptions { + private final PageRange pageRange; + private BlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link PageBlobOutputStreamOptions}. + * + * @param pageRange The {@link PageRange} for the write. Pages must be aligned with 512-byte boundaries. + * @throws NullPointerException If {@code pageRange} is null. + */ + public PageBlobOutputStreamOptions(PageRange pageRange) { + StorageImplUtils.assertNotNull("pageRange", pageRange); + this.pageRange = pageRange; + } + + /** + * Gets the page range. + * + * @return The page range. + */ + public PageRange getPageRange() { + return pageRange; + } + + /** + * Gets the {@link BlobRequestConditions}. + * + * @return The request conditions. + */ + public BlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link BlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public PageBlobOutputStreamOptions setRequestConditions(BlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public PageBlobOutputStreamOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobUploadPagesOptions.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobUploadPagesOptions.java new file mode 100644 index 000000000000..335bef5f7a1b --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/options/PageBlobUploadPagesOptions.java @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.options; + +import com.azure.core.annotation.Fluent; +import com.azure.core.util.CoreUtils; +import com.azure.storage.blob.models.PageBlobRequestConditions; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.StorageImplUtils; + +import reactor.core.publisher.Flux; + +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * Extended options that may be passed when uploading pages to a page blob. + */ +@Fluent +public final class PageBlobUploadPagesOptions { + private final PageRange pageRange; + private final Flux dataFlux; + private final InputStream dataStream; + private byte[] contentMd5; + private PageBlobRequestConditions requestConditions; + private ContentValidationAlgorithm contentValidationAlgorithm; + + /** + * Creates a new instance of {@link PageBlobUploadPagesOptions}. + * + * @param pageRange The {@link PageRange} for the upload. Pages must be aligned with 512-byte boundaries. + * @param body The data to upload. Must be replayable if retries are enabled. + * @throws NullPointerException If {@code pageRange} or {@code body} is null. + */ + public PageBlobUploadPagesOptions(PageRange pageRange, Flux body) { + StorageImplUtils.assertNotNull("pageRange", pageRange); + StorageImplUtils.assertNotNull("body", body); + this.pageRange = pageRange; + this.dataFlux = body; + this.dataStream = null; + } + + /** + * Creates a new instance of {@link PageBlobUploadPagesOptions}. + * + * @param pageRange The {@link PageRange} for the upload. Pages must be aligned with 512-byte boundaries. + * @param body The data to upload. Must be markable for retries. + * @throws NullPointerException If {@code pageRange} or {@code body} is null. + */ + public PageBlobUploadPagesOptions(PageRange pageRange, InputStream body) { + StorageImplUtils.assertNotNull("pageRange", pageRange); + StorageImplUtils.assertNotNull("body", body); + this.pageRange = pageRange; + this.dataFlux = null; + this.dataStream = body; + } + + /** + * Gets the page range. + * + * @return The page range. + */ + public PageRange getPageRange() { + return pageRange; + } + + /** + * Gets the body as a Flux. Null if constructed with InputStream. + * + * @return The body flux, or null. + */ + public Flux getDataFlux() { + return dataFlux; + } + + /** + * Gets the body as an InputStream. Null if constructed with Flux. + * + * @return The body stream, or null. + */ + public InputStream getDataStream() { + return dataStream; + } + + /** + * Gets the MD5 hash of the page content. + * + * @return An MD5 hash of the content, or null. + */ + public byte[] getContentMd5() { + return CoreUtils.clone(contentMd5); + } + + /** + * Sets the MD5 hash of the page content for transactional verification. + * + * @param contentMd5 An MD5 hash of the page content. + * @return The updated options. + */ + public PageBlobUploadPagesOptions setContentMd5(byte[] contentMd5) { + this.contentMd5 = CoreUtils.clone(contentMd5); + return this; + } + + /** + * Gets the {@link PageBlobRequestConditions}. + * + * @return The request conditions. + */ + public PageBlobRequestConditions getRequestConditions() { + return requestConditions; + } + + /** + * Sets the {@link PageBlobRequestConditions}. + * + * @param requestConditions The request conditions. + * @return The updated options. + */ + public PageBlobUploadPagesOptions setRequestConditions(PageBlobRequestConditions requestConditions) { + this.requestConditions = requestConditions; + return this; + } + + /** + * Gets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @return The transfer validation checksum algorithm. + */ + public ContentValidationAlgorithm getContentValidationAlgorithm() { + return contentValidationAlgorithm; + } + + /** + * Sets the algorithm to use for transfer content validation on the request. See {@link ContentValidationAlgorithm} + * for more details. + * + * @param contentValidationAlgorithm The transfer validation checksum algorithm. + * @return The updated options. + */ + public PageBlobUploadPagesOptions + setContentValidationAlgorithm(ContentValidationAlgorithm contentValidationAlgorithm) { + this.contentValidationAlgorithm = contentValidationAlgorithm; + return this; + } +} diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobAsyncClient.java index 42b7b3f76f1c..3a47a4be200a 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobAsyncClient.java @@ -34,10 +34,14 @@ import com.azure.storage.blob.models.CpkInfo; import com.azure.storage.blob.models.CustomerProvidedKey; import com.azure.storage.blob.models.EncryptionAlgorithmType; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; import com.azure.storage.blob.options.AppendBlobAppendBlockFromUrlOptions; import com.azure.storage.blob.options.AppendBlobCreateOptions; import com.azure.storage.blob.options.AppendBlobSealOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -451,9 +455,34 @@ public Mono appendBlock(Flux data, long length) { @ServiceMethod(returns = ReturnType.SINGLE) public Mono> appendBlockWithResponse(Flux data, long length, byte[] contentMd5, AppendBlobRequestConditions appendBlobRequestConditions) { + if (data == null) { + return monoError(LOGGER, new NullPointerException("'data' cannot be null.")); + } + return appendBlockWithResponse(new AppendBlobAppendBlockOptions(data, length).setContentMd5(contentMd5) + .setRequestConditions(appendBlobRequestConditions)); + } + + /** + * Commits a new block of data to the end of the existing append blob with options. + * + * @param options {@link AppendBlobAppendBlockOptions} containing the block data. + * @return A {@link Mono} containing {@link Response} whose value contains the append blob operation. + * @throws NullPointerException If {@code options} is null. + * @throws IllegalArgumentException If options were not constructed with Flux (async client). + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono> appendBlockWithResponse(AppendBlobAppendBlockOptions options) { try { - return withContext( - context -> appendBlockWithResponse(data, length, contentMd5, appendBlobRequestConditions, context)); + if (options == null) { + return monoError(LOGGER, new NullPointerException("'options' cannot be null.")); + } + if (options.getDataFlux() == null) { + return monoError(LOGGER, new IllegalArgumentException( + "AppendBlobAppendBlockOptions must be constructed with Flux for async client.")); + } + return withContext(context -> appendBlockWithResponseInternal(options.getDataFlux(), options.getLength(), + options.getContentMd5(), options.getRequestConditions(), options.getContentValidationAlgorithm(), + context)); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } @@ -461,14 +490,21 @@ public Mono> appendBlockWithResponse(Flux d Mono> appendBlockWithResponse(Flux data, long length, byte[] contentMd5, AppendBlobRequestConditions appendBlobRequestConditions, Context context) { + // Prevents revapi visibility increased error + return appendBlockWithResponseInternal(data, length, contentMd5, appendBlobRequestConditions, null, context); + } + Mono> appendBlockWithResponseInternal(Flux data, long length, + byte[] contentMd5, AppendBlobRequestConditions appendBlobRequestConditions, + ContentValidationAlgorithm contentValidationAlgorithm, Context context) { if (data == null) { - return Mono.error(new NullPointerException("'data' cannot be null.")); + return monoError(LOGGER, new NullPointerException("'data' cannot be null.")); } appendBlobRequestConditions = appendBlobRequestConditions == null ? new AppendBlobRequestConditions() : appendBlobRequestConditions; - context = context == null ? Context.NONE : context; + context = ContentValidationModeResolver.addContentValidationMode(context, contentValidationAlgorithm, length, + false); return this.azureBlobStorage.getAppendBlobs() .appendBlockWithResponseAsync(containerName, blobName, length, data, null, contentMd5, null, diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobClient.java index 0fd58f5ef575..320f54caa3a7 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/AppendBlobClient.java @@ -6,7 +6,6 @@ import com.azure.core.annotation.ReturnType; import com.azure.core.annotation.ServiceClient; import com.azure.core.annotation.ServiceMethod; -import com.azure.core.exception.UnexpectedLengthException; import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpResponse; import com.azure.core.http.rest.Response; @@ -32,8 +31,10 @@ import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.CpkInfo; import com.azure.storage.blob.models.CustomerProvidedKey; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; import com.azure.storage.blob.options.AppendBlobAppendBlockFromUrlOptions; import com.azure.storage.blob.options.AppendBlobCreateOptions; +import com.azure.storage.blob.options.AppendBlobOutputStreamOptions; import com.azure.storage.blob.options.AppendBlobSealOptions; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; @@ -45,7 +46,6 @@ import java.nio.ByteBuffer; import java.time.Duration; import java.util.Map; -import java.util.Objects; import java.util.concurrent.Callable; import static com.azure.storage.common.implementation.StorageImplUtils.sendRequest; @@ -181,8 +181,9 @@ public AppendBlobClient getCustomerProvidedKeyClient(CustomerProvidedKey custome * @return A {@link BlobOutputStream} object used to write data to the blob. * @throws BlobStorageException If a storage service error occurred. */ + @ServiceMethod(returns = ReturnType.SINGLE) public BlobOutputStream getBlobOutputStream() { - return getBlobOutputStream(null); + return getBlobOutputStream((AppendBlobRequestConditions) null); } /** @@ -194,6 +195,7 @@ public BlobOutputStream getBlobOutputStream() { * @param overwrite Whether an existing blob should be deleted and recreated, should data exist on the blob. * @throws BlobStorageException If a storage service error occurred. */ + @ServiceMethod(returns = ReturnType.SINGLE) public BlobOutputStream getBlobOutputStream(boolean overwrite) { AppendBlobRequestConditions requestConditions = null; if (!overwrite) { @@ -214,10 +216,24 @@ public BlobOutputStream getBlobOutputStream(boolean overwrite) { * @return A {@link BlobOutputStream} object used to write data to the blob. * @throws BlobStorageException If a storage service error occurred. */ + @ServiceMethod(returns = ReturnType.SINGLE) public BlobOutputStream getBlobOutputStream(AppendBlobRequestConditions requestConditions) { return BlobOutputStream.appendBlobOutputStream(appendBlobAsyncClient, requestConditions); } + /** + * Creates and opens an output stream to write data to the append blob. + * + * @param options {@link AppendBlobOutputStreamOptions} + * @return A {@link BlobOutputStream} object used to write data to the blob. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public BlobOutputStream getBlobOutputStream(AppendBlobOutputStreamOptions options) { + options = options == null ? new AppendBlobOutputStreamOptions() : options; + return BlobOutputStream.appendBlobOutputStream(appendBlobAsyncClient, options.getRequestConditions(), + options.getContentValidationAlgorithm()); + } + /** * Creates a 0-length append blob. Call appendBlock to append data to an append blob. By default this method will * not overwrite an existing blob. @@ -499,21 +515,43 @@ public AppendBlobItem appendBlock(InputStream data, long length) { * @param timeout An optional timeout value beyond which a {@link RuntimeException} will be raised. * @param context Additional context that is passed through the Http pipeline during the service call. * @return A {@link Response} whose {@link Response#getValue() value} contains the append blob operation. - * @throws UnexpectedLengthException when the length of data does not match the input {@code length}. * @throws NullPointerException if the input data is null. */ @ServiceMethod(returns = ReturnType.SINGLE) public Response appendBlockWithResponse(InputStream data, long length, byte[] contentMd5, AppendBlobRequestConditions appendBlobRequestConditions, Duration timeout, Context context) { - Objects.requireNonNull(data, "'data' cannot be null."); + return appendBlockWithResponse(new AppendBlobAppendBlockOptions(data, length).setContentMd5(contentMd5) + .setRequestConditions(appendBlobRequestConditions), timeout, context); + } + + /** + * Commits a new block of data to the end of the existing append blob with options. + * + * @param options {@link AppendBlobAppendBlockOptions} containing the block data. + * @param timeout An optional timeout value. + * @param context Additional context. + * @return The information of the append blob operation. + * @throws NullPointerException If {@code options} is null. + * @throws IllegalArgumentException If {@code options} is not constructed with {@link InputStream}. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Response appendBlockWithResponse(AppendBlobAppendBlockOptions options, Duration timeout, + Context context) { + StorageImplUtils.assertNotNull("options", options); + if (options.getDataStream() == null) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + "AppendBlobAppendBlockOptions must be constructed with InputStream for sync client.")); + } Flux fbb; // service versions 2022-11-02 and above support uploading block bytes up to 100MB, all older service versions // support up to 4MB - fbb = Utility.convertStreamToByteBuffer(data, length, getMaxAppendBlockBytes(), true); + fbb = Utility.convertStreamToByteBuffer(options.getDataStream(), options.getLength(), getMaxAppendBlockBytes(), + true); - Mono> response = appendBlobAsyncClient.appendBlockWithResponse(fbb, length, contentMd5, - appendBlobRequestConditions, context); + Mono> response + = appendBlobAsyncClient.appendBlockWithResponseInternal(fbb, options.getLength(), options.getContentMd5(), + options.getRequestConditions(), options.getContentValidationAlgorithm(), context); return StorageImplUtils.blockWithOptionalTimeout(response, timeout); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java index ef8ec9d2d4d8..c3392bfef4f9 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobAsyncClientBase.java @@ -73,16 +73,21 @@ import com.azure.storage.blob.models.UserDelegationKey; import com.azure.storage.blob.options.BlobBeginCopyOptions; import com.azure.storage.blob.options.BlobCopyFromUrlOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; import com.azure.storage.blob.options.BlobDownloadToFileOptions; import com.azure.storage.blob.options.BlobGetTagsOptions; import com.azure.storage.blob.options.BlobQueryOptions; import com.azure.storage.blob.options.BlobSetAccessTierOptions; import com.azure.storage.blob.options.BlobSetTagsOptions; import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.SasImplUtils; import com.azure.storage.common.implementation.StorageImplUtils; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.publisher.SignalType; @@ -529,7 +534,7 @@ Mono> existsWithResponse(Context context) { return Mono.just(new SimpleResponse<>(response.getRequest(), response.getStatusCode(), response.getHeaders(), false)); } else { - return Mono.error(e); + return monoError(LOGGER, e); } }); } @@ -722,8 +727,8 @@ public PollerFlux beginCopy(BlobBeginCopyOptions options) { } }, (pollingContext, firstResponse) -> { if (firstResponse == null || firstResponse.getValue() == null) { - return Mono.error(LOGGER.logExceptionAsError( - new IllegalArgumentException("Cannot cancel a poll response that never started."))); + return monoError(LOGGER, + new IllegalArgumentException("Cannot cancel a poll response that never started.")); } final String copyIdentifier = firstResponse.getValue().getCopyId(); @@ -1166,9 +1171,25 @@ public Mono downloadWithResponse(BlobRange range, Dow @ServiceMethod(returns = ReturnType.SINGLE) public Mono downloadStreamWithResponse(BlobRange range, DownloadRetryOptions options, BlobRequestConditions requestConditions, boolean getRangeContentMd5) { + return downloadStreamWithResponse(new BlobDownloadStreamOptions().setRange(range) + .setDownloadRetryOptions(options) + .setRequestConditions(requestConditions) + .setRetrieveContentRangeMd5(getRangeContentMd5)); + } + + /** + * Reads a range of bytes from a blob with options. + * + * @param options {@link BlobDownloadStreamOptions} + * @return A reactive response containing the blob data. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono downloadStreamWithResponse(BlobDownloadStreamOptions options) { try { - return withContext( - context -> downloadStreamWithResponse(range, options, requestConditions, getRangeContentMd5, context)); + BlobDownloadStreamOptions finalOptions = options == null ? new BlobDownloadStreamOptions() : options; + return withContext(context -> downloadStreamWithResponseInternal(finalOptions.getRange(), + finalOptions.getDownloadRetryOptions(), finalOptions.getRequestConditions(), + finalOptions.isRetrieveContentRangeMd5(), finalOptions.getContentValidationAlgorithm(), context)); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } @@ -1205,11 +1226,26 @@ public Mono downloadStreamWithResponse(BlobRange rang @ServiceMethod(returns = ReturnType.SINGLE) public Mono downloadContentWithResponse(DownloadRetryOptions options, BlobRequestConditions requestConditions) { + return downloadContentWithResponse( + new BlobDownloadContentOptions().setDownloadRetryOptions(options).setRequestConditions(requestConditions)); + } + + /** + * Reads blob content (full blob or range) with options. + * + * @param options {@link BlobDownloadContentOptions} + * @return A reactive response containing the blob content. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono downloadContentWithResponse(BlobDownloadContentOptions options) { try { - return withContext(context -> downloadStreamWithResponse(null, options, requestConditions, false, context) - .flatMap(r -> BinaryData.fromFlux(r.getValue()) - .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), r.getHeaders(), - data, r.getDeserializedHeaders())))); + BlobDownloadContentOptions finalOptions = options == null ? new BlobDownloadContentOptions() : options; + return withContext(context -> downloadStreamWithResponseInternal(finalOptions.getRange(), + finalOptions.getDownloadRetryOptions(), finalOptions.getRequestConditions(), + finalOptions.isRetrieveContentRangeMd5(), finalOptions.getContentValidationAlgorithm(), context) + .flatMap(r -> BinaryData.fromFlux(r.getValue()) + .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), + r.getHeaders(), data, r.getDeserializedHeaders())))); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } @@ -1217,17 +1253,27 @@ public Mono downloadContentWithResponse(Downlo Mono downloadStreamWithResponse(BlobRange range, DownloadRetryOptions options, BlobRequestConditions requestConditions, boolean getRangeContentMd5, Context context) { + // Prevents revapi visibility increased error + return downloadStreamWithResponseInternal(range, options, requestConditions, getRangeContentMd5, null, context); + } + + Mono downloadStreamWithResponseInternal(BlobRange range, DownloadRetryOptions options, + BlobRequestConditions requestConditions, boolean getRangeContentMd5, + ContentValidationAlgorithm contentValidationAlgorithm, Context context) { BlobRange finalRange = range == null ? new BlobRange(0) : range; Boolean getMD5 = getRangeContentMd5 ? getRangeContentMd5 : null; BlobRequestConditions finalRequestConditions = requestConditions == null ? new BlobRequestConditions() : requestConditions; DownloadRetryOptions finalOptions = (options == null) ? new DownloadRetryOptions() : options; + context + = ContentValidationModeResolver.addStructuredMessageDecodingToContext(context, contentValidationAlgorithm); + // The first range should eagerly convert headers as they'll be used to create response types. Context firstRangeContext = context == null ? new Context("azure-eagerly-convert-headers", true) : context.addData("azure-eagerly-convert-headers", true); - + Context nextRangeContext = context; return downloadRange(finalRange, finalRequestConditions, finalRequestConditions.getIfMatch(), getMD5, firstRangeContext).map(response -> { BlobsDownloadHeaders blobsDownloadHeaders = new BlobsDownloadHeaders(response.getHeaders()); @@ -1272,7 +1318,7 @@ Mono downloadStreamWithResponse(BlobRange range, Down try { return downloadRange(new BlobRange(initialOffset + offset, newCount), finalRequestConditions, - eTag, getMD5, context); + eTag, getMD5, nextRangeContext); } catch (Exception e) { return Mono.error(e); } @@ -1504,7 +1550,8 @@ Mono> downloadToFileWithResponse(BlobDownloadToFileOpti AsynchronousFileChannel channel = downloadToFileResourceSupplier(options.getFilePath(), openOptions); return Mono.just(channel) .flatMap(c -> this.downloadToFileImpl(c, finalRange, finalParallelTransferOptions, - options.getDownloadRetryOptions(), finalConditions, options.isRetrieveContentRangeMd5(), context)) + options.getDownloadRetryOptions(), finalConditions, options.isRetrieveContentRangeMd5(), + options.getContentValidationAlgorithm(), context)) .doFinally(signalType -> this.downloadToFileCleanup(channel, options.getFilePath(), signalType)); } @@ -1519,7 +1566,7 @@ private AsynchronousFileChannel downloadToFileResourceSupplier(String filePath, private Mono> downloadToFileImpl(AsynchronousFileChannel file, BlobRange finalRange, com.azure.storage.common.ParallelTransferOptions finalParallelTransferOptions, DownloadRetryOptions downloadRetryOptions, BlobRequestConditions requestConditions, boolean rangeGetContentMd5, - Context context) { + ContentValidationAlgorithm contentValidationAlgorithm, Context context) { // See ProgressReporter for an explanation on why this lock is necessary and why we use AtomicLong. ProgressListener progressReceiver = finalParallelTransferOptions.getProgressListener(); ProgressReporter progressReporter @@ -1529,8 +1576,8 @@ private Mono> downloadToFileImpl(AsynchronousFileChanne * Downloads the first chunk and gets the size of the data and etag if not specified by the user. */ BiFunction> downloadFunc - = (range, conditions) -> this.downloadStreamWithResponse(range, downloadRetryOptions, conditions, - rangeGetContentMd5, context); + = (range, conditions) -> this.downloadStreamWithResponseInternal(range, downloadRetryOptions, conditions, + rangeGetContentMd5, contentValidationAlgorithm, context); return ChunkedDownloadUtils .downloadFirstChunk(finalRange, finalParallelTransferOptions, requestConditions, downloadFunc, true, diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java index 9c44f4cc8e84..b35c654a1132 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobClientBase.java @@ -76,6 +76,8 @@ import com.azure.storage.blob.models.UserDelegationKey; import com.azure.storage.blob.options.BlobBeginCopyOptions; import com.azure.storage.blob.options.BlobCopyFromUrlOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; import com.azure.storage.blob.options.BlobDownloadToFileOptions; import com.azure.storage.blob.options.BlobGetTagsOptions; import com.azure.storage.blob.options.BlobInputStreamOptions; @@ -84,6 +86,7 @@ import com.azure.storage.blob.options.BlobSetAccessTierOptions; import com.azure.storage.blob.options.BlobSetTagsOptions; import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.StorageSharedKeyCredential; import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; @@ -91,6 +94,7 @@ import com.azure.storage.common.implementation.SasImplUtils; import com.azure.storage.common.implementation.StorageImplUtils; import com.azure.storage.common.implementation.StorageSeekableByteChannel; + import reactor.core.publisher.Mono; import java.io.IOException; @@ -500,6 +504,7 @@ public BlobInputStream openInputStream(BlobInputStreamOptions options) { public BlobInputStream openInputStream(BlobInputStreamOptions options, Context context) { Context contextFinal = context == null ? Context.NONE : context; options = options == null ? new BlobInputStreamOptions() : options; + final ContentValidationAlgorithm contentValidationAlgorithm = options.getContentValidationAlgorithm(); ConsistentReadControl consistentReadControl = options.getConsistentReadControl() == null ? ConsistentReadControl.ETAG : options.getConsistentReadControl(); @@ -511,8 +516,9 @@ public BlobInputStream openInputStream(BlobInputStreamOptions options, Context c com.azure.storage.common.ParallelTransferOptions parallelTransferOptions = new com.azure.storage.common.ParallelTransferOptions().setBlockSizeLong((long) chunkSize); - BiFunction> downloadFunc = (chunkRange, - conditions) -> client.downloadStreamWithResponse(chunkRange, null, conditions, false, contextFinal); + BiFunction> downloadFunc + = (chunkRange, conditions) -> client.downloadStreamWithResponseInternal(chunkRange, null, conditions, false, + contentValidationAlgorithm, contextFinal); return ChunkedDownloadUtils .downloadFirstChunk(range, parallelTransferOptions, requestConditions, downloadFunc, true) .flatMap(tuple3 -> { @@ -588,8 +594,12 @@ public BlobSeekableByteChannelReadResult openSeekableByteChannelRead(BlobSeekabl BlobDownloadResponse response; try (ByteBufferBackedOutputStreamUtil dstStream = new ByteBufferBackedOutputStreamUtil(initialRange)) { response = this.downloadStreamWithResponse(dstStream, - new BlobRange(initialPosition, (long) initialRange.remaining()), null /*downloadRetryOptions*/, - options.getRequestConditions(), false, null, context); + new BlobDownloadStreamOptions() + .setRange(new BlobRange(initialPosition, (long) initialRange.remaining())) + .setRequestConditions(options.getRequestConditions()) + .setRetrieveContentRangeMd5(false) + .setContentValidationAlgorithm(options.getContentValidationAlgorithm()), + null, context); properties = ModelHelper.buildBlobPropertiesResponse(response).getValue(); } catch (IOException e) { throw LOGGER.logExceptionAsError(new UncheckedIOException(e)); @@ -1266,12 +1276,35 @@ public BlobDownloadResponse downloadWithResponse(OutputStream stream, BlobRange @ServiceMethod(returns = ReturnType.SINGLE) public BlobDownloadResponse downloadStreamWithResponse(OutputStream stream, BlobRange range, DownloadRetryOptions options, BlobRequestConditions requestConditions, boolean getRangeContentMd5, + Duration timeout, Context context) { + return downloadStreamWithResponse(stream, + new BlobDownloadStreamOptions().setRange(range) + .setDownloadRetryOptions(options) + .setRequestConditions(requestConditions) + .setRetrieveContentRangeMd5(getRangeContentMd5), + timeout, context); + } + + /** + * Downloads a range of bytes from a blob into an output stream with options. + * + * @param stream The output stream where the downloaded data will be written. + * @param options {@link BlobDownloadStreamOptions} + * @param timeout An optional timeout value. + * @param context Additional context. + * @return A response containing status code and HTTP headers. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public BlobDownloadResponse downloadStreamWithResponse(OutputStream stream, BlobDownloadStreamOptions options, Duration timeout, Context context) { StorageImplUtils.assertNotNull("stream", stream); - Mono download - = client.downloadStreamWithResponse(range, options, requestConditions, getRangeContentMd5, context) - .flatMap(response -> FluxUtil.writeToOutputStream(response.getValue(), stream) - .thenReturn(new BlobDownloadResponse(response))); + options = options == null ? new BlobDownloadStreamOptions() : options; + Mono download = client + .downloadStreamWithResponseInternal(options.getRange(), options.getDownloadRetryOptions(), + options.getRequestConditions(), options.isRetrieveContentRangeMd5(), + options.getContentValidationAlgorithm(), context) + .flatMap(response -> FluxUtil.writeToOutputStream(response.getValue(), stream) + .thenReturn(new BlobDownloadResponse(response))); return blockWithOptionalTimeout(download, timeout); } @@ -1310,14 +1343,9 @@ public BlobDownloadResponse downloadStreamWithResponse(OutputStream stream, Blob @ServiceMethod(returns = ReturnType.SINGLE) public BlobDownloadContentResponse downloadContentWithResponse(DownloadRetryOptions options, BlobRequestConditions requestConditions, Duration timeout, Context context) { - Mono download - = client.downloadStreamWithResponse(null, options, requestConditions, false, context) - .flatMap(r -> BinaryData.fromFlux(r.getValue()) - .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), r.getHeaders(), - data, r.getDeserializedHeaders()))) - .map(BlobDownloadContentResponse::new); - - return blockWithOptionalTimeout(download, timeout); + return downloadContentWithResponse( + new BlobDownloadContentOptions().setDownloadRetryOptions(options).setRequestConditions(requestConditions), + timeout, context); } /** @@ -1358,12 +1386,32 @@ public BlobDownloadContentResponse downloadContentWithResponse(DownloadRetryOpti public BlobDownloadContentResponse downloadContentWithResponse(DownloadRetryOptions options, BlobRequestConditions requestConditions, BlobRange range, boolean getRangeContentMd5, Duration timeout, Context context) { - Mono download - = client.downloadStreamWithResponse(range, options, requestConditions, getRangeContentMd5, context) - .flatMap(r -> BinaryData.fromFlux(r.getValue()) - .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), r.getHeaders(), - data, r.getDeserializedHeaders()))) - .map(BlobDownloadContentResponse::new); + return downloadContentWithResponse(new BlobDownloadContentOptions().setDownloadRetryOptions(options) + .setRequestConditions(requestConditions) + .setRange(range) + .setRetrieveContentRangeMd5(getRangeContentMd5), timeout, context); + } + + /** + * Downloads blob content (full blob or range) with options. + * + * @param options {@link BlobDownloadContentOptions} + * @param timeout An optional timeout value. + * @param context Additional context. + * @return A response containing status code and HTTP headers. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public BlobDownloadContentResponse downloadContentWithResponse(BlobDownloadContentOptions options, Duration timeout, + Context context) { + options = options == null ? new BlobDownloadContentOptions() : options; + Mono download = client + .downloadStreamWithResponseInternal(options.getRange(), options.getDownloadRetryOptions(), + options.getRequestConditions(), options.isRetrieveContentRangeMd5(), + options.getContentValidationAlgorithm(), context) + .flatMap(r -> BinaryData.fromFlux(r.getValue()) + .map(data -> new BlobDownloadContentAsyncResponse(r.getRequest(), r.getStatusCode(), r.getHeaders(), + data, r.getDeserializedHeaders()))) + .map(BlobDownloadContentResponse::new); return blockWithOptionalTimeout(download, timeout); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobOutputStream.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobOutputStream.java index c43277ff30de..0ba385c12ca1 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobOutputStream.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlobOutputStream.java @@ -16,10 +16,13 @@ import com.azure.storage.blob.models.PageBlobRequestConditions; import com.azure.storage.blob.models.PageRange; import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; import com.azure.storage.blob.options.BlobParallelUploadOptions; import com.azure.storage.blob.options.BlockBlobOutputStreamOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.StorageOutputStream; import com.azure.storage.common.implementation.Constants; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -47,7 +50,13 @@ public abstract class BlobOutputStream extends StorageOutputStream { static BlobOutputStream appendBlobOutputStream(final AppendBlobAsyncClient client, final AppendBlobRequestConditions appendBlobRequestConditions) { - return new AppendBlobOutputStream(client, appendBlobRequestConditions); + return new AppendBlobOutputStream(client, appendBlobRequestConditions, null); + } + + static BlobOutputStream appendBlobOutputStream(final AppendBlobAsyncClient client, + final AppendBlobRequestConditions appendBlobRequestConditions, + final ContentValidationAlgorithm contentValidationAlgorithm) { + return new AppendBlobOutputStream(client, appendBlobRequestConditions, contentValidationAlgorithm); } /** @@ -104,12 +113,18 @@ public static BlobOutputStream blockBlobOutputStream(final BlobAsyncClient clien BlockBlobOutputStreamOptions options, Context context) { options = options == null ? new BlockBlobOutputStreamOptions() : options; return new BlockBlobOutputStream(client, options.getParallelTransferOptions(), options.getHeaders(), - options.getMetadata(), options.getTags(), options.getTier(), options.getRequestConditions(), context); + options.getMetadata(), options.getTags(), options.getTier(), options.getRequestConditions(), + options.getContentValidationAlgorithm(), context); } static BlobOutputStream pageBlobOutputStream(final PageBlobAsyncClient client, final PageRange pageRange, final BlobRequestConditions requestConditions) { - return new PageBlobOutputStream(client, pageRange, requestConditions); + return pageBlobOutputStream(client, pageRange, requestConditions, null); + } + + static BlobOutputStream pageBlobOutputStream(final PageBlobAsyncClient client, final PageRange pageRange, + final BlobRequestConditions requestConditions, final ContentValidationAlgorithm contentValidationAlgorithm) { + return new PageBlobOutputStream(client, pageRange, requestConditions, contentValidationAlgorithm); } abstract void commit(); @@ -157,9 +172,11 @@ private static final class AppendBlobOutputStream extends BlobOutputStream { private final AppendBlobRequestConditions appendBlobRequestConditions; private final AppendBlobAsyncClient client; + private final ContentValidationAlgorithm contentValidationAlgorithm; private AppendBlobOutputStream(final AppendBlobAsyncClient client, - final AppendBlobRequestConditions appendBlobRequestConditions) { + final AppendBlobRequestConditions appendBlobRequestConditions, + final ContentValidationAlgorithm contentValidationAlgorithm) { // service versions 2022-11-02 and above support uploading block bytes up to 100MB, all older service // versions support up to 4MB super(client.getServiceVersion().ordinal() < BlobServiceVersion.V2022_11_02.ordinal() @@ -170,6 +187,7 @@ private AppendBlobOutputStream(final AppendBlobAsyncClient client, this.appendBlobRequestConditions = (appendBlobRequestConditions == null) ? new AppendBlobRequestConditions() : appendBlobRequestConditions; + this.contentValidationAlgorithm = contentValidationAlgorithm; if (this.appendBlobRequestConditions.getAppendPosition() == null) { this.appendBlobRequestConditions.setAppendPosition(client.getProperties().block().getBlobSize()); @@ -178,7 +196,10 @@ private AppendBlobOutputStream(final AppendBlobAsyncClient client, private Mono appendBlock(Flux blockData, long writeLength) { long newAppendOffset = appendBlobRequestConditions.getAppendPosition() + writeLength; - return client.appendBlockWithResponse(blockData, writeLength, null, appendBlobRequestConditions) + AppendBlobAppendBlockOptions opts = new AppendBlobAppendBlockOptions(blockData, writeLength) + .setRequestConditions(appendBlobRequestConditions) + .setContentValidationAlgorithm(contentValidationAlgorithm); + return client.appendBlockWithResponse(opts) .doOnNext(ignored -> appendBlobRequestConditions.setAppendPosition(newAppendOffset)) .then() .onErrorResume(t -> t instanceof IOException || t instanceof BlobStorageException, e -> { @@ -223,7 +244,8 @@ private static final class BlockBlobOutputStream extends BlobOutputStream { private BlockBlobOutputStream(final BlobAsyncClient client, final ParallelTransferOptions parallelTransferOptions, final BlobHttpHeaders headers, final Map metadata, Map tags, final AccessTier tier, - final BlobRequestConditions requestConditions, Context context) { + final BlobRequestConditions requestConditions, final ContentValidationAlgorithm contentValidationAlgorithm, + Context context) { super(Integer.MAX_VALUE); // writeThreshold is effectively not used by BlockBlobOutputStream. // There is a bug in reactor core that does not handle converting Context.NONE to a reactor context. context = context == null || context.equals(Context.NONE) ? null : context; @@ -241,7 +263,8 @@ private BlockBlobOutputStream(final BlobAsyncClient client, .setMetadata(metadata) .setTags(tags) .setTier(tier) - .setRequestConditions(requestConditions)) + .setRequestConditions(requestConditions) + .setContentValidationAlgorithm(contentValidationAlgorithm)) // This allows the operation to continue while maintaining the error that occurred. .onErrorResume(e -> { if (e instanceof IOException) { @@ -319,12 +342,15 @@ private static final class PageBlobOutputStream extends BlobOutputStream { private final PageBlobAsyncClient client; private final PageBlobRequestConditions pageBlobRequestConditions; private final PageRange pageRange; + private final ContentValidationAlgorithm contentValidationAlgorithm; private PageBlobOutputStream(final PageBlobAsyncClient client, final PageRange pageRange, - final BlobRequestConditions blobRequestConditions) { + final BlobRequestConditions blobRequestConditions, + final ContentValidationAlgorithm contentValidationAlgorithm) { super(PageBlobClient.MAX_PUT_PAGES_BYTES); this.client = client; this.pageRange = pageRange; + this.contentValidationAlgorithm = contentValidationAlgorithm; if (blobRequestConditions != null) { this.pageBlobRequestConditions @@ -340,8 +366,8 @@ private PageBlobOutputStream(final PageBlobAsyncClient client, final PageRange p private Mono writePages(Flux pageData, int length, long offset) { return client - .uploadPagesWithResponse(new PageRange().setStart(offset).setEnd(offset + length - 1), pageData, null, - pageBlobRequestConditions) + .uploadPagesWithResponseInternal(new PageRange().setStart(offset).setEnd(offset + length - 1), pageData, + null, pageBlobRequestConditions, contentValidationAlgorithm, com.azure.core.util.Context.NONE) .then() .onErrorResume(BlobStorageException.class, e -> { this.lastError = new IOException(e); diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java index f938e21b4793..7546ae561a87 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobAsyncClient.java @@ -42,6 +42,8 @@ import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.implementation.StorageImplUtils; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -415,6 +417,9 @@ public Mono> uploadWithResponse(BlockBlobSimpleUploadOpt Mono> uploadWithResponse(BlockBlobSimpleUploadOptions options, Context context) { StorageImplUtils.assertNotNull("options", options); + + ContentValidationAlgorithm contentValidationAlgorithm = options.getContentValidationAlgorithm(); + Mono dataMono; BinaryData binaryData = options.getData(); if (binaryData == null) { @@ -428,25 +433,32 @@ Mono> uploadWithResponse(BlockBlobSimpleUploadOptions op } BlobRequestConditions requestConditions = options.getRequestConditions() == null ? new BlobRequestConditions() : options.getRequestConditions(); - Context finalContext = context == null ? Context.NONE : context; BlobImmutabilityPolicy immutabilityPolicy = options.getImmutabilityPolicy() == null ? new BlobImmutabilityPolicy() : options.getImmutabilityPolicy(); - return dataMono.flatMap(data -> this.azureBlobStorage.getBlockBlobs() - .uploadWithResponseAsync(containerName, blobName, options.getLength(), data, null, options.getContentMd5(), - options.getMetadata(), requestConditions.getLeaseId(), options.getTier(), - requestConditions.getIfModifiedSince(), requestConditions.getIfUnmodifiedSince(), - requestConditions.getIfMatch(), requestConditions.getIfNoneMatch(), - requestConditions.getTagsConditions(), null, ModelHelper.tagsToString(options.getTags()), - immutabilityPolicy.getExpiryTime(), immutabilityPolicy.getPolicyMode(), options.isLegalHold(), null, - null, null, options.getHeaders(), getCustomerProvidedKey(), encryptionScope, finalContext) - .map(rb -> { - BlockBlobsUploadHeaders hd = rb.getDeserializedHeaders(); - BlockBlobItem item = new BlockBlobItem(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), - hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), - hd.getXMsVersionId()); - return new SimpleResponse<>(rb, item); - })); + context = ContentValidationModeResolver.addContentValidationMode(context, contentValidationAlgorithm, + options.getLength(), false); + + Context finalContext = context; + + return dataMono.flatMap(data -> { + Mono> responseMono = this.azureBlobStorage.getBlockBlobs() + .uploadWithResponseAsync(containerName, blobName, options.getLength(), data, null, + options.getContentMd5(), options.getMetadata(), requestConditions.getLeaseId(), options.getTier(), + requestConditions.getIfModifiedSince(), requestConditions.getIfUnmodifiedSince(), + requestConditions.getIfMatch(), requestConditions.getIfNoneMatch(), + requestConditions.getTagsConditions(), null, ModelHelper.tagsToString(options.getTags()), + immutabilityPolicy.getExpiryTime(), immutabilityPolicy.getPolicyMode(), options.isLegalHold(), null, + null, null, options.getHeaders(), getCustomerProvidedKey(), encryptionScope, finalContext) + .map(rb -> { + BlockBlobsUploadHeaders hd = rb.getDeserializedHeaders(); + BlockBlobItem item = new BlockBlobItem(hd.getETag(), hd.getLastModified(), hd.getContentMD5(), + hd.isXMsRequestServerEncrypted(), hd.getXMsEncryptionKeySha256(), hd.getXMsEncryptionScope(), + hd.getXMsVersionId()); + return new SimpleResponse<>(rb, item); + }); + return responseMono; + }); } /** @@ -707,12 +719,10 @@ public Mono stageBlock(String base64BlockId, BinaryData data) { @ServiceMethod(returns = ReturnType.SINGLE) public Mono> stageBlockWithResponse(String base64BlockId, Flux data, long length, byte[] contentMd5, String leaseId) { - try { - return withContext( - context -> stageBlockWithResponse(base64BlockId, data, length, contentMd5, leaseId, context)); - } catch (RuntimeException ex) { - return monoError(LOGGER, ex); - } + return BinaryData.fromFlux(data, length, false) + .flatMap(binaryData -> stageBlockWithResponse( + new BlockBlobStageBlockOptions(base64BlockId, binaryData).setContentMd5(contentMd5) + .setLeaseId(leaseId))); } /** @@ -745,27 +755,24 @@ public Mono> stageBlockWithResponse(String base64BlockId, Flux> stageBlockWithResponse(BlockBlobStageBlockOptions options) { Objects.requireNonNull(options, "options must not be null"); try { - return withContext(context -> stageBlockWithResponse(options.getBase64BlockId(), options.getData(), - options.getContentMd5(), options.getLeaseId(), context)); + return withContext(context -> stageBlockWithResponseInternal(options, context)); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } } - Mono> stageBlockWithResponse(String base64BlockId, Flux data, long length, - byte[] contentMd5, String leaseId, Context context) { - return BinaryData.fromFlux(data, length, false) - .flatMap(binaryData -> stageBlockWithResponse(base64BlockId, binaryData, contentMd5, leaseId, context)); - } + Mono> stageBlockWithResponseInternal(BlockBlobStageBlockOptions options, Context context) { + Objects.requireNonNull(options.getData(), "data must not be null"); + Objects.requireNonNull(options.getData().getLength(), "data must have defined length"); + + context = ContentValidationModeResolver.addContentValidationMode(context, + options.getContentValidationAlgorithm(), options.getData().getLength(), false); - Mono> stageBlockWithResponse(String base64BlockId, BinaryData data, byte[] contentMd5, - String leaseId, Context context) { - Objects.requireNonNull(data, "data must not be null"); - Objects.requireNonNull(data.getLength(), "data must have defined length"); - context = context == null ? Context.NONE : context; return this.azureBlobStorage.getBlockBlobs() - .stageBlockNoCustomHeadersWithResponseAsync(containerName, blobName, base64BlockId, data.getLength(), data, - contentMd5, null, null, leaseId, null, null, null, getCustomerProvidedKey(), encryptionScope, context); + .stageBlockNoCustomHeadersWithResponseAsync(containerName, blobName, options.getBase64BlockId(), + options.getData().getLength(), options.getData(), options.getContentMd5(), null, null, + options.getLeaseId(), null, null, null, getCustomerProvidedKey(), encryptionScope, context); + } /** diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java index 8667e7afe28c..2ab1287cbe63 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/BlockBlobClient.java @@ -314,9 +314,7 @@ public SeekableByteChannel openSeekableByteChannelWrite(BlockBlobSeekableByteCha options.getBlockSizeInBytes() != null ? options.getBlockSizeInBytes().intValue() : BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE, - new StorageSeekableByteChannelBlockBlobWriteBehavior(this, options.getHeaders(), options.getMetadata(), - options.getTags(), options.getTier(), options.getRequestConditions(), internalMode, null), - startingPosition); + new StorageSeekableByteChannelBlockBlobWriteBehavior(this, options, internalMode, null), startingPosition); } private BlobClientBuilder prepareBuilder() { @@ -787,11 +785,14 @@ public void stageBlock(String base64BlockId, BinaryData data) { public Response stageBlockWithResponse(String base64BlockId, InputStream data, long length, byte[] contentMd5, String leaseId, Duration timeout, Context context) { StorageImplUtils.assertNotNull("data", data); + Flux fbb = Utility.convertStreamToByteBuffer(data, length, BlobAsyncClient.BLOB_DEFAULT_UPLOAD_BLOCK_SIZE, true); - Mono> response - = client.stageBlockWithResponse(base64BlockId, fbb, length, contentMd5, leaseId, context); + Mono> response = BinaryData.fromFlux(fbb, length, false) + .flatMap(binaryData -> client.stageBlockWithResponseInternal( + new BlockBlobStageBlockOptions(base64BlockId, binaryData).setContentMd5(contentMd5).setLeaseId(leaseId), + context)); return blockWithOptionalTimeout(response, timeout); } @@ -827,8 +828,8 @@ public Response stageBlockWithResponse(String base64BlockId, InputStream d public Response stageBlockWithResponse(BlockBlobStageBlockOptions options, Duration timeout, Context context) { Objects.requireNonNull(options, "options must not be null"); - Mono> response = client.stageBlockWithResponse(options.getBase64BlockId(), options.getData(), - options.getContentMd5(), options.getLeaseId(), context); + + Mono> response = client.stageBlockWithResponseInternal(options, context); return blockWithOptionalTimeout(response, timeout); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobAsyncClient.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobAsyncClient.java index 5943047cf49b..d46ea4e9fb0b 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobAsyncClient.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/PageBlobAsyncClient.java @@ -56,8 +56,12 @@ import com.azure.storage.blob.options.PageBlobCopyIncrementalOptions; import com.azure.storage.blob.options.PageBlobCreateOptions; import com.azure.storage.blob.options.PageBlobUploadPagesFromUrlOptions; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; import com.azure.storage.common.implementation.StorageImplUtils; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -488,18 +492,36 @@ public Mono uploadPages(PageRange pageRange, Flux body * operation will fail. * @param pageBlobRequestConditions {@link PageBlobRequestConditions} * @return A reactive response containing the information of the uploaded pages. - * - * @throws IllegalArgumentException If {@code pageRange} is {@code null} */ @ServiceMethod(returns = ReturnType.SINGLE) public Mono> uploadPagesWithResponse(PageRange pageRange, Flux body, byte[] contentMd5, PageBlobRequestConditions pageBlobRequestConditions) { if (body == null) { - return Mono.error(new NullPointerException("'body' cannot be null.")); + return monoError(LOGGER, new NullPointerException("'body' cannot be null.")); + } + return uploadPagesWithResponse(new PageBlobUploadPagesOptions(pageRange, body).setContentMd5(contentMd5) + .setRequestConditions(pageBlobRequestConditions)); + } + + /** + * Writes one or more pages to the page blob with options. + * + * @param options {@link PageBlobUploadPagesOptions} (must be constructed with {@link Flux} body for async). + * @return A reactive response containing the information of the uploaded pages. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Mono> uploadPagesWithResponse(PageBlobUploadPagesOptions options) { + if (options == null) { + return monoError(LOGGER, new NullPointerException("'options' cannot be null.")); + } + if (options.getDataFlux() == null) { + return monoError(LOGGER, new IllegalArgumentException( + "PageBlobUploadPagesOptions must be constructed with Flux for async client.")); } try { - return withContext( - context -> uploadPagesWithResponse(pageRange, body, contentMd5, pageBlobRequestConditions, context)); + return withContext(context -> uploadPagesWithResponseInternal(options.getPageRange(), options.getDataFlux(), + options.getContentMd5(), options.getRequestConditions(), options.getContentValidationAlgorithm(), + context)); } catch (RuntimeException ex) { return monoError(LOGGER, ex); } @@ -507,6 +529,13 @@ public Mono> uploadPagesWithResponse(PageRange pageRange, Mono> uploadPagesWithResponse(PageRange pageRange, Flux body, byte[] contentMd5, PageBlobRequestConditions pageBlobRequestConditions, Context context) { + // Prevents revapi visibility increased error + return uploadPagesWithResponseInternal(pageRange, body, contentMd5, pageBlobRequestConditions, null, context); + } + + Mono> uploadPagesWithResponseInternal(PageRange pageRange, Flux body, + byte[] contentMd5, PageBlobRequestConditions pageBlobRequestConditions, + ContentValidationAlgorithm contentValidationAlgorithm, Context context) { pageBlobRequestConditions = pageBlobRequestConditions == null ? new PageBlobRequestConditions() : pageBlobRequestConditions; @@ -515,12 +544,15 @@ Mono> uploadPagesWithResponse(PageRange pageRange, Flux uploadPagesWithResponse(PageRange pageRange, InputStream body, byte[] contentMd5, PageBlobRequestConditions pageBlobRequestConditions, Duration timeout, Context context) { - Objects.requireNonNull(body, "'body' cannot be null."); - final long length = pageRange.getEnd() - pageRange.getStart() + 1; - Flux fbb = Utility.convertStreamToByteBuffer(body, length, PAGE_BYTES, true); + return uploadPagesWithResponse(new PageBlobUploadPagesOptions(pageRange, body).setContentMd5(contentMd5) + .setRequestConditions(pageBlobRequestConditions), timeout, context); + } - Mono> response = pageBlobAsyncClient.uploadPagesWithResponse(pageRange, fbb, contentMd5, - pageBlobRequestConditions, context); + /** + * Writes one or more pages to the page blob with options. + * + * @param options {@link PageBlobUploadPagesOptions} + * @param timeout An optional timeout value. + * @param context Additional context. + * @return The information of the uploaded pages. + * @throws NullPointerException If {@code options} is {@code null}. + * @throws IllegalArgumentException if options is not constructed with InputStream. + * @throws UnexpectedLengthException If the length of the data read from the provided stream does not match the + * expected length based on the specified page range. + */ + @ServiceMethod(returns = ReturnType.SINGLE) + public Response uploadPagesWithResponse(PageBlobUploadPagesOptions options, Duration timeout, + Context context) { + StorageImplUtils.assertNotNull("options", options); + if (options.getDataStream() == null) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + "PageBlobUploadPagesOptions must be constructed with InputStream for sync client.")); + } + final long length = options.getPageRange().getEnd() - options.getPageRange().getStart() + 1; + Flux fbb = Utility.convertStreamToByteBuffer(options.getDataStream(), length, PAGE_BYTES, true); + Mono> response + = pageBlobAsyncClient.uploadPagesWithResponseInternal(options.getPageRange(), fbb, options.getContentMd5(), + options.getRequestConditions(), options.getContentValidationAlgorithm(), context); return StorageImplUtils.blockWithOptionalTimeout(response, timeout); } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/StorageSeekableByteChannelBlockBlobWriteBehavior.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/StorageSeekableByteChannelBlockBlobWriteBehavior.java index 71eb080a48cc..d98a78072519 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/StorageSeekableByteChannelBlockBlobWriteBehavior.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/specialized/StorageSeekableByteChannelBlockBlobWriteBehavior.java @@ -10,7 +10,9 @@ import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobRequestConditions; import com.azure.storage.blob.options.BlockBlobCommitBlockListOptions; +import com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions; import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.common.ContentValidationAlgorithm; import com.azure.storage.common.implementation.StorageSeekableByteChannel; import java.io.IOException; @@ -43,6 +45,7 @@ enum WriteMode { private final WriteMode mode; private final List existingBlockIds; private final List newBlockIds = new ArrayList<>(); + private final ContentValidationAlgorithm contentValidationAlgorithm; StorageSeekableByteChannelBlockBlobWriteBehavior(BlockBlobClient client, BlobHttpHeaders headers, Map metadata, Map tags, AccessTier tier, BlobRequestConditions conditions, @@ -55,6 +58,21 @@ enum WriteMode { this.conditions = conditions; this.mode = Objects.requireNonNull(mode); this.existingBlockIds = existingBlockIds != null ? existingBlockIds : Collections.emptyList(); + this.contentValidationAlgorithm = null; + } + + StorageSeekableByteChannelBlockBlobWriteBehavior(BlockBlobClient client, + BlockBlobSeekableByteChannelWriteOptions options, WriteMode mode, List existingBlockIds) { + this.client = Objects.requireNonNull(client); + Objects.requireNonNull(options); + this.headers = options.getHeaders(); + this.metadata = options.getMetadata(); + this.tags = options.getTags(); + this.tier = options.getTier(); + this.conditions = options.getRequestConditions(); + this.mode = Objects.requireNonNull(mode); + this.existingBlockIds = existingBlockIds != null ? existingBlockIds : Collections.emptyList(); + this.contentValidationAlgorithm = options.getContentValidationAlgorithm(); } BlockBlobClient getClient() { @@ -100,6 +118,9 @@ public void write(ByteBuffer src, long destOffset) throws IOException { if (conditions != null) { options.setLeaseId(conditions.getLeaseId()); } + if (contentValidationAlgorithm != null) { + options.setContentValidationAlgorithm(contentValidationAlgorithm); + } client.stageBlockWithResponse(options, null, null); newBlockIds.add(blockId); diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobApiTests.java index 50a9eb63ef21..71c474ba295c 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobApiTests.java @@ -543,6 +543,36 @@ public void downloadAllNullBinaryData() { // headers.getLastAccessedTime() /* TODO (gapra): re-enable when last access time enabled. */ } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void downloadSmartAccessTierHeaders() { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + bc.setAccessTier(AccessTier.SMART); + + BlobDownloadResponse response = bc.downloadStreamWithResponse(stream, null, null, null, false, null, null); + ByteBuffer body = ByteBuffer.wrap(stream.toByteArray()); + + assertEquals(DATA.getDefaultData(), body); + assertSmartAccessTierHeaders(response.getDeserializedHeaders()); + } + + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void downloadContentSmartAccessTierHeaders() { + bc.setAccessTier(AccessTier.SMART); + BlobDownloadContentResponse response = bc.downloadContentWithResponse(null, null, null, null); + + TestUtils.assertArraysEqual(DATA.getDefaultBytes(), response.getValue().toBytes()); + assertSmartAccessTierHeaders(response.getDeserializedHeaders()); + } + + private static void assertSmartAccessTierHeaders(BlobDownloadHeaders headers) { + assertEquals(AccessTier.SMART, headers.getAccessTier()); + assertNotNull(headers.getSmartAccessTier()); + assertFalse(headers.isAccessTierInferred()); + assertNotEquals(OffsetDateTime.now(), headers.getAccessTierChangeTime()); + } + @Test public void downloadEmptyFile() { AppendBlobClient bc = cc.getBlobClient("emptyAppendBlob").getAppendBlobClient(); diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobAsyncApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobAsyncApiTests.java index 049e4254e92a..ea01df338d18 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobAsyncApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobAsyncApiTests.java @@ -382,6 +382,33 @@ public void downloadAllNullBinaryData() { .verifyComplete(); } + @RequiredServiceVersion(clazz = BlobServiceVersion.class, min = "2026-10-06") + @Test + public void downloadSmartAccessTierHeaders() { + Mono response = bc.setAccessTier(AccessTier.SMART) + .then(bc.downloadStreamWithResponse(null, null, null, false)) + .flatMap(r -> { + assertSmartAccessTierHeaders(r.getDeserializedHeaders()); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + }) + .flatMap(r -> { + TestUtils.assertArraysEqual(DATA.getDefaultBytes(), r); + return bc.downloadContentWithResponse(null, null); + }); + + StepVerifier.create(response).assertNext(r -> { + assertSmartAccessTierHeaders(r.getDeserializedHeaders()); + TestUtils.assertArraysEqual(DATA.getDefaultBytes(), r.getValue().toBytes()); + }).verifyComplete(); + } + + private static void assertSmartAccessTierHeaders(BlobDownloadHeaders headers) { + assertEquals(AccessTier.SMART, headers.getAccessTier()); + assertNotNull(headers.getSmartAccessTier()); + assertFalse(headers.isAccessTierInferred()); + assertNotEquals(OffsetDateTime.now(), headers.getAccessTierChangeTime()); + } + @Test public void downloadEmptyFile() { AppendBlobAsyncClient bc = ccAsync.getBlobAsyncClient("emptyAppendBlob").getAppendBlobAsyncClient(); diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java new file mode 100644 index 000000000000..5691f78d6c72 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncDownloadTests.java @@ -0,0 +1,681 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.test.utils.TestUtils; +import com.azure.core.util.BinaryData; +import com.azure.core.util.FluxUtil; +import com.azure.core.util.ProgressListener; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; +import reactor.util.function.Tuples; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Async tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. + * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. + */ +public class BlobContentValidationAsyncDownloadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + private static final int BLOCK_SIZE = 4 * Constants.MB; + /** + * {@link BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases()} starts at ~96 MiB; above this threshold fuzzy + * parallel download helpers use temp files + {@link BlobTestBase#compareFiles(File, File, long, long)} so the full + * payload never lives twice in heap. + */ + private static final int FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES = 96 * Constants.MB; + + /** + * Live-only random payload band for the dedicated random-size parallel-download fuzzy test + * ({@link #fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm)}): each run draws a per-run + * payload size in {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) so the structured-message + * decoder is exercised against payloads whose size varies per run in addition to the random byte contents. + */ + private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB; + private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB; + + private final List createdFiles = new ArrayList<>(); + + private File createRandomFile(Path tempDir, int size) throws IOException { + File file = Files.createTempFile(tempDir, "blob-cv-source", ".bin").toFile(); + + if (size > Constants.MB) { + try (OutputStream outputStream = Files.newOutputStream(file.toPath())) { + byte[] data = getRandomByteArray(Constants.MB); + int mbChunks = size / Constants.MB; + int remaining = size % Constants.MB; + for (int i = 0; i < mbChunks; i++) { + outputStream.write(data); + } + if (remaining > 0) { + outputStream.write(data, 0, remaining); + } + } + } else { + Files.write(file.toPath(), getRandomByteArray(size)); + } + + return file; + } + + /** + * downloadStreamWithResponse with CRC64 content validation. + */ + @Test + public void downloadStreamWithResponseContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)).block(); + + BlobDownloadStreamOptions options + = new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(blobClient.downloadStreamWithResponse(options).flatMap(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + })).assertNext(result -> TestUtils.assertArraysEqual(data, result)).verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with CRC64 content validation. + */ + @Test + public void downloadContentWithResponseContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)).block(); + + BlobDownloadContentOptions options + = new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(blobClient.downloadContentWithResponse(options)).assertNext(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + TestUtils.assertArraysEqual(data, r.getValue().toBytes()); + }).verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation. + */ + @ParameterizedTest + @ValueSource( + ints = { + 0, // empty file + 20, // small file + 16 * 1024 * 1024, // medium file in several chunks + 8 * 1026 * 1024 + 10, // medium file not aligned to block + }) + public void downloadToFileWithResponseContentValidation(int fileSize, @TempDir Path tempDir) throws IOException { + File file = createRandomFile(tempDir, fileSize); + File outFile = tempDir.resolve("download.bin").toFile(); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.uploadFromFile(file.toPath().toString(), true).block(); + + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) BLOCK_SIZE); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(blobClient.downloadToFileWithResponse(options)).assertNext(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + assertNotNull(r.getValue()); + }).verifyComplete(); + + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @LiveOnly + @ParameterizedTest + @ValueSource( + ints = { + 50 * Constants.MB, //large file requiring multiple requests + 50 * Constants.MB + 22 // large file not on MB boundary + }) + public void downloadToFileLargeWithResponseContentValidation(int fileSize, @TempDir Path tempDir) + throws IOException { + File file = createRandomFile(tempDir, fileSize); + File outFile = tempDir.resolve("download.bin").toFile(); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.uploadFromFile(file.toPath().toString(), true).block(); + + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) BLOCK_SIZE); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(blobClient.downloadToFileWithResponse(options)).assertNext(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + assertNotNull(r.getValue()); + }).verifyComplete(); + + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Default behavior: when no algorithm is specified, default is NONE (no validation). + */ + @Test + public void downloadStreamDefaultAlgorithmIsNone() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StepVerifier.create(blobClient.downloadStreamWithResponse(new BlobDownloadStreamOptions()) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))).assertNext(result -> { + assertNotNull(result); + assertEquals(data.length, result.length); + }).verifyComplete(); + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + @Test + public void downloadStreamWithResponseContentValidationRange() { + byte[] data = getRandomByteArray(4 * Constants.KB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)).block(); + + BlobDownloadStreamOptions options = new BlobDownloadStreamOptions().setRange(new BlobRange(0, 512L)); + + StepVerifier.create(blobClient.downloadStreamWithResponse(options).flatMap(r -> { + assertFalse(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + })).assertNext(result -> assertEquals(512, result.length)).verifyComplete(); + + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * AUTO on downloadStream resolves to CRC64 behavior. + */ + @Test + public void downloadStreamWithAuto() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)).block(); + + StepVerifier.create(blobClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO)) + .flatMap(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + })).assertNext(result -> TestUtils.assertArraysEqual(data, result)).verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with NONE: no validation triggered. + */ + @Test + public void downloadContentWithNone() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + StepVerifier + .create(blobClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) + .assertNext(r -> TestUtils.assertArraysEqual(data, r.getValue().toBytes())) + .verifyComplete(); + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with AUTO resolves to CRC64 behavior. + */ + @Test + public void downloadContentWithAuto() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)).block(); + + StepVerifier + .create(blobClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO))) + .assertNext(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + TestUtils.assertArraysEqual(data, r.getValue().toBytes()); + }) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Interrupt with proper rewind to segment boundary; verifies retry range headers. + */ + @Test + public void interruptAndVerifyProperRewind() { + final int segmentSize = Constants.KB; + byte[] data = getRandomByteArray(2 * segmentSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + int interruptPos = segmentSize + (2 * (segmentSize / 4)) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + StepVerifier.create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .doFinally(signalType -> assertTrue(mockPolicy.getHits() > 0, "Mock interruption policy was not invoked")) + .flatMap(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + })).assertNext(result -> TestUtils.assertArraysEqual(data, result)).verifyComplete(); + + assertEquals(0, mockPolicy.getTriesRemaining(), "Expected the configured interruption to be consumed"); + assertTrue(mockPolicy.getRangeHeaders().size() >= 2, + "Expected at least the initial request and one retry with a range header"); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Proper decode across retries (single and multiple interrupts). + */ + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void interruptAndVerifyProperDecode(boolean multipleInterrupts) { + final int segmentSize = 128 * Constants.KB; + final int dataSize = 4 * Constants.KB; + byte[] data = getRandomByteArray(dataSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + int interruptPos = segmentSize + (3 * (8 * Constants.KB)) + 10; + MockPartialResponsePolicy mockPolicy + = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + + StepVerifier.create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> { + assertTrue(hasStructuredMessageDownloadResponseHeaders(r.getHeaders())); + return FluxUtil.collectBytesInByteBufferStream(r.getValue()); + })).assertNext(result -> { + assertEquals(dataSize, result.length, "Decoded data should have exactly " + dataSize + " bytes"); + TestUtils.assertArraysEqual(data, result); + }).verifyComplete(); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * After consuming the response stream with CRC64 validation, decoded payload preserves the expected CRC64. + */ + @Test + public void structuredMessageVerifiesDecodedCrc64DownloadStreaming() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)).block(); + + long expectedCrc = StorageCrc64Calculator.compute(data, 0); + + StepVerifier + .create(blobClient + .downloadStreamWithResponse( + new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()).map(bytes -> Tuples.of(r, bytes)))) + .assertNext(tuple -> { + TestUtils.assertArraysEqual(data, tuple.getT2()); + long actualCrc = StorageCrc64Calculator.compute(tuple.getT2(), 0); + assertEquals(expectedCrc, actualCrc); + }) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Single interrupt with data intact: fault policy + decoder; structured message retry recovers. + */ + @Test + public void interruptWithDataIntact() { + final int segmentSize = Constants.KB; + byte[] data = getRandomByteArray(4 * segmentSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + int interruptPos = segmentSize + (3 * 128) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + StepVerifier + .create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + } + + /** + * Multiple interrupts with data intact: fault policy + decoder; structured message retry recovers. + */ + @Test + public void interruptMultipleTimesWithDataIntact() { + final int segmentSize = Constants.KB; + byte[] data = getRandomByteArray(4 * segmentSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recordedRequestHeaders); + + int interruptPos = segmentSize + (3 * 128) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(3, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + blobClient.upload(Flux.just(ByteBuffer.wrap(data)), null, true).block(); + + BlobAsyncClient downloadClient = getBlobAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + + StepVerifier + .create(downloadClient + .downloadStreamWithResponse(new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64)) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(result -> TestUtils.assertArraysEqual(data, result)) + .verifyComplete(); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + } + + @Test + public void verifyProgressListenerIsCompatibleWithContentValidation(@TempDir Path tempDir) throws IOException { + byte[] data = getRandomByteArray(10 * Constants.MB); + + BlobAsyncClient client = ccAsync.getBlobAsyncClient(generateBlobName()); + + MockProgressListener mockListenerWithContentVal = new MockProgressListener(); + MockProgressListener mockListenerWithoutContentVal = new MockProgressListener(); + + ParallelTransferOptions parallelOptionsWithContentVal + = new ParallelTransferOptions().setProgressListener(mockListenerWithContentVal); + ParallelTransferOptions parallelOptionsWithoutContentVal + = new ParallelTransferOptions().setProgressListener(mockListenerWithoutContentVal); + + File fileWithContentVal = createRandomFile(tempDir, 10 * Constants.MB); + File outFileWithContentVal = tempDir.resolve("withcontentval.bin").toFile(); + File fileWithoutContentVal = createRandomFile(tempDir, 10 * Constants.MB); + File outFileWithoutContentVal = tempDir.resolve("withoutcontentval.bin").toFile(); + + Files.deleteIfExists(outFileWithContentVal.toPath()); + Files.deleteIfExists(outFileWithoutContentVal.toPath()); + + BlobDownloadToFileOptions optionsWithContentVal + = new BlobDownloadToFileOptions(outFileWithContentVal.getAbsolutePath()) + .setParallelTransferOptions(parallelOptionsWithContentVal) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobDownloadToFileOptions optionsWithoutContentVal + = new BlobDownloadToFileOptions(outFileWithoutContentVal.getAbsolutePath()) + .setParallelTransferOptions(parallelOptionsWithoutContentVal); + + StepVerifier.create(client.upload(BinaryData.fromBytes(data)) + .then(client.downloadToFileWithResponse(optionsWithContentVal)) + .then(client.downloadToFileWithResponse(optionsWithoutContentVal))).assertNext(ignored -> { + long expectedBytes = data.length; + assertEquals(expectedBytes, mockListenerWithContentVal.getReportedByteCount()); + assertEquals(expectedBytes, mockListenerWithoutContentVal.getReportedByteCount()); + }).verifyComplete(); + } + + private static final class MockProgressListener implements ProgressListener { + private final AtomicLong reportedByteCount = new AtomicLong(0L); + + @Override + public void handleProgress(long bytesTransferred) { + this.reportedByteCount.updateAndGet(current -> Math.max(current, bytesTransferred)); + } + + long getReportedByteCount() { + return this.reportedByteCount.get(); + } + } + + // ---------- Fuzzy parallel download (deterministic grids) ---------- + + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadReplayableCases") + public void fuzzyParallelDownloadReplayableRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("replayable", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload > blockSize with tiny totals; many small range GETs not replayable under the proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSmallMultiPartCases") + public void fuzzyParallelDownloadSmallMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("smallMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // sub-4 MiB chunked range GETs not replayable under the proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSubFourMiBCases") + public void fuzzyParallelDownloadSubFourMiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("subFourMiB", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // 4 MiB boundary tuples that fan out into chunked range GETs. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadFourMiBBoundaryCases") + public void fuzzyParallelDownloadFourMiBBoundaryRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("fourMiBBoundary", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload > blockSize for every tuple; chunked range GETs across many requests. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadMediumMultiPartCases") + public void fuzzyParallelDownloadMediumMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("mediumMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload >> blockSize; ~96-320 MiB downloads. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases") + public void fuzzyParallelDownloadLargeMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("largeMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // ~1 GiB single case; far too large for the test proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadOneGiBCases") + public void fuzzyParallelDownloadOneGiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTripAsync("oneGiB", payloadBytes, blockSizeBytes, maxConcurrency); + } + + /** + * Live-only random-size parallel download fuzzy round-trip. Each run draws a per-run payload size in + * {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) and exercises both CRC64 and AUTO + * content-validation algorithms so the structured-message decoder is tested against payloads whose total size + * varies per run in addition to the random byte contents that the deterministic grids already exercise. Kept + * separate from the parameterized {@link #fuzzyParallelDownloadLargeMultiPartRoundTrip(int, long, int)} so the + * deterministic per-grid round-trips and the randomized round-trip don't share work or cost. + */ + @LiveOnly + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm algorithm) throws IOException { + int sizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + assertParallelDownloadFuzzyRoundTripAsync("liveRandom", sizeBytes, 8L * Constants.MB, 8, algorithm); + } + + private void assertParallelDownloadFuzzyRoundTripAsync(String caseKind, int payloadBytes, long blockSizeBytes, + int maxConcurrency) throws IOException { + assertParallelDownloadFuzzyRoundTripAsync(caseKind, payloadBytes, blockSizeBytes, maxConcurrency, + ContentValidationAlgorithm.CRC64); + } + + private void assertParallelDownloadFuzzyRoundTripAsync(String caseKind, int payloadBytes, long blockSizeBytes, + int maxConcurrency, ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + ParallelTransferOptions parallelOptions + = new ParallelTransferOptions().setBlockSizeLong(blockSizeBytes).setMaxConcurrency(maxConcurrency); + + String assertionMessage = "Fuzzy parallel download [" + caseKind + "] payloadBytes=" + payloadBytes + + ", blockSize=" + blockSizeBytes + ", maxConcurrency=" + maxConcurrency + ", algorithm=" + algorithm; + + if (payloadBytes >= FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES) { + File sourceFile = getRandomFile(payloadBytes); + sourceFile.deleteOnExit(); + createdFiles.add(sourceFile); + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-async", ".bin").toFile(); + outFile.deleteOnExit(); + createdFiles.add(outFile); + Files.deleteIfExists(outFile.toPath()); + + BlobUploadFromFileOptions uploadOptions + = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()).setParallelTransferOptions( + new com.azure.storage.blob.models.ParallelTransferOptions().setBlockSizeLong(blockSizeBytes) + .setMaxConcurrency(maxConcurrency)); + assertNotNull(client.uploadFromFileWithResponse(uploadOptions).block().getValue().getETag(), + assertionMessage); + + BlobDownloadToFileOptions downloadOptions + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.downloadToFileWithResponse(downloadOptions)) + .assertNext(r -> assertNotNull(r.getValue(), assertionMessage)) + .verifyComplete(); + + assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage); + } else { + byte[] randomData = getRandomByteArray(payloadBytes); + client.upload(BinaryData.fromBytes(randomData), true).block(); + + if (payloadBytes > blockSizeBytes) { + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-async-mp", ".bin").toFile(); + outFile.deleteOnExit(); + createdFiles.add(outFile); + Files.deleteIfExists(outFile.toPath()); + + BlobDownloadToFileOptions downloadOptions = new BlobDownloadToFileOptions(outFile.toPath().toString()) + .setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.downloadToFileWithResponse(downloadOptions)) + .assertNext(r -> assertNotNull(r.getValue(), assertionMessage)) + .verifyComplete(); + + byte[] downloaded = Files.readAllBytes(outFile.toPath()); + assertArrayEquals(randomData, downloaded, assertionMessage); + } else { + BlobDownloadContentOptions downloadOptions + = new BlobDownloadContentOptions().setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.downloadContentWithResponse(downloadOptions)) + .assertNext(r -> assertArrayEquals(randomData, r.getValue().toBytes(), assertionMessage)) + .verifyComplete(); + + BlobDownloadStreamOptions streamOptions + = new BlobDownloadStreamOptions().setContentValidationAlgorithm(algorithm); + StepVerifier + .create(client.downloadStreamWithResponse(streamOptions) + .flatMap(r -> FluxUtil.collectBytesInByteBufferStream(r.getValue()))) + .assertNext(bytes -> assertArrayEquals(randomData, bytes, assertionMessage)) + .verifyComplete(); + } + } + assertTrue(hasStructuredMessageDownloadRequestHeaders(recorded, false), assertionMessage); + } + +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java new file mode 100644 index 000000000000..1ee2466e1fb6 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationAsyncUploadTests.java @@ -0,0 +1,1073 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.util.BinaryData; +import com.azure.core.util.FluxUtil; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; +import com.azure.storage.blob.options.BlobParallelUploadOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; +import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.blob.specialized.AppendBlobAsyncClient; +import com.azure.storage.blob.specialized.BlockBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobAsyncClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.AsynchronousFileChannel; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests content validation (CRC64 / structured message) for upload operations using async clients. + * Upload types that have no async counterpart (e.g. OutputStream, SeekableByteChannel) are tested in + * {@link BlobContentValidationUploadTests}. + */ +public class BlobContentValidationAsyncUploadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + /* Single-part uploads with length < 4MB use CRC64 header; >= 4MB use structured message. */ + private static final int UNDER_4MB = 2 * Constants.MB; + + /** + * Live-only random payload band (256–500 MiB, inclusive upper bound via {@code randomLongFromNamer}+1) for + * {@code uploadWithResponse}, {@code uploadFromFileWithResponse}, and single-block {@code stageBlock}. + */ + private static final long LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB; + private static final long LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB; + + /** + * Live-only random payload band for sequential append-block puts only ({@link + * #appendBlockLiveRandomRoundTripDataIntegrity()}): {@code Flux.concatMap} issues one append REST call per chunk in + * order (not parallel staging); use a smaller band than {@link #LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE}. + */ + private static final long LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MIN_BYTES_EXCLUSIVE = 32L * Constants.MB; + private static final long LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MAX_BYTES_INCLUSIVE = 64L * Constants.MB; + + private static final String MD5_AND_CRC64_EXCLUSIVE_MESSAGE + = "Only one form of transactional content validation may be used."; + + // =========================================================================================== + // BlobAsyncClient.uploadWithResponse + // =========================================================================================== + + /** + * Single-part upload under 4MB: content validation uses CRC64 header only (no structured message). + */ + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + /** + * Single-part upload >= 4MB: content validation uses structured message. + */ + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + /** + * Multipart (chunked) upload; content validation uses structured message on each stage block. + */ + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + long blockSize = 2 * (long) Constants.MB; + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void uploadWithoutContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + /** + * Blob parallel upload rejects using both computeMd5 (SDK-computed MD5) and CRC64 (transfer validation checksum algorithm) at once. + */ + @Test + public void uploadWithComputeMd5AndCrc64Throws() { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setComputeMd5(true) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options)) + .verifyErrorMatches(throwable -> throwable instanceof IllegalArgumentException + && throwable.getMessage().contains(MD5_AND_CRC64_EXCLUSIVE_MESSAGE)); + } + + // =========================================================================================== + // BlockBlobAsyncClient.uploadWithResponse (BlockBlobSimpleUpload / Put Blob) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void blockBlobSimpleUploadWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // BlockBlobAsyncClient.stageBlockWithResponse (Put Block) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.stageBlockWithResponse(options)) + .assertNext(response -> assertTrue(hasOnlyCrc64Headers(recorded))) + .verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.stageBlockWithResponse(options)) + .assertNext(response -> assertTrue(hasOnlyStructuredMessageHeaders(recorded))) + .verifyComplete(); + } + + @Test + public void stageBlockWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options = new BlockBlobStageBlockOptions(getBlockID(), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.stageBlockWithResponse(options)) + .assertNext(response -> assertTrue(hasNoContentValidationHeaders(recorded))) + .verifyComplete(); + } + + // =========================================================================================== + // AppendBlobAsyncClient.appendBlockWithResponse (Append Block) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + client.create().block(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions(data, UNDER_4MB).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.appendBlockWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + client.create().block(); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions(data, TEN_MB).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.appendBlockWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void appendBlockWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + client.create().block(); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + AppendBlobAppendBlockOptions options = new AppendBlobAppendBlockOptions(data, TEN_MB) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.appendBlockWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // PageBlobAsyncClient.uploadPagesWithResponse (Put Page) tests + // =========================================================================================== + + private static final int PAGE_BYTES = PageBlobClient.PAGE_BYTES; + private static final int UNDER_4MB_PAGE_ALIGNED = (UNDER_4MB / PAGE_BYTES) * PAGE_BYTES; + private static final int FOUR_MB_PAGE_ALIGNED = (4 * Constants.MB / PAGE_BYTES) * PAGE_BYTES; + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + PageBlobAsyncClient client = blobClient.getPageBlobAsyncClient(); + client.create(UNDER_4MB_PAGE_ALIGNED).block(); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadPagesWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + PageBlobAsyncClient client = blobClient.getPageBlobAsyncClient(); + client.create(FOUR_MB_PAGE_ALIGNED).block(); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadPagesWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void uploadPagesWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + PageBlobAsyncClient client = blobClient.getPageBlobAsyncClient(); + client.create(FOUR_MB_PAGE_ALIGNED).block(); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.uploadPagesWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // BlobAsyncClient.uploadFromFileWithResponse tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithCrc64Header(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(UNDER_4MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadFromFileWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithStructuredMessage(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadFromFileWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + long blockSize = 2 * (long) Constants.MB; + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadFromFileWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @Test + public void uploadFromFileWithNoContentValidation() throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + StepVerifier.create(client.uploadFromFileWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // Exact 4MB boundary tests + // + // The cutoff between CRC64 header and structured message is exactly 4MB. + // Uploads of exactly 4MB should use structured message (>= threshold), not CRC64 header. + // =========================================================================================== + + private static final int EXACTLY_4MB = 4 * Constants.MB; + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) EXACTLY_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(recorded); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)).assertNext(response -> { + assertNotNull(response.getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + }).verifyComplete(); + } + + // =========================================================================================== + // Progress reporting (transfer validation must be NONE/null when a progress listener is set) + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithProgressAndNonNoneContentValidationThrows(ContentValidationAlgorithm algorithm) { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data).setParallelTransferOptions( + new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB).setProgressListener(l -> { + })).setRequestConditions(new BlobRequestConditions()).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadWithResponse(options)) + .expectErrorSatisfies(ex -> assertEquals( + ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, ex.getMessage())) + .verify(); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithProgressAndNonNoneContentValidationThrows(ContentValidationAlgorithm algorithm) + throws IOException { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + File tempFile = getRandomFile(TEN_MB); + + BlobUploadFromFileOptions options + = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()).setParallelTransferOptions( + new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB).setProgressListener(l -> { + })).setContentValidationAlgorithm(algorithm); + + StepVerifier.create(client.uploadFromFileWithResponse(options)) + .expectErrorSatisfies(ex -> assertEquals( + ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, ex.getMessage())) + .verify(); + } + + // =========================================================================================== + // Data integrity round-trip tests (upload with content validation, download, verify) + // + // Previous tests verify that the correct headers are sent. These tests verify end-to-end + // integrity: the data uploaded with CRC64/structured message can be downloaded and matches + // the original byte-for-byte. + // =========================================================================================== + + @Test + public void uploadWithCrc64RoundTripDataIntegrity() { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @Test + public void uploadWithStructuredMessageRoundTripDataIntegrity() { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @Test + public void uploadChunkedWithStructuredMessageRoundTripDataIntegrity() { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + long blockSize = 2 * (long) Constants.MB; + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @Test + public void blockBlobSimpleUploadRoundTripDataIntegrity() { + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @Test + public void appendBlockRoundTripDataIntegrity() { + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + client.create().block(); + + byte[] randomData = getRandomByteArray(TEN_MB); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + AppendBlobAppendBlockOptions options = new AppendBlobAppendBlockOptions(data, TEN_MB) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.appendBlockWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + @Test + public void uploadPagesRoundTripDataIntegrity() { + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + PageBlobAsyncClient client = blobClient.getPageBlobAsyncClient(); + client.create(FOUR_MB_PAGE_ALIGNED).block(); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadPagesWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes())) + .verifyComplete(); + } + + // =========================================================================================== + // Live-only random payload bands. + // - 256–500 MiB: parallelUpload, uploadFromFile, stageBlock — parallel staging / giant block; default transfer + // options where applicable. + // - 32–64 MiB (sequential append blocks only): appendBlockLiveRandom… — one append REST call per chunk in order. + // =========================================================================================== + + @LiveOnly // This test is too large for the test proxy. + @Test + public void parallelUploadLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-par-dl-async", ".bin").toFile(); + outFile.deleteOnExit(); + + try { + try (InputStream data = new FileInputStream(sourceFile)) { + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options).block(); + } + + client.downloadToFile(outFile.getPath(), true).block(); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void stageBlockLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + BlockBlobAsyncClient client = blobClient.getBlockBlobAsyncClient(); + String blockId = getBlockID(); + + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-stage-async-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + BinaryData binaryData = BinaryData.fromFile(sourceFile.toPath()); + BlockBlobStageBlockOptions stageOptions = new BlockBlobStageBlockOptions(blockId, binaryData) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.stageBlockWithResponse(stageOptions).block(); + client.commitBlockList(Collections.singletonList(blockId)).block(); + blobClient.downloadToFile(outFile.getPath(), true).block(); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void appendBlockLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes + = (int) randomLongFromNamer(LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobAsyncClient blobClient = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + AppendBlobAsyncClient client = blobClient.getAppendBlobAsyncClient(); + client.create().block(); + + int maxAppendBlockBytes = client.getMaxAppendBlockBytes(); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-append-async-dl", ".bin").toFile(); + outFile.deleteOnExit(); + + try { + try (AsynchronousFileChannel channel + = AsynchronousFileChannel.open(sourceFile.toPath(), StandardOpenOption.READ)) { + FluxUtil.readFile(channel, maxAppendBlockBytes, 0, chosenPayloadSizeBytes).concatMap(bb -> { + AppendBlobAppendBlockOptions appendOptions + = new AppendBlobAppendBlockOptions(Flux.just(bb), bb.remaining()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + return client.appendBlockWithResponse(appendOptions); + }).then().block(); + } + + blobClient.downloadToFile(outFile.getPath(), true).block(); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void uploadFromFileLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-uploadfromfile-async-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + assertNotNull(client.uploadFromFileWithResponse(options).block().getValue().getETag(), + prefix + "Missing E-Tag on upload-from-file."); + client.downloadToFile(outFile.getPath(), true).block(); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + // ---------- Deterministic parallel upload (async) ---------- + + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadPutBlobReplayableCases") + public void fuzzyParallelUploadPutBlobReplayableRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTripAsync("putBlobReplay", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Staging-only cases: Put Block URLs include random IDs. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadSmallPayloadStagingCases") + public void fuzzyParallelUploadSmallPayloadRoundTripRequiresLiveStaging(int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + assertParallelUploadFuzzyRoundTripAsync("smallPayloadStaging", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // payload > segment for every tuple; always staging/Put Block. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadSub4MiBCases") + public void fuzzyParallelUploadSubFourMiBBlobRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTripAsync("subFourMiB", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Staging-only cases. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadFourMiBBoundaryStagingCases") + public void fuzzyParallelUploadFourMiBBoundaryRoundTripRequiresLiveStaging(int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + assertParallelUploadFuzzyRoundTripAsync("fourMiBBoundaryStaging", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Chunked uploads only. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadMediumMultiPartCases") + public void fuzzyParallelUploadMediumMultiPartRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTripAsync("mediumMultiPart", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Large chunked uploads. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadLargeMultiPartCases") + public void fuzzyParallelUploadLargeMultiPartRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTripAsync("largeMultiPart", payloadBytes, segmentBytes, maxConcurrency); + } + + private void assertParallelUploadFuzzyRoundTripAsync(String caseKind, int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + BlobAsyncClient client = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong(segmentBytes) + .setMaxSingleUploadSizeLong(segmentBytes) + .setMaxConcurrency(maxConcurrency); + + String assertionMessage = "Fuzzy parallel upload [" + caseKind + "] payloadBytes=" + payloadBytes + + ", segmentBytes=" + segmentBytes + ", maxConcurrency=" + maxConcurrency; + + // above this threshold the fuzzy parallel upload helpers stream from a temp source file + // to avoid materializing the full payload twice in heap. + if (payloadBytes >= 96 * Constants.MB) { + File sourceFile = getRandomFile(payloadBytes); + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-async", ".bin").toFile(); + outFile.deleteOnExit(); + int readChunkSize = (int) Math.min(8L * Constants.MB, Math.max(64 * Constants.KB, segmentBytes)); + AsynchronousFileChannel channel + = AsynchronousFileChannel.open(sourceFile.toPath(), StandardOpenOption.READ); + try { + try { + Flux data = FluxUtil.readFile(channel, readChunkSize, 0, payloadBytes); + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setParallelTransferOptions(parallelOptions) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options).block(); + } finally { + channel.close(); + } + client.downloadToFile(outFile.getPath(), true).block(); + assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } else { + byte[] randomData = getRandomByteArray(payloadBytes); + Flux data = Flux.just(ByteBuffer.wrap(randomData)); + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setParallelTransferOptions(parallelOptions) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + StepVerifier.create(client.uploadWithResponse(options).then(client.downloadContent())) + .assertNext(downloaded -> assertArrayEquals(randomData, downloaded.toBytes(), assertionMessage)) + .verifyComplete(); + } + } + + // =========================================================================================== + // Customer Provided MD5 Byte[] with Content Validation Algorithm + // =========================================================================================== + + private static final byte[] DEFAULT_MD5 = createDefaultMd5(); + private static final String MESSAGE = "Both x-ms-content-crc64 header and Content-MD5 header are present."; + + private static byte[] createDefaultMd5() { + try { + return Base64.getEncoder().encode(MessageDigest.getInstance("MD5").digest(DATA.getDefaultBytes())); + } catch (NoSuchAlgorithmException ex) { + throw LOGGER.logExceptionAsError(new RuntimeException("MD5 algorithm unavailable.", ex)); + } + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobUploadWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + BlockBlobAsyncClient client + = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getBlockBlobAsyncClient(); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(DATA.getDefaultBinaryData()).setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + StepVerifier.create(client.uploadWithResponse(options)).verifyErrorSatisfies(ex -> { + BlobStorageException e = assertInstanceOf(BlobStorageException.class, ex); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + }); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + BlockBlobAsyncClient client + = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getBlockBlobAsyncClient(); + + BlockBlobStageBlockOptions options = new BlockBlobStageBlockOptions(getBlockID(), DATA.getDefaultBinaryData()) + .setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + StepVerifier.create(client.stageBlockWithResponse(options)).verifyErrorSatisfies(ex -> { + BlobStorageException e = assertInstanceOf(BlobStorageException.class, ex); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + }); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + AppendBlobAsyncClient client + = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getAppendBlobAsyncClient(); + client.create().block(); + + byte[] randomData = DATA.getDefaultBytes(); + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions(Flux.just(ByteBuffer.wrap(randomData)), randomData.length) + .setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + StepVerifier.create(client.appendBlockWithResponse(options)).verifyErrorSatisfies(ex -> { + BlobStorageException e = assertInstanceOf(BlobStorageException.class, ex); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + }); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) + throws NoSuchAlgorithmException { + PageBlobAsyncClient client + = createBlobAsyncClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getPageBlobAsyncClient(); + client.create(UNDER_4MB_PAGE_ALIGNED).block(); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + byte[] md5 = MessageDigest.getInstance("MD5").digest(randomData); + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1), + Flux.just(ByteBuffer.wrap(randomData))).setContentValidationAlgorithm(algorithm).setContentMd5(md5); + + StepVerifier.create(client.uploadPagesWithResponse(options)).verifyErrorSatisfies(ex -> { + BlobStorageException e = assertInstanceOf(BlobStorageException.class, ex); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + }); + } +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java new file mode 100644 index 000000000000..ef710cf69054 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationDownloadTests.java @@ -0,0 +1,658 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.rest.Response; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.test.utils.TestUtils; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.core.util.ProgressListener; +import com.azure.storage.blob.models.BlobDownloadContentResponse; +import com.azure.storage.blob.models.BlobDownloadResponse; +import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.BlobSeekableByteChannelReadResult; +import com.azure.storage.blob.models.BlobRange; +import com.azure.storage.blob.models.DownloadRetryOptions; +import com.azure.storage.blob.options.BlobDownloadContentOptions; +import com.azure.storage.blob.options.BlobDownloadStreamOptions; +import com.azure.storage.blob.options.BlobDownloadToFileOptions; +import com.azure.storage.blob.options.BlobInputStreamOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.options.BlobSeekableByteChannelReadOptions; +import com.azure.storage.blob.specialized.BlobInputStream; +import com.azure.storage.common.ParallelTransferOptions; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import com.azure.storage.common.test.shared.policy.MockPartialResponsePolicy; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +import static com.azure.storage.blob.specialized.BlobSeekableByteChannelTests.copy; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Sync tests for structured message decoding during blob downloads using StorageContentValidationDecoderPolicy. + * These tests verify that the pipeline policy correctly decodes structured messages when content validation is enabled. + */ +public class BlobContentValidationDownloadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + private static final int BLOCK_SIZE = 4 * Constants.MB; + /** + * {@link BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases()} starts at ~96 MiB; above this threshold fuzzy + * parallel download helpers use temp files + {@link BlobTestBase#compareFiles(File, File, long, long)} so the full + * payload never lives twice in heap. + */ + private static final int FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES = 96 * Constants.MB; + + /** + * Live-only random payload band for the dedicated random-size parallel-download fuzzy test + * ({@link #fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm)}): each run draws a per-run + * payload size in {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) so the structured-message + * decoder is exercised against payloads whose size varies per run in addition to the random byte contents. + */ + private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB; + private static final long LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB; + + private final List createdFiles = new ArrayList<>(); + + private File createRandomFile(Path tempDir, int size) throws IOException { + File file = Files.createTempFile(tempDir, "blob-cv-source", ".bin").toFile(); + + if (size > Constants.MB) { + try (OutputStream outputStream = Files.newOutputStream(file.toPath())) { + byte[] data = getRandomByteArray(Constants.MB); + int mbChunks = size / Constants.MB; + int remaining = size % Constants.MB; + for (int i = 0; i < mbChunks; i++) { + outputStream.write(data); + } + if (remaining > 0) { + outputStream.write(data, 0, remaining); + } + } + } else { + Files.write(file.toPath(), getRandomByteArray(size)); + } + + return file; + } + + /** + * downloadStreamWithResponse with CRC64 content validation. + */ + @Test + public void downloadStreamWithResponseContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadResponse response = blobClient.downloadStreamWithResponse(outputStream, + new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, + Context.NONE); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with CRC64 content validation. + */ + @Test + public void downloadContentWithResponseContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + BlobDownloadContentResponse response = blobClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64), null, + Context.NONE); + byte[] result = response.getValue().toBytes(); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, result); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @ParameterizedTest + @ValueSource( + ints = { + 0, // empty file + 20, // small file + 16 * 1024 * 1024, // medium file in several chunks + 8 * 1026 * 1024 + 10, // medium file not aligned to block + }) + public void downloadToFileWithResponseContentValidation(int fileSize, @TempDir Path tempDir) throws IOException { + File file = createRandomFile(tempDir, fileSize); + File outFile = tempDir.resolve("download.bin").toFile(); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.uploadFromFile(file.toPath().toString(), true); + + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) BLOCK_SIZE); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + Response response = blobClient.downloadToFileWithResponse(options, null, Context.NONE); + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + assertNotNull(response.getValue()); + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadToFileWithResponse with CRC64 content validation (parallel, multiple block sizes). + */ + @LiveOnly + @ParameterizedTest + @ValueSource( + ints = { + 50 * Constants.MB, //large file requiring multiple requests + 50 * Constants.MB + 22 // large file not on MB boundary + }) + public void downloadToFileLargeWithResponseContentValidation(int fileSize, @TempDir Path tempDir) + throws IOException { + File file = createRandomFile(tempDir, fileSize); + File outFile = tempDir.resolve("download.bin").toFile(); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.uploadFromFile(file.toPath().toString(), true); + + Files.deleteIfExists(outFile.toPath()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong((long) BLOCK_SIZE); + BlobDownloadToFileOptions options + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + Response response = blobClient.downloadToFileWithResponse(options, null, Context.NONE); + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + assertNotNull(response.getValue()); + assertTrue(compareFiles(file, outFile, 0, fileSize)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Default behavior: when no algorithm is specified, default is NONE (no validation). + */ + @Test + public void downloadStreamDefaultAlgorithmIsNone() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + blobClient.downloadStreamWithResponse(outputStream, new BlobDownloadStreamOptions(), null, Context.NONE); + + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * AUTO on downloadStream resolves to CRC64 behavior. + */ + @Test + public void downloadStreamWithAuto() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options + = new BlobDownloadStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO); + BlobDownloadResponse response + = blobClient.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with NONE: no validation triggered. + */ + @Test + public void downloadContentWithNone() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + byte[] result + = blobClient + .downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE), + null, Context.NONE) + .getValue() + .toBytes(); + + TestUtils.assertArraysEqual(data, result); + assertFalse(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * downloadContentWithResponse with AUTO resolves to CRC64 behavior. + */ + @Test + public void downloadContentWithAuto() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + BlobDownloadContentResponse response = blobClient.downloadContentWithResponse( + new BlobDownloadContentOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.AUTO), null, + Context.NONE); + byte[] result = response.getValue().toBytes(); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, result); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Interrupt with proper rewind to segment boundary; verifies retry range headers. + */ + @Test + public void interruptAndVerifyProperRewind() { + final int segmentSize = Constants.KB; + byte[] data = getRandomByteArray(2 * segmentSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + + blobClient.upload(BinaryData.fromBytes(data)); + + int interruptPos = segmentSize + (2 * (segmentSize / 4)) + 10; + MockPartialResponsePolicy mockPolicy = new MockPartialResponsePolicy(1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + BlobClient downloadClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(5); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options = new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobDownloadResponse response + = downloadClient.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + TestUtils.assertArraysEqual(data, outputStream.toByteArray()); + assertEquals(0, mockPolicy.getTriesRemaining(), "Expected the configured interruption to be consumed"); + assertTrue(mockPolicy.getRangeHeaders().size() >= 2, + "Expected at least the initial request and one retry with a range header"); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * Proper decode across retries (single and multiple interrupts). + */ + @ParameterizedTest + @ValueSource(booleans = { false, true }) + public void interruptAndVerifyProperDecode(boolean multipleInterrupts) { + final int segmentSize = 128 * Constants.KB; + final int dataSize = 4 * Constants.KB; + byte[] data = getRandomByteArray(dataSize); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + HttpHeaders recordedResponseHeaders = new HttpHeaders(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + + blobClient.upload(BinaryData.fromBytes(data)); + + int interruptPos = segmentSize + (3 * (8 * Constants.KB)) + 10; + MockPartialResponsePolicy mockPolicy + = new MockPartialResponsePolicy(multipleInterrupts ? 2 : 1, interruptPos, blobClient.getBlobUrl()); + HttpPipelinePolicy sniffPolicy = getRequestAndResponseHeaderSniffer(blobClient.getBlobUrl(), + recordedRequestHeaders, recordedResponseHeaders); + + BlobClient downloadClient = getBlobClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + blobClient.getBlobUrl(), sniffPolicy, mockPolicy); + DownloadRetryOptions retryOptions = new DownloadRetryOptions().setMaxRetryRequests(10); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + BlobDownloadStreamOptions options = new BlobDownloadStreamOptions().setDownloadRetryOptions(retryOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobDownloadResponse response + = downloadClient.downloadStreamWithResponse(outputStream, options, null, Context.NONE); + + assertTrue(hasStructuredMessageDownloadResponseHeaders(response.getHeaders())); + byte[] result = outputStream.toByteArray(); + assertEquals(dataSize, result.length, "Decoded data should have exactly " + dataSize + " bytes"); + TestUtils.assertArraysEqual(data, result); + assertTrue(hasStructuredMessageDownloadResponseHeaders(recordedResponseHeaders)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + // Only run this test in live mode as BlobOutputStream dynamically assigns blocks + @LiveOnly + @Test + public void openInputStreamContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + blobClient.upload(BinaryData.fromBytes(data)); + + BlobInputStreamOptions options + = new BlobInputStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobInputStream inputStream = blobClient.openInputStream(options, Context.NONE); + + TestUtils.assertArraysEqual(data, convertInputStreamToByteArray(inputStream)); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + // Only run this test in live mode as BlobOutputStream dynamically assigns blocks + @LiveOnly + @Test + public void openInputStreamRangeContentValidation() { + byte[] data = getRandomByteArray(TEN_MB); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + + int start = Constants.MB; + int count = 3 * Constants.MB + 257; + + blobClient.upload(BinaryData.fromBytes(data)); + + BlobInputStreamOptions options = new BlobInputStreamOptions().setRange(new BlobRange(start, (long) count)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64) + .setBlockSize(Constants.MB); + BlobInputStream inputStream = blobClient.openInputStream(options, Context.NONE); + + byte[] downloadedRange = convertInputStreamToByteArray(inputStream); + assertEquals(count, downloadedRange.length); + TestUtils.assertArraysEqual(data, start, downloadedRange, 0, count); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + /** + * openSeekableByteChannelRead with CRC64 content validation. + */ + @ParameterizedTest + @MethodSource("channelReadDataSupplier") + public void openSeekableByteChannelReadContentValidation(Integer streamBufferSize, int copyBufferSize, + int dataLength) throws IOException { + byte[] data = getRandomByteArray(dataLength); + List recordedRequestHeaders = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recordedRequestHeaders); + + blobClient.upload(BinaryData.fromBytes(data)); + + // when: "Channel initialized" + BlobSeekableByteChannelReadOptions options + = new BlobSeekableByteChannelReadOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64) + .setReadSizeInBytes(streamBufferSize); + BlobSeekableByteChannelReadResult result = blobClient.openSeekableByteChannelRead(options, Context.NONE); + SeekableByteChannel channel = result.getChannel(); + + // then: "Channel initialized to position zero" + assertEquals(0, channel.position()); + assertNotNull(result.getProperties()); + assertEquals(data.length, result.getProperties().getBlobSize()); + + // when: "read from channel" + ByteArrayOutputStream downloadedData = new ByteArrayOutputStream(); + int copied = copy(channel, downloadedData, copyBufferSize); + + // then: "channel position updated accordingly" + assertEquals(dataLength, copied); + assertEquals(dataLength, channel.position()); + + // and: "expected data downloaded" + TestUtils.assertArraysEqual(data, downloadedData.toByteArray()); + assertTrue(hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, false)); + } + + @Test + public void verifyProgressListenerIsCompatibleWithContentValidation(@TempDir Path tempDir) throws IOException { + byte[] data = getRandomByteArray(10 * Constants.MB); + + BlobClient client = cc.getBlobClient(generateBlobName()); + client.upload(BinaryData.fromBytes(data)); + + MockProgressListener mockListenerWithContentVal = new MockProgressListener(); + MockProgressListener mockListenerWithoutContentVal = new MockProgressListener(); + + ParallelTransferOptions parallelOptionsWithContentVal + = new ParallelTransferOptions().setProgressListener(mockListenerWithContentVal); + ParallelTransferOptions parallelOptionsWithoutContentVal + = new ParallelTransferOptions().setProgressListener(mockListenerWithoutContentVal); + + File fileWithContentVal = createRandomFile(tempDir, 10 * Constants.MB); + File outFileWithContentVal = tempDir.resolve("withcontentval.bin").toFile(); + File fileWithoutContentVal = createRandomFile(tempDir, 10 * Constants.MB); + File outFileWithoutContentVal = tempDir.resolve("withoutcontentval.bin").toFile(); + + Files.deleteIfExists(outFileWithContentVal.toPath()); + Files.deleteIfExists(outFileWithoutContentVal.toPath()); + + BlobDownloadToFileOptions optionsWithContentVal + = new BlobDownloadToFileOptions(outFileWithContentVal.getAbsolutePath()) + .setParallelTransferOptions(parallelOptionsWithContentVal) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + BlobDownloadToFileOptions optionsWithoutContentVal + = new BlobDownloadToFileOptions(outFileWithoutContentVal.getAbsolutePath()) + .setParallelTransferOptions(parallelOptionsWithoutContentVal); + + client.downloadToFileWithResponse(optionsWithContentVal, null, Context.NONE); + client.downloadToFileWithResponse(optionsWithoutContentVal, null, Context.NONE); + + long expectedBytes = data.length; + assertEquals(expectedBytes, mockListenerWithContentVal.getReportedByteCount()); + assertEquals(expectedBytes, mockListenerWithoutContentVal.getReportedByteCount()); + } + + private static final class MockProgressListener implements ProgressListener { + private final AtomicLong reportedByteCount = new AtomicLong(0L); + + @Override + public void handleProgress(long bytesTransferred) { + this.reportedByteCount.updateAndGet(current -> Math.max(current, bytesTransferred)); + } + + long getReportedByteCount() { + return this.reportedByteCount.get(); + } + } + + // ---------- Fuzzy parallel download (deterministic grids) ---------- + + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadReplayableCases") + public void fuzzyParallelDownloadReplayableRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("replayable", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload > blockSize with tiny totals; many small range GETs not replayable under the proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSmallMultiPartCases") + public void fuzzyParallelDownloadSmallMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("smallMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // sub-4 MiB chunked range GETs not replayable under the proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadSubFourMiBCases") + public void fuzzyParallelDownloadSubFourMiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("subFourMiB", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // 4 MiB boundary tuples that fan out into chunked range GETs. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadFourMiBBoundaryCases") + public void fuzzyParallelDownloadFourMiBBoundaryRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("fourMiBBoundary", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload > blockSize for every tuple; chunked range GETs across many requests. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadMediumMultiPartCases") + public void fuzzyParallelDownloadMediumMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("mediumMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // payload >> blockSize; ~96-320 MiB downloads. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadLargeMultiPartCases") + public void fuzzyParallelDownloadLargeMultiPartRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("largeMultiPart", payloadBytes, blockSizeBytes, maxConcurrency); + } + + @LiveOnly // ~1 GiB single case; far too large for the test proxy. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelDownloadOneGiBCases") + public void fuzzyParallelDownloadOneGiBRoundTrip(int payloadBytes, long blockSizeBytes, int maxConcurrency) + throws IOException { + assertParallelDownloadFuzzyRoundTrip("oneGiB", payloadBytes, blockSizeBytes, maxConcurrency); + } + + /** + * Live-only random-size parallel download fuzzy round-trip. Each run draws a per-run payload size in + * {@code (256 MiB, 500 MiB]} (matches the encoder fuzzy upload range) and exercises both CRC64 and AUTO + * content-validation algorithms so the structured-message decoder is tested against payloads whose total size + * varies per run in addition to the random byte contents that the deterministic grids already exercise. Kept + * separate from the parameterized {@link #fuzzyParallelDownloadLargeMultiPartRoundTrip(int, long, int)} so the + * deterministic per-grid round-trips and the randomized round-trip don't share work or cost. + */ + @LiveOnly + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void fuzzyParallelDownloadLiveRandomRoundTrip(ContentValidationAlgorithm algorithm) throws IOException { + int sizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_DOWNLOAD_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + assertParallelDownloadFuzzyRoundTrip("liveRandom", sizeBytes, 8L * Constants.MB, 8, algorithm); + } + + private void assertParallelDownloadFuzzyRoundTrip(String caseKind, int payloadBytes, long blockSizeBytes, + int maxConcurrency) throws IOException { + assertParallelDownloadFuzzyRoundTrip(caseKind, payloadBytes, blockSizeBytes, maxConcurrency, + ContentValidationAlgorithm.CRC64); + } + + private void assertParallelDownloadFuzzyRoundTrip(String caseKind, int payloadBytes, long blockSizeBytes, + int maxConcurrency, ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + ParallelTransferOptions parallelOptions + = new ParallelTransferOptions().setBlockSizeLong(blockSizeBytes).setMaxConcurrency(maxConcurrency); + + String assertionMessage = "Fuzzy parallel download [" + caseKind + "] payloadBytes=" + payloadBytes + + ", blockSize=" + blockSizeBytes + ", maxConcurrency=" + maxConcurrency + ", algorithm=" + algorithm; + + if (payloadBytes >= FUZZY_PARALLEL_DOWNLOAD_FILE_ROUND_TRIP_THRESHOLD_BYTES) { + File sourceFile = getRandomFile(payloadBytes); + sourceFile.deleteOnExit(); + createdFiles.add(sourceFile); + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl", ".bin").toFile(); + outFile.deleteOnExit(); + createdFiles.add(outFile); + Files.deleteIfExists(outFile.toPath()); + + BlobUploadFromFileOptions uploadOptions + = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()).setParallelTransferOptions( + new com.azure.storage.blob.models.ParallelTransferOptions().setBlockSizeLong(blockSizeBytes) + .setMaxConcurrency(maxConcurrency)); + assertNotNull(client.uploadFromFileWithResponse(uploadOptions, null, Context.NONE).getValue().getETag(), + assertionMessage); + + BlobDownloadToFileOptions downloadOptions + = new BlobDownloadToFileOptions(outFile.toPath().toString()).setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(algorithm); + assertNotNull(client.downloadToFileWithResponse(downloadOptions, null, Context.NONE).getValue(), + assertionMessage); + + assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage); + } else { + byte[] randomData = getRandomByteArray(payloadBytes); + client.upload(BinaryData.fromBytes(randomData), true); + + if (payloadBytes > blockSizeBytes) { + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl-mp", ".bin").toFile(); + outFile.deleteOnExit(); + createdFiles.add(outFile); + Files.deleteIfExists(outFile.toPath()); + + BlobDownloadToFileOptions downloadOptions = new BlobDownloadToFileOptions(outFile.toPath().toString()) + .setParallelTransferOptions(parallelOptions) + .setContentValidationAlgorithm(algorithm); + assertNotNull(client.downloadToFileWithResponse(downloadOptions, null, Context.NONE).getValue(), + assertionMessage); + + byte[] downloaded = readAllBytesFromFile(outFile); + assertArrayEquals(randomData, downloaded, assertionMessage); + } else { + BlobDownloadContentOptions downloadOptions + = new BlobDownloadContentOptions().setContentValidationAlgorithm(algorithm); + byte[] downloaded + = client.downloadContentWithResponse(downloadOptions, null, Context.NONE).getValue().toBytes(); + assertArrayEquals(randomData, downloaded, assertionMessage); + } + } + assertTrue(hasStructuredMessageDownloadRequestHeaders(recorded, false), assertionMessage); + } + + private static byte[] readAllBytesFromFile(File file) throws IOException { + try (InputStream is = Files.newInputStream(file.toPath())) { + byte[] buffer = new byte[(int) file.length()]; + int offset = 0; + int read; + while (offset < buffer.length && (read = is.read(buffer, offset, buffer.length - offset)) != -1) { + offset += read; + } + return buffer; + } + } + + static Stream channelReadDataSupplier() { + return Stream.of(Arguments.of(50, 40, Constants.KB), Arguments.of(Constants.KB + 50, 40, Constants.KB), + Arguments.of(null, Constants.MB, TEN_MB)); + } + +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationUploadTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationUploadTests.java new file mode 100644 index 000000000000..f038d0dad772 --- /dev/null +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobContentValidationUploadTests.java @@ -0,0 +1,1406 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob; + +import com.azure.core.http.HttpHeaders; +import com.azure.core.util.BinaryData; +import com.azure.core.util.Context; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.PageRange; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.AppendBlobAppendBlockOptions; +import com.azure.storage.blob.options.BlobParallelUploadOptions; +import com.azure.storage.blob.options.AppendBlobOutputStreamOptions; +import com.azure.storage.blob.options.BlobUploadFromFileOptions; +import com.azure.storage.blob.options.BlockBlobOutputStreamOptions; +import com.azure.storage.blob.options.BlockBlobSeekableByteChannelWriteOptions; +import com.azure.storage.blob.options.BlockBlobSimpleUploadOptions; +import com.azure.storage.blob.options.BlockBlobStageBlockOptions; +import com.azure.storage.blob.options.PageBlobOutputStreamOptions; +import com.azure.storage.blob.options.PageBlobUploadPagesOptions; +import com.azure.storage.blob.specialized.AppendBlobClient; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.blob.specialized.BlockBlobClient; +import com.azure.storage.blob.specialized.PageBlobClient; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.implementation.contentvalidation.ContentValidationModeResolver; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.test.shared.extensions.LiveOnly; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.ByteArrayInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests content validation (CRC64 / structured message) for upload operations using sync clients. + * Upload types that have no async counterpart (OutputStream, SeekableByteChannel) are tested only here. + * Async counterparts of the same operations are in {@link BlobContentValidationAsyncUploadTests}. + */ +public class BlobContentValidationUploadTests extends BlobTestBase { + private static final int TEN_MB = 10 * Constants.MB; + /* single-shot uploads with length < 4MB use CRC64 header; >= 4MB use structured message. */ + private static final int UNDER_4MB = 2 * Constants.MB; + + /** + * Live-only random payload band (256–500 MiB, inclusive upper bound via {@code randomLongFromNamer}+1) for + * {@code uploadWithResponse}, {@code uploadFromFileWithResponse}, and single-block {@code stageBlock}. + */ + private static final long LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE = 256L * Constants.MB; + private static final long LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE = 500L * Constants.MB; + + /** + * Live-only random payload band for sequential append-block puts only. + * {@code Flux.concatMap} issues one append REST call per chunk in + * order (not parallel staging); use a smaller band than {@link #LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE}. + */ + private static final long LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MIN_BYTES_EXCLUSIVE = 32L * Constants.MB; + private static final long LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MAX_BYTES_INCLUSIVE = 64L * Constants.MB; + + private static final String MD5_AND_CRC64_EXCLUSIVE_MESSAGE + = "Only one form of transactional content validation may be used."; + + // =========================================================================================== + // BlobClient.uploadWithResponse + // =========================================================================================== + + /** + * Single-shot upload under 4MB: content validation uses CRC64 header only (no structured message). + */ + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + /** + * Single-shot upload >= 4MB: content validation uses structured message. + */ + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + /** + * Multi-shot (chunked) upload; content validation uses structured message on each stage block. + */ + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + long blockSize = 2 * (long) Constants.MB; + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void uploadWithoutContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + /** + * Blob parallel upload rejects using both computeMd5 (SDK-computed MD5) and CRC64 (transfer validation checksum algorithm) at once. + */ + @Test + public void uploadWithComputeMd5AndCrc64Throws() { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setComputeMd5(true) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> client.uploadWithResponse(options, null, Context.NONE)); + assertTrue(ex.getMessage().contains(MD5_AND_CRC64_EXCLUSIVE_MESSAGE)); + } + + // =========================================================================================== + // BlockBlobClient.uploadWithResponse (BlockBlobSimpleUpload / Put Blob) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void blockBlobSimpleUploadWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // BlockBlobClient.stageBlockWithResponse (Put Block) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + client.stageBlockWithResponse(options, null, Context.NONE); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + client.stageBlockWithResponse(options, null, Context.NONE); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void stageBlockWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options = new BlockBlobStageBlockOptions(getBlockID(), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + client.stageBlockWithResponse(options, null, Context.NONE); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // AppendBlobClient.appendBlockWithResponse (Append Block) tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions(data, UNDER_4MB).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.appendBlockWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions(data, TEN_MB).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.appendBlockWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void appendBlockWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + AppendBlobAppendBlockOptions options = new AppendBlobAppendBlockOptions(data, TEN_MB) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.appendBlockWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // PageBlobClient.uploadPagesWithResponse (Put Page) tests + // =========================================================================================== + + private static final int PAGE_BYTES = PageBlobClient.PAGE_BYTES; + private static final int UNDER_4MB_PAGE_ALIGNED = (UNDER_4MB / PAGE_BYTES) * PAGE_BYTES; + private static final int FOUR_MB_PAGE_ALIGNED = (4 * Constants.MB / PAGE_BYTES) * PAGE_BYTES; + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithCrc64Header(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(UNDER_4MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + InputStream data = new ByteArrayInputStream(randomData); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadPagesWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + InputStream data = new ByteArrayInputStream(randomData); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadPagesWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void uploadPagesWithNoContentValidation() { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + InputStream data = new ByteArrayInputStream(randomData); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.uploadPagesWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // BlobClient.uploadFromFileWithResponse tests + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithCrc64Header(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(UNDER_4MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadFromFileWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithStructuredMessage(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadFromFileWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + long blockSize = 2 * (long) Constants.MB; + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadFromFileWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void uploadFromFileWithNoContentValidation() throws IOException { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + File tempFile = getRandomFile(TEN_MB); + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE); + + assertNotNull(client.uploadFromFileWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // Sync BlobOutputStream tests (getBlobOutputStream) + // =========================================================================================== + + // --- AppendBlobClient.getBlobOutputStream --- + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlobOutputStreamWithCrc64Header(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + + try (BlobOutputStream os = client + .getBlobOutputStream(new AppendBlobOutputStreamOptions().setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlobOutputStreamWithStructuredMessage(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (BlobOutputStream os = client + .getBlobOutputStream(new AppendBlobOutputStreamOptions().setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void appendBlobOutputStreamWithNoContentValidation() throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (BlobOutputStream os = client.getBlobOutputStream( + new AppendBlobOutputStreamOptions().setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) { + os.write(randomData); + } + + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // --- BlockBlobClient.getBlobOutputStream --- + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobOutputStreamWithCrc64Header(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + + try (BlobOutputStream os = client.getBlobOutputStream(new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobOutputStreamWithStructuredMessage(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (BlobOutputStream os = client.getBlobOutputStream(new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobOutputStreamChunkedWithStructuredMessage(ContentValidationAlgorithm algorithm) + throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + long blockSize = 2 * (long) Constants.MB; + + try (BlobOutputStream os = client.getBlobOutputStream(new BlockBlobOutputStreamOptions() + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void blockBlobOutputStreamWithNoContentValidation() throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (BlobOutputStream os = client.getBlobOutputStream(new BlockBlobOutputStreamOptions() + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) { + os.write(randomData); + } + + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // --- PageBlobClient.getBlobOutputStream --- + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void pageBlobOutputStreamWithCrc64Header(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(UNDER_4MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + + try (BlobOutputStream os = client.getBlobOutputStream( + new PageBlobOutputStreamOptions(new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void pageBlobOutputStreamWithStructuredMessage(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + + try (BlobOutputStream os = client.getBlobOutputStream( + new PageBlobOutputStreamOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1)) + .setContentValidationAlgorithm(algorithm))) { + os.write(randomData); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @Test + public void pageBlobOutputStreamWithNoContentValidation() throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + + try (BlobOutputStream os = client.getBlobOutputStream( + new PageBlobOutputStreamOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) { + os.write(randomData); + } + + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // BlockBlobClient.openSeekableByteChannelWrite tests + // =========================================================================================== + + @LiveOnly // Seekable channel staging uses Put Block with random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void seekableByteChannelWriteWithCrc64Header(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + + try (java.nio.channels.SeekableByteChannel channel = client.openSeekableByteChannelWrite( + new BlockBlobSeekableByteChannelWriteOptions(BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE) + .setContentValidationAlgorithm(algorithm))) { + channel.write(ByteBuffer.wrap(randomData)); + } + + assertTrue(hasOnlyCrc64Headers(recorded)); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void seekableByteChannelWriteWithStructuredMessage(ContentValidationAlgorithm algorithm) throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (java.nio.channels.SeekableByteChannel channel = client.openSeekableByteChannelWrite( + new BlockBlobSeekableByteChannelWriteOptions(BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE) + .setContentValidationAlgorithm(algorithm))) { + channel.write(ByteBuffer.wrap(randomData)); + } + + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @Test + public void seekableByteChannelWriteWithNoContentValidation() throws Exception { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + + try (java.nio.channels.SeekableByteChannel channel = client.openSeekableByteChannelWrite( + new BlockBlobSeekableByteChannelWriteOptions(BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE) + .setContentValidationAlgorithm(ContentValidationAlgorithm.NONE))) { + channel.write(ByteBuffer.wrap(randomData)); + } + + assertTrue(hasNoContentValidationHeaders(recorded)); + } + + // =========================================================================================== + // Exact 4MB boundary tests + // + // The cutoff between CRC64 header and structured message is exactly 4MB. + // Uploads of exactly 4MB should use structured message (>= threshold), not CRC64 header. + // =========================================================================================== + + private static final int EXACTLY_4MB = 4 * Constants.MB; + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient client = createBlobClientWithRequestSniffer(recorded); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) EXACTLY_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobSimpleUploadAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(algorithm); + + assertNotNull(client.uploadWithResponse(options, null, Context.NONE).getValue().getETag()); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockAtExactly4MBUsesStructuredMessage(ContentValidationAlgorithm algorithm) { + List recorded = new CopyOnWriteArrayList<>(); + BlobClient blobClient = createBlobClientWithRequestSniffer(recorded); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(EXACTLY_4MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobStageBlockOptions options + = new BlockBlobStageBlockOptions(getBlockID(), data).setContentValidationAlgorithm(algorithm); + + client.stageBlockWithResponse(options, null, Context.NONE); + assertTrue(hasOnlyStructuredMessageHeaders(recorded)); + } + + // =========================================================================================== + // Progress reporting (transfer validation must be NONE/null when a progress listener is set) + // =========================================================================================== + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadWithProgressAndNonNoneContentValidationThrows(ContentValidationAlgorithm algorithm) { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data).setParallelTransferOptions( + new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB).setProgressListener(l -> { + })).setRequestConditions(new BlobRequestConditions()).setContentValidationAlgorithm(algorithm); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> client.uploadWithResponse(options, null, Context.NONE)); + assertEquals(ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, + ex.getMessage()); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadFromFileWithProgressAndNonNoneContentValidationThrows(ContentValidationAlgorithm algorithm) + throws IOException { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + File tempFile = getRandomFile(TEN_MB); + + BlobUploadFromFileOptions options + = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()).setParallelTransferOptions( + new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB).setProgressListener(l -> { + })).setContentValidationAlgorithm(algorithm); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> client.uploadFromFileWithResponse(options, null, Context.NONE)); + assertEquals(ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, + ex.getMessage()); + } + + // =========================================================================================== + // Data integrity round-trip tests (upload with content validation, download, verify) + // + // Previous tests verify that the correct headers are sent. These tests verify end-to-end + // integrity: the data uploaded with CRC64/structured message can be downloaded and matches + // the original byte-for-byte. + // =========================================================================================== + + @Test + public void uploadWithCrc64RoundTripDataIntegrity() { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(UNDER_4MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) UNDER_4MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadWithResponse(options, null, Context.NONE); + + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded data (CRC64 header path)"); + } + + @Test + public void uploadWithStructuredMessageRoundTripDataIntegrity() { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadWithResponse(options, null, Context.NONE); + + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded data (structured message path)"); + } + + @LiveOnly // Put Block URLs include random block IDs; not replayable with the test proxy. + @Test + public void uploadChunkedWithStructuredMessageRoundTripDataIntegrity() { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + long blockSize = 2 * (long) Constants.MB; + + BlobParallelUploadOptions options = new BlobParallelUploadOptions(data) + .setParallelTransferOptions( + new ParallelTransferOptions().setBlockSizeLong(blockSize).setMaxSingleUploadSizeLong(blockSize)) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadWithResponse(options, null, Context.NONE); + + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, + "Downloaded data must match uploaded data (chunked structured message path)"); + } + + @Test + public void blockBlobSimpleUploadRoundTripDataIntegrity() { + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + byte[] randomData = getRandomByteArray(TEN_MB); + BinaryData data = BinaryData.fromBytes(randomData); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(data).setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadWithResponse(options, null, Context.NONE); + + byte[] downloaded = blobClient.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, + "Downloaded data must match uploaded data (block blob simple upload)"); + } + + @Test + public void appendBlockRoundTripDataIntegrity() { + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + AppendBlobClient client = blobClient.getAppendBlobClient(); + client.create(); + + byte[] randomData = getRandomByteArray(TEN_MB); + InputStream data = new ByteArrayInputStream(randomData); + + AppendBlobAppendBlockOptions options = new AppendBlobAppendBlockOptions(data, TEN_MB) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.appendBlockWithResponse(options, null, Context.NONE); + + byte[] downloaded = blobClient.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded data (append block)"); + } + + @Test + public void uploadPagesRoundTripDataIntegrity() { + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + PageBlobClient client = blobClient.getPageBlobClient(); + client.create(FOUR_MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(FOUR_MB_PAGE_ALIGNED); + InputStream data = new ByteArrayInputStream(randomData); + + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(FOUR_MB_PAGE_ALIGNED - 1), data) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadPagesWithResponse(options, null, Context.NONE); + + byte[] downloaded = blobClient.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded data (page blob upload pages)"); + } + + @Test + public void uploadFromFileRoundTripDataIntegrity() throws IOException { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + byte[] randomData = getRandomByteArray(TEN_MB); + File tempFile = File.createTempFile("blob-cv-roundtrip", ".bin"); + tempFile.deleteOnExit(); + try (java.io.FileOutputStream fos = new java.io.FileOutputStream(tempFile)) { + fos.write(randomData); + } + + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(tempFile.getAbsolutePath()) + .setParallelTransferOptions(new ParallelTransferOptions().setMaxSingleUploadSizeLong((long) TEN_MB)) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + + client.uploadFromFileWithResponse(options, null, Context.NONE); + + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, "Downloaded data must match uploaded file data"); + } + + // =========================================================================================== + // Live-only random payload bands. + // - 256–500 MiB: parallelUpload, uploadFromFile, stageBlock, block BlobOutputStream, SeekableByteChannel — + // parallel staging or single giant block / default transfer options as applicable. + // - 32–64 MiB (sequential append blocks only): appendBlobAppendBlocksLiveRandom… — one append REST call per + // chunk in order. + // =========================================================================================== + + @LiveOnly // This test is too large for the test proxy. + @Test + public void parallelUploadLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-par-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + try (InputStream data = new FileInputStream(sourceFile)) { + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options, null, Context.NONE); + } + client.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void stageBlockLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + BlockBlobClient client = blobClient.getBlockBlobClient(); + String blockId = getBlockID(); + + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-stage-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + BinaryData binaryData = BinaryData.fromFile(sourceFile.toPath()); + BlockBlobStageBlockOptions stageOptions = new BlockBlobStageBlockOptions(blockId, binaryData) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.stageBlockWithResponse(stageOptions, null, Context.NONE); + client.commitBlockList(Collections.singletonList(blockId)); + blobClient.downloadToFile(outFile.getPath(), true); + + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void appendBlobAppendBlocksLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes + = (int) randomLongFromNamer(LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_SEQUENTIAL_APPEND_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + AppendBlobClient client = blobClient.getAppendBlobClient(); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-append-blocks-dl", ".bin").toFile(); + outFile.deleteOnExit(); + client.create(); + int maxAppendBlockBytes = client.getMaxAppendBlockBytes(); + try { + try (FileInputStream fis = new FileInputStream(sourceFile)) { + long remaining = chosenPayloadSizeBytes; + byte[] buf = new byte[maxAppendBlockBytes]; + while (remaining > 0) { + int chunk = (int) Math.min(maxAppendBlockBytes, remaining); + int totalRead = 0; + while (totalRead < chunk) { + int n = fis.read(buf, totalRead, chunk - totalRead); + if (n == -1) { + throw new EOFException( + prefix + "Unexpected EOF after " + totalRead + " bytes of chunk."); + } + totalRead += n; + } + ByteArrayInputStream chunkStream = new ByteArrayInputStream(buf, 0, chunk); + AppendBlobAppendBlockOptions appendOptions + = new AppendBlobAppendBlockOptions(chunkStream, chunk) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.appendBlockWithResponse(appendOptions, null, Context.NONE); + remaining -= chunk; + } + } + blobClient.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void uploadFromFileLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-uploadfromfile-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + BlobUploadFromFileOptions options = new BlobUploadFromFileOptions(sourceFile.getAbsolutePath()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadFromFileWithResponse(options, null, Context.NONE); + client.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void blockBlobOutputStreamLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + BlockBlobClient client = blobClient.getBlockBlobClient(); + + // Explicit parallel-transfer tuning (8 MiB blocks × concurrency 8) on the stream ingest path. + ParallelTransferOptions parallelTransferOptions + = new ParallelTransferOptions().setBlockSizeLong(8L * Constants.MB) + .setMaxSingleUploadSizeLong(8L * Constants.MB) + .setMaxConcurrency(8); + + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-block-os-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + try (BlobOutputStream outputStream = client.getBlobOutputStream( + new BlockBlobOutputStreamOptions().setParallelTransferOptions(parallelTransferOptions) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))) { + Files.copy(sourceFile.toPath(), outputStream); + } + + blobClient.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + @LiveOnly // This test is too large for the test proxy. + @Test + public void seekableByteChannelWriteLiveRandomRoundTripDataIntegrity() throws Exception { + int chosenPayloadSizeBytes = (int) randomLongFromNamer(LIVE_RANDOM_PARALLEL_PAYLOAD_MIN_BYTES_EXCLUSIVE + 1, + LIVE_RANDOM_PARALLEL_PAYLOAD_MAX_BYTES_INCLUSIVE + 1); + try { + String prefix = "chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". "; + BlobClient blobClient = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + BlockBlobClient client = blobClient.getBlockBlobClient(); + File sourceFile = getRandomFile(chosenPayloadSizeBytes); + File outFile = Files.createTempFile("blob-cv-live-sbc-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + try (java.nio.channels.SeekableByteChannel seekableByteChannel + = client.openSeekableByteChannelWrite(new BlockBlobSeekableByteChannelWriteOptions( + BlockBlobSeekableByteChannelWriteOptions.WriteMode.OVERWRITE) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64))) { + Files.copy(sourceFile.toPath(), Channels.newOutputStream(seekableByteChannel)); + } + + blobClient.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, chosenPayloadSizeBytes), prefix); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } catch (Exception e) { + throw new Exception("chosenPayloadSizeBytes=" + chosenPayloadSizeBytes + ". " + e.getMessage(), e); + } + } + + // ---------- Deterministic parallel upload ---------- + + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadPutBlobReplayableCases") + public void fuzzyParallelUploadPutBlobReplayableRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTrip("putBlobReplay", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Staging-only cases: Put Block URLs include random IDs + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadSmallPayloadStagingCases") + public void fuzzyParallelUploadSmallPayloadRoundTripRequiresLiveStaging(int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + assertParallelUploadFuzzyRoundTrip("smallPayloadStaging", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // payload > segment for every tuple; always staging/Put Block. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadSub4MiBCases") + public void fuzzyParallelUploadSubFourMiBBlobRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTrip("subFourMiB", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // Staging-only cases. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadFourMiBBoundaryStagingCases") + public void fuzzyParallelUploadFourMiBBoundaryRoundTripRequiresLiveStaging(int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + assertParallelUploadFuzzyRoundTrip("fourMiBBoundaryStaging", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // payload > segment throughout; chunked upload. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadMediumMultiPartCases") + public void fuzzyParallelUploadMediumMultiPartRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTrip("mediumMultiPart", payloadBytes, segmentBytes, maxConcurrency); + } + + @LiveOnly // payload >> segment throughout; chunked upload / large payloads. + @ParameterizedTest + @MethodSource("com.azure.storage.blob.BlobTestBase#fuzzyParallelUploadLargeMultiPartCases") + public void fuzzyParallelUploadLargeMultiPartRoundTrip(int payloadBytes, long segmentBytes, int maxConcurrency) + throws IOException { + assertParallelUploadFuzzyRoundTrip("largeMultiPart", payloadBytes, segmentBytes, maxConcurrency); + } + + private void assertParallelUploadFuzzyRoundTrip(String caseKind, int payloadBytes, long segmentBytes, + int maxConcurrency) throws IOException { + BlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()); + + ParallelTransferOptions parallelOptions = new ParallelTransferOptions().setBlockSizeLong(segmentBytes) + .setMaxSingleUploadSizeLong(segmentBytes) + .setMaxConcurrency(maxConcurrency); + + String assertionMessage = "Fuzzy parallel upload [" + caseKind + "] payloadBytes=" + payloadBytes + + ", segmentBytes=" + segmentBytes + ", maxConcurrency=" + maxConcurrency; + + // above this threshold the fuzzy parallel upload helpers stream from a temp source file + // to avoid materializing the full payload twice in heap. + if (payloadBytes >= 96 * Constants.MB) { + File sourceFile = getRandomFile(payloadBytes); + File outFile = Files.createTempFile("blob-cv-fuzzy-parallel-dl", ".bin").toFile(); + outFile.deleteOnExit(); + try { + try (InputStream data = new FileInputStream(sourceFile)) { + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setParallelTransferOptions(parallelOptions) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options, null, Context.NONE); + } + client.downloadToFile(outFile.getPath(), true); + assertTrue(compareFiles(sourceFile, outFile, 0, payloadBytes), assertionMessage); + } finally { + if (!sourceFile.delete()) { + sourceFile.deleteOnExit(); + } + if (!outFile.delete()) { + outFile.deleteOnExit(); + } + } + } else { + byte[] randomData = getRandomByteArray(payloadBytes); + InputStream data = new ByteArrayInputStream(randomData); + BlobParallelUploadOptions options + = new BlobParallelUploadOptions(data).setParallelTransferOptions(parallelOptions) + .setRequestConditions(new BlobRequestConditions()) + .setContentValidationAlgorithm(ContentValidationAlgorithm.CRC64); + client.uploadWithResponse(options, null, Context.NONE); + byte[] downloaded = client.downloadContent().toBytes(); + assertArrayEquals(randomData, downloaded, assertionMessage); + } + } + + // =========================================================================================== + // Customer Provided MD5 Byte[] with Content Validation Algorithm + // =========================================================================================== + + private static final byte[] DEFAULT_MD5 = createDefaultMd5(); + private static final String MESSAGE = "Both x-ms-content-crc64 header and Content-MD5 header are present."; + + private static byte[] createDefaultMd5() { + try { + return Base64.getEncoder().encode(MessageDigest.getInstance("MD5").digest(DATA.getDefaultBytes())); + } catch (NoSuchAlgorithmException ex) { + throw LOGGER.logExceptionAsError(new RuntimeException("MD5 algorithm unavailable.", ex)); + } + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void blockBlobUploadWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + BlockBlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getBlockBlobClient(); + + BlockBlobSimpleUploadOptions options + = new BlockBlobSimpleUploadOptions(DATA.getDefaultBinaryData()).setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + BlobStorageException e + = assertThrows(BlobStorageException.class, () -> client.uploadWithResponse(options, null, Context.NONE)); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void stageBlockWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + BlockBlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getBlockBlobClient(); + + BlockBlobStageBlockOptions options = new BlockBlobStageBlockOptions(getBlockID(), DATA.getDefaultBinaryData()) + .setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + BlobStorageException e = assertThrows(BlobStorageException.class, + () -> client.stageBlockWithResponse(options, null, Context.NONE)); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void appendBlockWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) { + AppendBlobClient client + = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getAppendBlobClient(); + client.create(); + + byte[] randomData = DATA.getDefaultBytes(); + AppendBlobAppendBlockOptions options + = new AppendBlobAppendBlockOptions(new ByteArrayInputStream(randomData), randomData.length) + .setContentValidationAlgorithm(algorithm) + .setContentMd5(DEFAULT_MD5); + + BlobStorageException e = assertThrows(BlobStorageException.class, + () -> client.appendBlockWithResponse(options, null, Context.NONE)); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + } + + @ParameterizedTest + @EnumSource(value = ContentValidationAlgorithm.class, names = { "CRC64", "AUTO" }) + public void uploadPagesWithCustomerProvidedMd5AndCrc64Header(ContentValidationAlgorithm algorithm) + throws NoSuchAlgorithmException { + PageBlobClient client = createBlobClientWithRequestSniffer(new CopyOnWriteArrayList<>()).getPageBlobClient(); + client.create(UNDER_4MB_PAGE_ALIGNED); + + byte[] randomData = getRandomByteArray(UNDER_4MB_PAGE_ALIGNED); + byte[] md5 = MessageDigest.getInstance("MD5").digest(randomData); + PageBlobUploadPagesOptions options + = new PageBlobUploadPagesOptions(new PageRange().setStart(0).setEnd(UNDER_4MB_PAGE_ALIGNED - 1), + new ByteArrayInputStream(randomData)).setContentValidationAlgorithm(algorithm).setContentMd5(md5); + + BlobStorageException e = assertThrows(BlobStorageException.class, + () -> client.uploadPagesWithResponse(options, null, Context.NONE)); + assertEquals(400, e.getStatusCode()); + assertTrue(e.getMessage().contains(MESSAGE)); + } +} diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java index 99d925bff60c..71345472c393 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/BlobTestBase.java @@ -12,6 +12,9 @@ import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpPipeline; import com.azure.core.http.HttpPipelineBuilder; +import com.azure.core.http.HttpPipelineCallContext; +import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpPipelinePosition; import com.azure.core.http.HttpRequest; import com.azure.core.http.HttpResponse; import com.azure.core.http.policy.AddDatePolicy; @@ -61,6 +64,9 @@ import com.azure.storage.common.Utility; import com.azure.storage.common.implementation.Constants; import com.azure.storage.common.policy.RequestRetryOptions; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageEncoder; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageFlags; import com.azure.storage.common.test.shared.StorageCommonTestUtils; import com.azure.storage.common.test.shared.TestAccount; import com.azure.storage.common.test.shared.TestDataFactory; @@ -91,7 +97,10 @@ import java.util.Map; import java.util.Objects; import java.util.Queue; +import java.util.Random; +import java.util.UUID; import java.util.concurrent.Callable; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -904,6 +913,37 @@ protected File getRandomFile(int size) throws IOException { return StorageCommonTestUtils.getRandomFile(size, testResourceNamer); } + /** + * Pseudorandom {@link Random} seeded from {@code testResourceNamer.randomUuid()} so values replay in playback + * mode (unlike {@link java.util.concurrent.ThreadLocalRandom}). + */ + protected Random newRandomFromNamer() { + long seed = UUID.fromString(testResourceNamer.randomUuid()).getMostSignificantBits() & Long.MAX_VALUE; + return new Random(seed); + } + + /** + * Pseudorandom int in {@code [origin, bound)} from the namer. Consumes one recorded UUID. + */ + protected int randomIntFromNamer(int origin, int bound) { + int span = bound - origin; + if (span <= 0) { + throw new IllegalArgumentException("bound must be greater than origin"); + } + return origin + newRandomFromNamer().nextInt(span); + } + + /** + * Pseudorandom long in {@code [origin, bound)} from the namer. Consumes one recorded UUID. + */ + protected long randomLongFromNamer(long origin, long bound) { + long span = bound - origin; + if (span <= 0) { + throw new IllegalArgumentException("bound must be greater than origin"); + } + return origin + Math.floorMod(newRandomFromNamer().nextLong(), span); + } + /*https://learn.microsoft.com/en-us/rest/api/storageservices/define-stored-access-policy#creating-or-modifying-a-stored-access-policy Second note, it can take up to 30 seconds to set/create an access policy and this was causing flakeyness in the live test pipeline */ @@ -1344,4 +1384,469 @@ public static HttpPipelinePolicy getAddHeadersAndQueryPolicy(Map return next.process(); }; } + + protected static boolean hasOnlyStructuredMessageHeaders(List recordedRequestHeaders) { + return hasStructuredMessageDownloadRequestHeaders(recordedRequestHeaders, true); + } + + protected static boolean hasStructuredMessageDownloadRequestHeaders(HttpHeaders recordedRequestHeaders) { + if (recordedRequestHeaders == null || recordedRequestHeaders.getSize() == 0) { + return false; + } + return hasStructuredMessageDownloadRequestHeaders(Collections.singletonList(recordedRequestHeaders), false); + } + + protected static boolean hasStructuredMessageDownloadResponseHeaders(HttpHeaders headers) { + return validateBasicHeaders(headers) + && StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE + .equalsIgnoreCase(headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME)) + && hasValidStructuredContentLengthHeader(headers); + } + + protected static HttpPipelinePolicy getRequestAndResponseHeaderSniffer(String targetUrlPrefix, + HttpHeaders recordedRequestHeaders, HttpHeaders recordedResponseHeaders) { + return getRequestAndResponseHeaderSniffer(targetUrlPrefix, headers -> { + synchronized (recordedRequestHeaders) { + recordedRequestHeaders.setAllHttpHeaders(headers); + } + }, recordedResponseHeaders); + } + + protected static HttpPipelinePolicy getRequestAndResponseHeaderSniffer(String targetUrlPrefix, + List recordedRequestHeaders, HttpHeaders recordedResponseHeaders) { + return getRequestAndResponseHeaderSniffer(targetUrlPrefix, recordedRequestHeaders::add, + recordedResponseHeaders); + } + + private static HttpPipelinePolicy getRequestAndResponseHeaderSniffer(String targetUrlPrefix, + Consumer requestRecorder, HttpHeaders recordedResponseHeaders) { + return new HttpPipelinePolicy() { + @Override + public HttpPipelinePosition getPipelinePosition() { + return HttpPipelinePosition.PER_RETRY; + } + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + requestRecorder.accept(context.getHttpRequest().getHeaders()); + return next.process().map(response -> { + if (response.getRequest().getHttpMethod() == HttpMethod.GET + && response.getRequest().getUrl().toString().startsWith(targetUrlPrefix)) { + synchronized (recordedResponseHeaders) { + recordedResponseHeaders.setAllHttpHeaders(response.getHeaders()); + } + } + return response; + }); + } + }; + } + + protected static boolean hasStructuredMessageDownloadRequestHeaders(List recordedRequestHeaders, + boolean requireStructuredContentLength) { + if (recordedRequestHeaders == null || recordedRequestHeaders.isEmpty()) { + return false; + } + // Only consider requests where any of the structured-message or CRC64-related headers is present. + List headersWithContentValidation = recordedRequestHeaders.stream().filter(headers -> { + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + return bodyType != null || contentLength != null || contentCrc64 != null; + }).collect(Collectors.toList()); + // If no requests had any content-validation headers at all, we cannot claim structured-message was applied. + if (headersWithContentValidation.isEmpty()) { + return false; + } + // All requests that used any content-validation header must be consistent structured-message requests. + return headersWithContentValidation.stream().allMatch(headers -> { + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + if (!StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE.equals(bodyType) || contentCrc64 != null) { + return false; + } + if (!requireStructuredContentLength) { + return true; + } + // Require non-blank content length that parses as non-negative long (same format as policy uses). + // Rejects empty string, whitespace, or non-numeric values so we never return true when + // structured message was not actually applied. + if (contentLength == null || contentLength.trim().isEmpty()) { + return false; + } + try { + return Long.parseLong(contentLength.trim()) >= 0; + } catch (NumberFormatException e) { + return false; + } + }); + } + + protected static boolean hasOnlyCrc64Headers(List recordedRequestHeaders) { + if (recordedRequestHeaders == null || recordedRequestHeaders.isEmpty()) { + return false; + } + // Only consider requests where any of the structured-message or CRC64-related headers is present. + List headersWithContentValidation = recordedRequestHeaders.stream().filter(headers -> { + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + return bodyType != null || contentLength != null || contentCrc64 != null; + }).collect(Collectors.toList()); + // If no requests had any content-validation headers at all, we cannot claim CRC64 was applied. + if (headersWithContentValidation.isEmpty()) { + return false; + } + // All requests that used any content-validation header must be consistent CRC64-only requests. + return headersWithContentValidation.stream().allMatch(headers -> { + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + // Require CRC64 header to be present and non-blank (policy sets Base64-encoded 8 bytes). + // Reject empty/whitespace so we never return true when CRC64 was not actually applied. + if (contentCrc64 == null || contentCrc64.trim().isEmpty() || bodyType != null || contentLength != null) { + return false; + } + return true; + }); + } + + protected static boolean hasNoContentValidationHeaders(List recordedRequestHeaders) { + if (recordedRequestHeaders == null || recordedRequestHeaders.isEmpty()) { + return false; + } + return recordedRequestHeaders.stream().allMatch(headers -> { + String bodyType = headers.getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String contentLength = headers.getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + String contentCrc64 = headers.getValue(Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME); + // All three must be absent (null). If any header is present, even with empty value, we return false. + return bodyType == null && contentLength == null && contentCrc64 == null; + }); + } + + /** + * Creates a BlobClient that records all outgoing request headers into the supplied list. + * Each test should use its own list so tests can run concurrently. + */ + protected BlobClient createBlobClientWithRequestSniffer(List recordedRequestHeaders) { + HttpPipelinePolicy sniffPolicy = (context, next) -> { + recordedRequestHeaders.add(context.getHttpRequest().getHeaders()); + return next.process(); + }; + BlobServiceClient serviceClient = getServiceClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + ENVIRONMENT.getPrimaryAccount().getBlobEndpoint(), sniffPolicy); + return serviceClient.getBlobContainerClient(containerName).getBlobClient(generateBlobName()); + } + + /** + * Creates a BlobAsyncClient that records all outgoing request headers into the supplied list. + */ + protected BlobAsyncClient createBlobAsyncClientWithRequestSniffer(List recordedRequestHeaders) { + HttpPipelinePolicy sniffPolicy = (context, next) -> { + recordedRequestHeaders.add(context.getHttpRequest().getHeaders()); + return next.process(); + }; + BlobServiceAsyncClient serviceClient = getServiceAsyncClient(ENVIRONMENT.getPrimaryAccount().getCredential(), + ENVIRONMENT.getPrimaryAccount().getBlobEndpoint(), sniffPolicy); + return serviceClient.getBlobContainerAsyncClient(containerName).getBlobAsyncClient(generateBlobName()); + } + + protected static long expectedStructuredMessageEncodedLength(int unencodedContentBytes) { + return new StructuredMessageEncoder(unencodedContentBytes, + StructuredMessageConstants.V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64) + .getEncodedMessageLength(); + } + + /** + * Sum of encoded lengths per block upload (each HTTP request carries its own structured message wrapper). + */ + protected static long expectedStructuredMessageEncodedLengthChunked(int totalUnencodedBytes, long blockSizeBytes) { + long sum = 0; + int remaining = totalUnencodedBytes; + while (remaining > 0) { + int chunk = (int) Math.min(remaining, blockSizeBytes); + sum += expectedStructuredMessageEncodedLength(chunk); + remaining -= chunk; + } + return sum; + } + + /** + * Every tuple keeps payloadBytes <= blockSizeBytes, so the parallel download path issues a single GET (no + * follow-on range requests for additional blocks), which replays under the test proxy. + *

+ * Sizes are deliberately non-power-of-two (e.g. 7 * KB + 3) and use mixed block ceilings (64 KiB through + * multi-MiB) to catch alignment and decoder edge cases at structural boundaries (message header, segment + * footer, message footer); the 4 MiB boundary row exercises the exact service-side default segment length. + */ + protected static Stream fuzzyParallelDownloadReplayableCases() { + return Stream.of(Arguments.of(1, 64L * Constants.KB, 1), + Arguments.of(7 * Constants.KB + 3, 64L * Constants.KB, 1), + Arguments.of(7 * Constants.KB + 3, 128L * Constants.KB, 4), + Arguments.of(41 * Constants.KB + 17, 256L * Constants.KB, 1), + Arguments.of(41 * Constants.KB + 17, 256L * Constants.KB, 8), + Arguments.of(199 * Constants.KB + 5, 512L * Constants.KB, 2), + Arguments.of(512 * Constants.KB - 31, 1L * Constants.MB, 8), + Arguments.of(896 * Constants.KB + 101, 1L * Constants.MB, 6), + Arguments.of(2 * Constants.MB - 1, 4L * Constants.MB, 4), + Arguments.of(2 * Constants.MB + 33, 4L * Constants.MB, 1), + Arguments.of(4 * Constants.MB - 1, 4L * Constants.MB, 2), + Arguments.of(4 * Constants.MB, 4L * Constants.MB, 1), + Arguments.of(4 * Constants.MB, 7L * Constants.MB + 919, 3)); + } + + /** + * Every tuple keeps payloadBytes <= segmentBytes, so the parallel upload path issues a single Put Blob (no + * Put Block IDs), which replays under the test proxy unlike staging-heavy cases. + *

+ * Sizes are deliberately non-power-of-two (e.g. 7 * KB + 3) and use mixed segment ceilings (64 KiB + * through multi-MiB) to catch alignment and buffer edge cases; rows include the exact 4 MiB service boundary + * and several concurrency values (1–8) to exercise parallel request fan-out without live-only staging. + */ + protected static Stream fuzzyParallelUploadPutBlobReplayableCases() { + return Stream.of(Arguments.of(7 * Constants.KB + 3, 64L * Constants.KB, 1), + Arguments.of(7 * Constants.KB + 3, 128L * Constants.KB, 4), + Arguments.of(41 * Constants.KB + 17, 256L * Constants.KB, 1), + Arguments.of(41 * Constants.KB + 17, 256L * Constants.KB, 8), + Arguments.of(199 * Constants.KB + 5, 512L * Constants.KB, 2), + Arguments.of(512 * Constants.KB - 31, 1L * Constants.MB, 8), + Arguments.of(896 * Constants.KB + 101, 1L * Constants.MB, 6), + Arguments.of(4 * Constants.MB, 4L * Constants.MB, 1), + Arguments.of(4 * Constants.MB, 7L * Constants.MB + 919, 3)); + } + + /** + * payloadBytes > blockSizeBytes but totals stay in the hundreds of KiB, so downloads issue many small ranged + * GETs even though the blob itself is small. Live-only because chunked range GETs across many tiny requests + * produce per-block proxy churn similar to the upload-side staging cases. + *

+ * One row pairs a ~200 KiB payload with a 64 KiB block size and moderate concurrency; the other uses a + * ~512 KiB payload with a 1 KiB block size to force many tiny range GETs (stress decoder framing and + * scheduling) without the cost of the large multi-part grids. + */ + protected static Stream fuzzyParallelDownloadSmallMultiPartCases() { + return Stream.of(Arguments.of(200 * Constants.KB, 64L * Constants.KB, 3), + Arguments.of(512 * Constants.KB - 31, 1L * Constants.KB, 1)); + } + + /** + * payloadBytes > segmentBytes, so uploads still go through Put Block staging even though totals are only + * hundreds of KiB—too small for the proxy when block IDs vary per run. + *

+ * One row pairs a ~200 KiB payload with a 64 KiB segment and moderate concurrency; the other uses a + * ~512 KiB payload with a 1 KiB segment to force many tiny blocks (stress scheduling and per-block CRC64 + * framing) without the cost of the large multi-part grids. + */ + protected static Stream fuzzyParallelUploadSmallPayloadStagingCases() { + return Stream.of(Arguments.of(200 * Constants.KB, 64L * Constants.KB, 3), + Arguments.of(512 * Constants.KB - 31, 1L * Constants.KB, 1)); + } + + /** + * payloadBytes > blockSizeBytes and payloadBytes <= 4 * Constants.MB - 1 (the ceiling field), so the blob + * stays strictly under the 4 MiB single-shot CRC64-header path while downloads remain chunked across + * multiple range GETs. + *

+ * Values mix MiB/KiB block sizes with offsets (e.g. + 19, - 903) so part counts and last-block lengths are + * not powers of two; the last rows hug ceiling with awkward divisors in blockSizeBytes to stress remainder + * handling near the sub-4 MiB limit. + */ + protected static Stream fuzzyParallelDownloadSubFourMiBCases() { + final int ceiling = (4 * Constants.MB) - 1; + return Stream.of(Arguments.of(1 * Constants.MB + 1, 1L * Constants.MB, 1), + Arguments.of(1 * Constants.MB + 1, 2L * Constants.KB, 8), + Arguments.of((5 * Constants.MB) / 4 + 19, 256L * Constants.KB, 4), + Arguments.of(2 * Constants.MB - 903, 1L * Constants.MB, 2), + Arguments.of(2 * Constants.MB + 33, 1L * Constants.KB, 1), + Arguments.of(2 * Constants.MB + 33, 1L * Constants.MB, 8), + Arguments.of((11 * Constants.MB) / 4 - 17, 512L * Constants.KB, 6), + Arguments.of(3 * Constants.MB - 777, 2L * Constants.MB, 8), + Arguments.of(3 * Constants.MB - 1, 1L * Constants.MB, 1), Arguments.of(ceiling - 511, 1L * Constants.MB, 4), + Arguments.of(ceiling - 511, 1L * Constants.MB + 511, 2), + Arguments.of(ceiling, (long) (ceiling / 7 + 17), 3), Arguments.of(ceiling, (long) (ceiling / 2 + 1), 8)); + } + + /** + * payloadBytes > segmentBytes and payloadBytes <= 4 * Constants.MB - 1 (the ceiling field), so the blob + * stays strictly under the 4 MiB transactional CRC64-header path while uploads remain + * chunked—live-only because of Put Block identity churn. + *

+ * Values mix MiB/KiB segment sizes with offsets (e.g. + 19, - 903) so part counts and last-block + * lengths are not powers of two; the last rows hug ceiling with awkward divisors in segmentBytes to + * stress remainder handling near the sub-4 MiB limit. + */ + protected static Stream fuzzyParallelUploadSub4MiBCases() { + final int ceiling = (4 * Constants.MB) - 1; + return Stream.of(Arguments.of(1 * Constants.MB + 1, 1L * Constants.MB, 1), + Arguments.of(1 * Constants.MB + 1, 2L * Constants.KB, 8), + Arguments.of((5 * Constants.MB) / 4 + 19, 256L * Constants.KB, 4), + Arguments.of(2 * Constants.MB - 903, 1L * Constants.MB, 2), + Arguments.of(2 * Constants.MB + 33, 1L * Constants.KB, 1), + Arguments.of(2 * Constants.MB + 33, 1L * Constants.MB, 8), + Arguments.of((11 * Constants.MB) / 4 - 17, 512L * Constants.KB, 6), + Arguments.of(3 * Constants.MB - 777, 2L * Constants.MB, 8), + Arguments.of(3 * Constants.MB - 1, 1L * Constants.MB, 1), Arguments.of(ceiling - 511, 1L * Constants.MB, 4), + Arguments.of(ceiling - 511, 1L * Constants.MB + 511, 2), + Arguments.of(ceiling, (long) (ceiling / 7 + 17), 3), Arguments.of(ceiling, (long) (ceiling / 2 + 1), 8)); + } + + /** + * Centers on 4 * Constants.MB - 1, exactly 4 * Constants.MB, and just above 4 MiB, with block sizes spanning + * KiB through multi-MiB - exercising the SDK/service boundary where single-shot vs chunked range GET and + * CRC64 header vs structured-message rules flip, while keeping deterministic single-GET coverage in the + * replayable supplier above. + *

+ * Includes near-boundary payloads (e.g. -8192, +31, +8191 from 4 MiB) so neither total size nor last segment + * length aligns to typical buffer multiples. + */ + protected static Stream fuzzyParallelDownloadFourMiBBoundaryCases() { + final int minus = (4 * Constants.MB) - 1; + final int plus = (4 * Constants.MB) + 1; + return Stream.of(Arguments.of(minus, 1L * Constants.MB, 1), Arguments.of(minus, 512L * Constants.KB, 6), + Arguments.of(minus, 2L * Constants.MB, 8), Arguments.of((4 * Constants.MB) - 8192, 1L * Constants.KB, 4), + Arguments.of(4 * Constants.MB, (long) (4 * Constants.MB / 2), 8), + Arguments.of(4 * Constants.MB, 256L * Constants.KB, 2), Arguments.of(plus, 1L * Constants.MB, 1), + Arguments.of(plus, 2L * Constants.MB, 8), Arguments.of(plus, 1L * Constants.KB, 7), + Arguments.of((4 * Constants.MB) + 31, 511L * Constants.KB + 409, 4), + Arguments.of((4 * Constants.MB) + 8191, 3L * Constants.MB - 413, 6)); + } + + /** + * Centers on 4 * Constants.MB - 1, exactly 4 * Constants.MB, and just above 4 MiB, with segment + * sizes spanning KiB through multi-MiB—exercising the SDK/service boundary where single-shot vs block staging and + * CRC64 header vs structured-message rules flip, while keeping deterministic Put Blob coverage in the replayable + * supplier above. + *

+ * Includes near-boundary payloads (e.g. -8192, +31, +8191 from 4 MiB) so neither total size nor last segment + * length aligns to typical buffer multiples. + */ + protected static Stream fuzzyParallelUploadFourMiBBoundaryStagingCases() { + final int minus = (4 * Constants.MB) - 1; + final int plus = (4 * Constants.MB) + 1; + return Stream.of(Arguments.of(minus, 1L * Constants.MB, 1), Arguments.of(minus, 512L * Constants.KB, 6), + Arguments.of(minus, 2L * Constants.MB, 8), Arguments.of((4 * Constants.MB) - 8192, 1L * Constants.KB, 4), + Arguments.of(4 * Constants.MB, (long) (4 * Constants.MB / 2), 8), + Arguments.of(4 * Constants.MB, 256L * Constants.KB, 2), Arguments.of(plus, 1L * Constants.MB, 1), + Arguments.of(plus, 2L * Constants.MB, 8), Arguments.of(plus, 1L * Constants.KB, 7), + Arguments.of((4 * Constants.MB) + 31, 511L * Constants.KB + 409, 4), + Arguments.of((4 * Constants.MB) + 8191, 3L * Constants.MB - 413, 6)); + } + + /** + * payloadBytes > blockSizeBytes, so downloads always go through multiple ranged GETs (parallel download + * fan-out) with totals roughly 6-80 MiB. Large enough to exercise the structured-message decoder over + * multiple HTTP responses, but cheaper than {@link #fuzzyParallelDownloadLargeMultiPartCases}. + *

+ * Block sizes step through common service limits (1-8 MiB, half-MiB tail values); concurrency 1-8 pairs + * with imbalanced payloads (e.g. 701, 333) to flush merge/retry edge cases. + */ + protected static Stream fuzzyParallelDownloadMediumMultiPartCases() { + return Stream.of(Arguments.of(6 * Constants.MB + 701, Constants.MB, 1), + Arguments.of(6 * Constants.MB + 701, 3L * Constants.MB + 271, 4), + Arguments.of(9 * Constants.MB + 333, 2L * Constants.MB, 1), + Arguments.of(9 * Constants.MB + 333, 3L * Constants.MB + 199, 8), + Arguments.of(12 * Constants.MB + 901, 4L * Constants.MB + 901, 2), + Arguments.of(14 * Constants.MB, 500L * Constants.KB + 13, 6), + Arguments.of(18 * Constants.MB - 4021, 5L * Constants.MB - 701, 3), + Arguments.of(24 * Constants.MB, 8L * Constants.MB, 8), + Arguments.of(28 * Constants.MB + 56789, 7L * Constants.MB + 13, 2), + Arguments.of(31 * Constants.MB, 1024L * Constants.KB + 17, 4), + Arguments.of(40 * Constants.MB + 12345, 7L * Constants.MB + 13, 3), + Arguments.of(48 * Constants.MB - 777, 5L * Constants.MB + 809L * Constants.KB, 6), + Arguments.of(56 * Constants.MB + 19, 9L * Constants.MB + 4096, 8), + Arguments.of(72 * Constants.MB, 4L * Constants.MB + 65536, 8), + Arguments.of(80 * Constants.MB + 321, 13L * Constants.MB - 3073, 1)); + } + + /** + * All rows keep payloadBytes > segmentBytes with totals roughly 6–80 MiB—large enough for meaningful parallel + * block fan-out and structured-message segments, but cheaper than {@link #fuzzyParallelUploadLargeMultiPartCases}. + *

+ * Block sizes step through common service limits (1–8 MiB, half-MiB tail values); concurrency 1–8 pairs with + * imbalanced payloads (e.g. 701, 333) to flush merge/retry edge cases. + */ + protected static Stream fuzzyParallelUploadMediumMultiPartCases() { + return Stream.of(Arguments.of(6 * Constants.MB + 701, Constants.MB, 1), + Arguments.of(6 * Constants.MB + 701, 3L * Constants.MB + 271, 4), + Arguments.of(9 * Constants.MB + 333, 2L * Constants.MB, 1), + Arguments.of(9 * Constants.MB + 333, 3L * Constants.MB + 199, 8), + Arguments.of(12 * Constants.MB + 901, 4L * Constants.MB + 901, 2), + Arguments.of(14 * Constants.MB, 500L * Constants.KB + 13, 6), + Arguments.of(18 * Constants.MB - 4021, 5L * Constants.MB - 701, 3), + Arguments.of(24 * Constants.MB, 8L * Constants.MB, 8), + Arguments.of(28 * Constants.MB + 56789, 7L * Constants.MB + 13, 2), + Arguments.of(31 * Constants.MB, 1024L * Constants.KB + 17, 4), + Arguments.of(40 * Constants.MB + 12345, 7L * Constants.MB + 13, 3), + Arguments.of(48 * Constants.MB - 777, 5L * Constants.MB + 809L * Constants.KB, 6), + Arguments.of(56 * Constants.MB + 19, 9L * Constants.MB + 4096, 8), + Arguments.of(72 * Constants.MB, 4L * Constants.MB + 65536, 8), + Arguments.of(80 * Constants.MB + 321, 13L * Constants.MB - 3073, 1)); + } + + /** + * Stresses high block counts and long-running parallel downloads (~96-320 MiB payloads) with service-realistic + * block sizes (8-61 MiB class) and heavy concurrency. + *

+ * The final rows use named near-256/288/320 MiB totals with irregular byte tails to keep total bytes and block + * remainders off common multiples while still bounding runtime for Live-only CI. + */ + protected static Stream fuzzyParallelDownloadLargeMultiPartCases() { + final int payload257MiBPlus = (int) (257L * Constants.MB + 18881); + final int payload288MiBPlus = (int) (288L * Constants.MB + 7777); + final int payload320MiBPlus = (int) (320L * Constants.MB + 1999); + return Stream.of(Arguments.of(96 * Constants.MB + 17, 8L * Constants.MB + 511, 2), + Arguments.of(112 * Constants.MB, 15L * Constants.MB + 4096, 8), + Arguments.of(128 * Constants.MB + 45673, 17L * Constants.MB - 11264 + 173, 4), + Arguments.of(160 * Constants.MB + 12345, 12L * Constants.MB + 8192, 8), + Arguments.of(192 * Constants.MB + 9876, 31L * Constants.MB - 513, 8), + Arguments.of(224 * Constants.MB, 23L * Constants.MB + 524288, 8), + Arguments.of(payload257MiBPlus, 61L * Constants.MB + 23L * Constants.KB, 6), + Arguments.of(payload288MiBPlus, 36L * Constants.MB + 513, 8), + Arguments.of(payload320MiBPlus, 16L * Constants.MB + 511, 8)); + } + + /** + * Stresses high block counts and long-running parallel uploads (~96–320 MiB payloads) with service-realistic segment + * sizes (8–61 MiB class) and heavy concurrency. + *

+ * The final rows use named near-256/288/320 MiB totals with irregular byte tails to keep total bytes and + * block remainders off common multiples while still bounding runtime for Live-only CI. + */ + protected static Stream fuzzyParallelUploadLargeMultiPartCases() { + final int payload257MiBPlus = (int) (257L * Constants.MB + 18881); + final int payload288MiBPlus = (int) (288L * Constants.MB + 7777); + final int payload320MiBPlus = (int) (320L * Constants.MB + 1999); + return Stream.of(Arguments.of(96 * Constants.MB + 17, 8L * Constants.MB + 511, 2), + Arguments.of(112 * Constants.MB, 15L * Constants.MB + 4096, 8), + Arguments.of(128 * Constants.MB + 45673, 17L * Constants.MB - 11264 + 173, 4), + Arguments.of(160 * Constants.MB + 12345, 12L * Constants.MB + 8192, 8), + Arguments.of(192 * Constants.MB + 9876, 31L * Constants.MB - 513, 8), + Arguments.of(224 * Constants.MB, 23L * Constants.MB + 524288, 8), + Arguments.of(payload257MiBPlus, 61L * Constants.MB + 23L * Constants.KB, 6), + Arguments.of(payload288MiBPlus, 36L * Constants.MB + 513, 8), + Arguments.of(payload320MiBPlus, 16L * Constants.MB + 511, 8)); + } + + /** + * Single ~1 GiB download with high concurrency and an awkward (non-aligned) tail to exercise the structured + * message decoder under a sustained, fan-out-heavy parallel download. Live-only and file-backed so payload + * never materializes twice in heap. + */ + protected static Stream fuzzyParallelDownloadOneGiBCases() { + return Stream.of(Arguments.of((int) (1L * Constants.GB + 1377), 16L * Constants.MB + 511, 8)); + } + + private static boolean hasValidStructuredContentLengthHeader(HttpHeaders headers) { + String structuredContentLength = headers.getValue("x-ms-structured-content-length"); + if (CoreUtils.isNullOrEmpty(structuredContentLength) + || CoreUtils.isNullOrEmpty(structuredContentLength.trim())) { + return false; + } + try { + return Long.parseLong(structuredContentLength.trim()) >= 0; + } catch (NumberFormatException ex) { + return false; + } + } } diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java index 4ff8054894a6..24a9ca7e781a 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobSeekableByteChannelTests.java @@ -106,7 +106,7 @@ static Stream channelReadDataSupplier() { * @param copySize Size of array to copy contents with. * @return Total number of bytes read from src. */ - private static int copy(SeekableByteChannel src, OutputStream dst, int copySize) throws IOException { + public static int copy(SeekableByteChannel src, OutputStream dst, int copySize) throws IOException { int read; int totalRead = 0; byte[] temp = new byte[copySize]; diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobApiTests.java index c2a903145440..ba9d985b308f 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobApiTests.java @@ -210,7 +210,7 @@ public void stageBlockIllegalArguments(boolean getBlockId, InputStream stream, i private static Stream stageBlockIllegalArgumentsSupplier() { return Stream.of( - Arguments.of(false, DATA.getDefaultInputStream(), DATA.getDefaultDataSize(), BlobStorageException.class), + Arguments.of(false, DATA.getDefaultInputStream(), DATA.getDefaultDataSize(), NullPointerException.class), Arguments.of(true, null, DATA.getDefaultDataSize(), NullPointerException.class), Arguments.of(true, DATA.getDefaultInputStream(), DATA.getDefaultDataSize() + 1, UnexpectedLengthException.class), diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAsyncApiTests.java b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAsyncApiTests.java index 664fe555846d..184f4eed38ed 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAsyncApiTests.java +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlockBlobAsyncApiTests.java @@ -250,7 +250,7 @@ public void stageBlockIllegalArguments(boolean getBlockId, Flux stre private static Stream stageBlockIllegalArgumentsSupplier() { return Stream.of( - Arguments.of(false, DATA.getDefaultFlux(), DATA.getDefaultDataSize(), BlobStorageException.class), + Arguments.of(false, DATA.getDefaultFlux(), DATA.getDefaultDataSize(), NullPointerException.class), Arguments.of(true, null, DATA.getDefaultDataSize(), NullPointerException.class), Arguments.of(true, DATA.getDefaultFlux(), DATA.getDefaultDataSize() + 1, UnexpectedLengthException.class), Arguments.of(true, DATA.getDefaultFlux(), DATA.getDefaultDataSize() - 1, UnexpectedLengthException.class)); diff --git a/sdk/storage/azure-storage-blob/swagger/README.md b/sdk/storage/azure-storage-blob/swagger/README.md index 292d2f7c231d..98afe0c616dc 100644 --- a/sdk/storage/azure-storage-blob/swagger/README.md +++ b/sdk/storage/azure-storage-blob/swagger/README.md @@ -16,7 +16,7 @@ autorest ### Code generation settings ``` yaml use: '@autorest/java@4.1.63' -input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/15d7f54a5389d5906ffb4e56bb2f38fe5525c0d3/specification/storage/data-plane/Microsoft.BlobStorage/stable/2026-06-06/blob.json +input-file: https://raw.githubusercontent.com/seanmcc-msft/azure-rest-api-specs/eb29a830edf5db50758e7d044160c7f18077f7f7/specification/storage/data-plane/Microsoft.BlobStorage/stable/2026-10-06/blob.json java: true output-folder: ../ namespace: com.azure.storage.blob diff --git a/sdk/storage/azure-storage-common/ci.system.properties b/sdk/storage/azure-storage-common/ci.system.properties index 1d1c46cd13b4..dc3baf90585b 100644 --- a/sdk/storage/azure-storage-common/ci.system.properties +++ b/sdk/storage/azure-storage-common/ci.system.properties @@ -1,2 +1,2 @@ -AZURE_LIVE_TEST_SERVICE_VERSION=V2026_06_06 -AZURE_STORAGE_SAS_SERVICE_VERSION=2026-06-06 +AZURE_LIVE_TEST_SERVICE_VERSION=V2026_12_06 +AZURE_STORAGE_SAS_SERVICE_VERSION=2026-12-06 diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ContentValidationAlgorithm.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ContentValidationAlgorithm.java new file mode 100644 index 000000000000..0d1a7ea22b07 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/ContentValidationAlgorithm.java @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common; + +/** + * Algorithm for content validation on upload and download operations. When enabled, the SDK computes checksums and + * validates content integrity using the selected algorithm. Content validation is off by default. + *

+ * Supported in Azure Storage Blob, Data Lake, and File Share packages for methods that use APIs supporting + * transactional CRC64, or structured message format. + */ +public enum ContentValidationAlgorithm { + + /** + * No content validation. This is the default; no checksum is computed or validated. + */ + NONE, + + /** + * Allow the SDK to choose the validation algorithm. Currently resolves to CRC64 where supported. Different + * library versions may make different choices. The resolution may change in the future. Please set an + * explicit algorithm if you need a specific behavior. + */ + AUTO, + + /** + * Azure Storage custom 64-bit CRC. The SDK computes and validates CRC64 checksums for the transfer. + */ + CRC64 +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java index e3b88b661134..687e80c71961 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/Constants.java @@ -88,7 +88,7 @@ public final class Constants { public static final String PROPERTY_AZURE_STORAGE_SAS_SERVICE_VERSION = "AZURE_STORAGE_SAS_SERVICE_VERSION"; public static final String SAS_SERVICE_VERSION - = Configuration.getGlobalConfiguration().get(PROPERTY_AZURE_STORAGE_SAS_SERVICE_VERSION, "2026-06-06"); + = Configuration.getGlobalConfiguration().get(PROPERTY_AZURE_STORAGE_SAS_SERVICE_VERSION, "2026-12-06"); public static final String ADJUSTED_BLOB_LENGTH_KEY = "adjustedBlobLength"; @@ -220,7 +220,7 @@ public static final class HeaderConstants { * @deprecated For SAS Service Version use {@link Constants#SAS_SERVICE_VERSION}. */ @Deprecated - public static final String TARGET_STORAGE_VERSION = "2026-06-06"; + public static final String TARGET_STORAGE_VERSION = "2026-12-06"; /** * Error code returned from the service. @@ -250,6 +250,20 @@ public static final class HeaderConstants { public static final String ETAG_WILDCARD = "*"; + public static final String CONTENT_CRC64 = "x-ms-content-crc64"; + + public static final HttpHeaderName CONTENT_CRC64_HEADER_NAME = HttpHeaderName.fromString(CONTENT_CRC64); + + public static final String STRUCTURED_BODY_TYPE = "x-ms-structured-body"; + + public static final HttpHeaderName STRUCTURED_BODY_TYPE_HEADER_NAME + = HttpHeaderName.fromString(STRUCTURED_BODY_TYPE); + + public static final String STRUCTURED_CONTENT_LENGTH = "x-ms-structured-content-length"; + + public static final HttpHeaderName STRUCTURED_CONTENT_LENGTH_HEADER_NAME + = HttpHeaderName.fromString(STRUCTURED_CONTENT_LENGTH); + /** * Metadata key ("hdi_isfolder") used to mark virtual directories in Azure Blob Storage. * diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/UploadUtils.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/UploadUtils.java index 32e10374b249..0b81cff41227 100644 --- a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/UploadUtils.java +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/UploadUtils.java @@ -98,7 +98,9 @@ public static Flux chunkSource(Flux data, ParallelTransf } int numSplits = (int) Math.ceil(buffer.remaining() / (double) chunkSize); return Flux.range(0, numSplits).map(i -> { - ByteBuffer duplicate = buffer.duplicate().asReadOnlyBuffer(); + // While duplicate.asReadOnlyBuffer() is safer, it significantly slows down crc64 calculation because it forces a copy of the buffer. + // No downstream buffers should be modifying the buffer anyways. + ByteBuffer duplicate = buffer.duplicate(); duplicate.position(i * chunkSize); duplicate.limit(Math.min(duplicate.limit(), (i + 1) * chunkSize)); return duplicate; diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java new file mode 100644 index 000000000000..5c3394117971 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolver.java @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CONTENT_VALIDATION_MODE_KEY; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_CRC64_CHECKSUM_HEADER_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_STRUCTURED_MESSAGE_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY; + +import com.azure.core.util.Context; +import com.azure.core.util.FluxUtil; +import com.azure.core.util.ProgressListener; +import com.azure.storage.common.ContentValidationAlgorithm; +import com.azure.storage.common.ParallelTransferOptions; +import reactor.core.publisher.Mono; + +/** + * Determines the content validation mode string to pass in the pipeline context for upload operations. + * Callers put the returned value under {@link StructuredMessageConstants#CONTENT_VALIDATION_MODE_KEY}. + *

+ * Single-shot: use CRC64 header when length < 4 MiB, otherwise structured message. + * Chunked (multi-shot): always use structured message. + */ +public final class ContentValidationModeResolver { + + private ContentValidationModeResolver() { + } + + public static final String CONFLICTING_TRANSACTIONAL_CONTENT_VALIDATION_MESSAGE + = "Individual MD5 option and checksum algorithm option bag are both used. Only one form of transactional content validation may be used."; + + /** + * Progress reporting counts bytes on the wire; transfer validation (CRC64/AUTO) may use structured messages, so the + * two cannot be combined. Use {@link ContentValidationAlgorithm#NONE} or null, or omit the progress listener. + */ + public static final String PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE + = "Progress reporting cannot be combined with ContentValidationAlgorithm.CRC64 or ContentValidationAlgorithm.AUTO. " + + "Set ContentValidationAlgorithm to NONE or null, or remove the progress listener."; + + /** + * Resolves content validation mode and adds it to the Azure {@link Context} when non-null. + * + * @param context The request context; {@code null} is treated as {@link Context#NONE}. + * @param algorithm The transfer validation checksum algorithm. + * @param contentLength The upload length in bytes. + * @param chunkedUpload Whether this request is part of a multi-shot upload. + * @return The context, with {@link StructuredMessageConstants#CONTENT_VALIDATION_MODE_KEY} set when applicable. + */ + public static Context addContentValidationMode(Context context, ContentValidationAlgorithm algorithm, + long contentLength, boolean chunkedUpload) { + Context baseContext = context == null ? Context.NONE : context; + String mode + = chunkedUpload ? getModeForChunkedUpload(algorithm) : getModeForSingleShotUpload(algorithm, contentLength); + return mode == null ? baseContext : baseContext.addData(CONTENT_VALIDATION_MODE_KEY, mode); + } + + /** + * Resolves content validation mode and propagates it on the Reactor context for {@code mono} when non-null. + * + * @param mono The reactive sequence to augment. + * @param algorithm The transfer validation checksum algorithm. + * @param contentLength The upload length in bytes. + * @param chunkedUpload Whether this request is part of a multi-shot upload. + * @param The type of the elements in the reactive sequence. + * @return {@code mono}, possibly augmented with Reactor context writes. + */ + public static Mono addContentValidationMode(Mono mono, ContentValidationAlgorithm algorithm, + long contentLength, boolean chunkedUpload) { + String mode + = chunkedUpload ? getModeForChunkedUpload(algorithm) : getModeForSingleShotUpload(algorithm, contentLength); + if (mode == null) { + return mono; + } + return mono.contextWrite(FluxUtil.toReactorContext(new Context(CONTENT_VALIDATION_MODE_KEY, mode))); + } + + /** + * Mode for a single-shot upload. Use CRC64 header when length is less than 4MB, otherwise structured + * message. + */ + private static String getModeForSingleShotUpload(ContentValidationAlgorithm algorithm, long length) { + if (isCrc64OrAuto(algorithm)) { + return length < MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER + ? USE_CRC64_CHECKSUM_HEADER_CONTEXT + : USE_STRUCTURED_MESSAGE_CONTEXT; + } + return null; + } + + /** + * Mode for a chunked (multi-shot) upload. Always use structured message. + */ + private static String getModeForChunkedUpload(ContentValidationAlgorithm algorithm) { + if (isCrc64OrAuto(algorithm)) { + return USE_STRUCTURED_MESSAGE_CONTEXT; + } + return null; + } + + /** + * Validates transactional checksum options when MD5 may be SDK-computed. Throws if {@code computeMd5} and a + * non-none {@code contentValidationAlgorithm} are both active. + * + * @param computeMd5 Whether the SDK will compute transactional MD5. + * @param contentValidationAlgorithm Transfer validation checksum algorithm from options. + * @throws IllegalArgumentException if options conflict. + */ + public static void validateTransactionalChecksumOptions(boolean computeMd5, + ContentValidationAlgorithm contentValidationAlgorithm) { + if (computeMd5 && contentValidationAlgorithm != null) { + throw new IllegalArgumentException(CONFLICTING_TRANSACTIONAL_CONTENT_VALIDATION_MESSAGE); + } + } + + /** + * @return {@code true} when {@code contentValidationAlgorithm} enables CRC64 or AUTO transfer validation (not + * {@code null} and not {@link ContentValidationAlgorithm#NONE}). + */ + public static boolean isContentValidationAlgorithmPresent(ContentValidationAlgorithm contentValidationAlgorithm) { + return contentValidationAlgorithm != null && contentValidationAlgorithm != ContentValidationAlgorithm.NONE; + } + + /** + * @return {@code true} when {@code algorithm} is {@link ContentValidationAlgorithm#CRC64} or + * {@link ContentValidationAlgorithm#AUTO}. Upload and download structured-message validation use this rule. + */ + public static boolean isCrc64OrAuto(ContentValidationAlgorithm algorithm) { + return algorithm == ContentValidationAlgorithm.CRC64 || algorithm == ContentValidationAlgorithm.AUTO; + } + + /** + * When the transfer validation mode is {@link ContentValidationAlgorithm#CRC64} or + * {@link ContentValidationAlgorithm#AUTO}, adds + * {@link StructuredMessageConstants#STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY} so the HTTP + * pipeline can decode/validate the structured message response. For {@code null} or + * {@link ContentValidationAlgorithm#NONE}, returns the context unchanged (no key added), matching "no + * structured-message validation" for that download. + * + * @param context The base {@link Context}; null is treated as {@link Context#NONE}. + * @param contentValidationAlgorithm The algorithm from download options, or null. + * @return The same context, or a copy with the decoding key set when applicable. + */ + public static Context addStructuredMessageDecodingToContext(Context context, + ContentValidationAlgorithm contentValidationAlgorithm) { + Context base = context == null ? Context.NONE : context; + if (!isCrc64OrAuto(contentValidationAlgorithm)) { + return base; + } + return base.addData(STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + } + + /** + * Validates that progress reporting is not combined with CRC64/AUTO content validation. + * + * @param progressListener Progress listener from {@link ParallelTransferOptions} or equivalent; may be null. + * @param contentValidationAlgorithm Transfer validation algorithm from options. + * @throws IllegalArgumentException if {@code progressListener} is non-null and {@link #isContentValidationAlgorithmPresent} + * is true. + */ + public static void validateProgressWithContentValidation(ProgressListener progressListener, + ContentValidationAlgorithm contentValidationAlgorithm) { + if (progressListener != null && isContentValidationAlgorithmPresent(contentValidationAlgorithm)) { + throw new IllegalArgumentException(PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE); + } + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java new file mode 100644 index 000000000000..ed21ef153823 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64Calculator.java @@ -0,0 +1,2660 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * This class provides methods to compute and manipulate CRC64 checksums using the Azure Storage CRC64 polynomial. + * It includes methods for computing CRC64 checksums for byte arrays, updating CRC values using lookup tables, and + * concatenating CRC values. + *

+ * RESERVED FOR INTERNAL USE. + * + */ +public class StorageCrc64Calculator { + + private static final long POLY = 0x9A6C9329AC4BC9B5L; + + private static final long[] M_U1 = { + 0x0000000000000000L, + 0x7f6ef0c830358979L, + 0xfedde190606b12f2L, + 0x81b31158505e9b8bL, + 0xc962e5739841b68fL, + 0xb60c15bba8743ff6L, + 0x37bf04e3f82aa47dL, + 0x48d1f42bc81f2d04L, + 0xa61cecb46814fe75L, + 0xd9721c7c5821770cL, + 0x58c10d24087fec87L, + 0x27affdec384a65feL, + 0x6f7e09c7f05548faL, + 0x1010f90fc060c183L, + 0x91a3e857903e5a08L, + 0xeecd189fa00bd371L, + 0x78e0ff3b88be6f81L, + 0x078e0ff3b88be6f8L, + 0x863d1eabe8d57d73L, + 0xf953ee63d8e0f40aL, + 0xb1821a4810ffd90eL, + 0xceecea8020ca5077L, + 0x4f5ffbd87094cbfcL, + 0x30310b1040a14285L, + 0xdefc138fe0aa91f4L, + 0xa192e347d09f188dL, + 0x2021f21f80c18306L, + 0x5f4f02d7b0f40a7fL, + 0x179ef6fc78eb277bL, + 0x68f0063448deae02L, + 0xe943176c18803589L, + 0x962de7a428b5bcf0L, + 0xf1c1fe77117cdf02L, + 0x8eaf0ebf2149567bL, + 0x0f1c1fe77117cdf0L, + 0x7072ef2f41224489L, + 0x38a31b04893d698dL, + 0x47cdebccb908e0f4L, + 0xc67efa94e9567b7fL, + 0xb9100a5cd963f206L, + 0x57dd12c379682177L, + 0x28b3e20b495da80eL, + 0xa900f35319033385L, + 0xd66e039b2936bafcL, + 0x9ebff7b0e12997f8L, + 0xe1d10778d11c1e81L, + 0x606216208142850aL, + 0x1f0ce6e8b1770c73L, + 0x8921014c99c2b083L, + 0xf64ff184a9f739faL, + 0x77fce0dcf9a9a271L, + 0x08921014c99c2b08L, + 0x4043e43f0183060cL, + 0x3f2d14f731b68f75L, + 0xbe9e05af61e814feL, + 0xc1f0f56751dd9d87L, + 0x2f3dedf8f1d64ef6L, + 0x50531d30c1e3c78fL, + 0xd1e00c6891bd5c04L, + 0xae8efca0a188d57dL, + 0xe65f088b6997f879L, + 0x9931f84359a27100L, + 0x1882e91b09fcea8bL, + 0x67ec19d339c963f2L, + 0xd75adabd7a6e2d6fL, + 0xa8342a754a5ba416L, + 0x29873b2d1a053f9dL, + 0x56e9cbe52a30b6e4L, + 0x1e383fcee22f9be0L, + 0x6156cf06d21a1299L, + 0xe0e5de5e82448912L, + 0x9f8b2e96b271006bL, + 0x71463609127ad31aL, + 0x0e28c6c1224f5a63L, + 0x8f9bd7997211c1e8L, + 0xf0f5275142244891L, + 0xb824d37a8a3b6595L, + 0xc74a23b2ba0eececL, + 0x46f932eaea507767L, + 0x3997c222da65fe1eL, + 0xafba2586f2d042eeL, + 0xd0d4d54ec2e5cb97L, + 0x5167c41692bb501cL, + 0x2e0934dea28ed965L, + 0x66d8c0f56a91f461L, + 0x19b6303d5aa47d18L, + 0x980521650afae693L, + 0xe76bd1ad3acf6feaL, + 0x09a6c9329ac4bc9bL, + 0x76c839faaaf135e2L, + 0xf77b28a2faafae69L, + 0x8815d86aca9a2710L, + 0xc0c42c4102850a14L, + 0xbfaadc8932b0836dL, + 0x3e19cdd162ee18e6L, + 0x41773d1952db919fL, + 0x269b24ca6b12f26dL, + 0x59f5d4025b277b14L, + 0xd846c55a0b79e09fL, + 0xa72835923b4c69e6L, + 0xeff9c1b9f35344e2L, + 0x90973171c366cd9bL, + 0x1124202993385610L, + 0x6e4ad0e1a30ddf69L, + 0x8087c87e03060c18L, + 0xffe938b633338561L, + 0x7e5a29ee636d1eeaL, + 0x0134d92653589793L, + 0x49e52d0d9b47ba97L, + 0x368bddc5ab7233eeL, + 0xb738cc9dfb2ca865L, + 0xc8563c55cb19211cL, + 0x5e7bdbf1e3ac9decL, + 0x21152b39d3991495L, + 0xa0a63a6183c78f1eL, + 0xdfc8caa9b3f20667L, + 0x97193e827bed2b63L, + 0xe877ce4a4bd8a21aL, + 0x69c4df121b863991L, + 0x16aa2fda2bb3b0e8L, + 0xf86737458bb86399L, + 0x8709c78dbb8deae0L, + 0x06bad6d5ebd3716bL, + 0x79d4261ddbe6f812L, + 0x3105d23613f9d516L, + 0x4e6b22fe23cc5c6fL, + 0xcfd833a67392c7e4L, + 0xb0b6c36e43a74e9dL, + 0x9a6c9329ac4bc9b5L, + 0xe50263e19c7e40ccL, + 0x64b172b9cc20db47L, + 0x1bdf8271fc15523eL, + 0x530e765a340a7f3aL, + 0x2c608692043ff643L, + 0xadd397ca54616dc8L, + 0xd2bd67026454e4b1L, + 0x3c707f9dc45f37c0L, + 0x431e8f55f46abeb9L, + 0xc2ad9e0da4342532L, + 0xbdc36ec59401ac4bL, + 0xf5129aee5c1e814fL, + 0x8a7c6a266c2b0836L, + 0x0bcf7b7e3c7593bdL, + 0x74a18bb60c401ac4L, + 0xe28c6c1224f5a634L, + 0x9de29cda14c02f4dL, + 0x1c518d82449eb4c6L, + 0x633f7d4a74ab3dbfL, + 0x2bee8961bcb410bbL, + 0x548079a98c8199c2L, + 0xd53368f1dcdf0249L, + 0xaa5d9839ecea8b30L, + 0x449080a64ce15841L, + 0x3bfe706e7cd4d138L, + 0xba4d61362c8a4ab3L, + 0xc52391fe1cbfc3caL, + 0x8df265d5d4a0eeceL, + 0xf29c951de49567b7L, + 0x732f8445b4cbfc3cL, + 0x0c41748d84fe7545L, + 0x6bad6d5ebd3716b7L, + 0x14c39d968d029fceL, + 0x95708ccedd5c0445L, + 0xea1e7c06ed698d3cL, + 0xa2cf882d2576a038L, + 0xdda178e515432941L, + 0x5c1269bd451db2caL, + 0x237c997575283bb3L, + 0xcdb181ead523e8c2L, + 0xb2df7122e51661bbL, + 0x336c607ab548fa30L, + 0x4c0290b2857d7349L, + 0x04d364994d625e4dL, + 0x7bbd94517d57d734L, + 0xfa0e85092d094cbfL, + 0x856075c11d3cc5c6L, + 0x134d926535897936L, + 0x6c2362ad05bcf04fL, + 0xed9073f555e26bc4L, + 0x92fe833d65d7e2bdL, + 0xda2f7716adc8cfb9L, + 0xa54187de9dfd46c0L, + 0x24f29686cda3dd4bL, + 0x5b9c664efd965432L, + 0xb5517ed15d9d8743L, + 0xca3f8e196da80e3aL, + 0x4b8c9f413df695b1L, + 0x34e26f890dc31cc8L, + 0x7c339ba2c5dc31ccL, + 0x035d6b6af5e9b8b5L, + 0x82ee7a32a5b7233eL, + 0xfd808afa9582aa47L, + 0x4d364994d625e4daL, + 0x3258b95ce6106da3L, + 0xb3eba804b64ef628L, + 0xcc8558cc867b7f51L, + 0x8454ace74e645255L, + 0xfb3a5c2f7e51db2cL, + 0x7a894d772e0f40a7L, + 0x05e7bdbf1e3ac9deL, + 0xeb2aa520be311aafL, + 0x944455e88e0493d6L, + 0x15f744b0de5a085dL, + 0x6a99b478ee6f8124L, + 0x224840532670ac20L, + 0x5d26b09b16452559L, + 0xdc95a1c3461bbed2L, + 0xa3fb510b762e37abL, + 0x35d6b6af5e9b8b5bL, + 0x4ab846676eae0222L, + 0xcb0b573f3ef099a9L, + 0xb465a7f70ec510d0L, + 0xfcb453dcc6da3dd4L, + 0x83daa314f6efb4adL, + 0x0269b24ca6b12f26L, + 0x7d0742849684a65fL, + 0x93ca5a1b368f752eL, + 0xeca4aad306bafc57L, + 0x6d17bb8b56e467dcL, + 0x12794b4366d1eea5L, + 0x5aa8bf68aecec3a1L, + 0x25c64fa09efb4ad8L, + 0xa4755ef8cea5d153L, + 0xdb1bae30fe90582aL, + 0xbcf7b7e3c7593bd8L, + 0xc399472bf76cb2a1L, + 0x422a5673a732292aL, + 0x3d44a6bb9707a053L, + 0x759552905f188d57L, + 0x0afba2586f2d042eL, + 0x8b48b3003f739fa5L, + 0xf42643c80f4616dcL, + 0x1aeb5b57af4dc5adL, + 0x6585ab9f9f784cd4L, + 0xe436bac7cf26d75fL, + 0x9b584a0fff135e26L, + 0xd389be24370c7322L, + 0xace74eec0739fa5bL, + 0x2d545fb4576761d0L, + 0x523aaf7c6752e8a9L, + 0xc41748d84fe75459L, + 0xbb79b8107fd2dd20L, + 0x3acaa9482f8c46abL, + 0x45a459801fb9cfd2L, + 0x0d75adabd7a6e2d6L, + 0x721b5d63e7936bafL, + 0xf3a84c3bb7cdf024L, + 0x8cc6bcf387f8795dL, + 0x620ba46c27f3aa2cL, + 0x1d6554a417c62355L, + 0x9cd645fc4798b8deL, + 0xe3b8b53477ad31a7L, + 0xab69411fbfb21ca3L, + 0xd407b1d78f8795daL, + 0x55b4a08fdfd90e51L, + 0x2ada5047efec8728L, }; + + private static final long[] M_U32 = { + 0x0000000000000000L, + 0xb8c533c1177eb231L, + 0x455341d1766af709L, + 0xfd96721061144538L, + 0x8aa683a2ecd5ee12L, + 0x3263b063fbab5c23L, + 0xcff5c2739abf191bL, + 0x7730f1b28dc1ab2aL, + 0x21942116813c4f4fL, + 0x995112d79642fd7eL, + 0x64c760c7f756b846L, + 0xdc025306e0280a77L, + 0xab32a2b46de9a15dL, + 0x13f791757a97136cL, + 0xee61e3651b835654L, + 0x56a4d0a40cfde465L, + 0x4328422d02789e9eL, + 0xfbed71ec15062cafL, + 0x067b03fc74126997L, + 0xbebe303d636cdba6L, + 0xc98ec18feead708cL, + 0x714bf24ef9d3c2bdL, + 0x8cdd805e98c78785L, + 0x3418b39f8fb935b4L, + 0x62bc633b8344d1d1L, + 0xda7950fa943a63e0L, + 0x27ef22eaf52e26d8L, + 0x9f2a112be25094e9L, + 0xe81ae0996f913fc3L, + 0x50dfd35878ef8df2L, + 0xad49a14819fbc8caL, + 0x158c92890e857afbL, + 0x8650845a04f13d3cL, + 0x3e95b79b138f8f0dL, + 0xc303c58b729bca35L, + 0x7bc6f64a65e57804L, + 0x0cf607f8e824d32eL, + 0xb4333439ff5a611fL, + 0x49a546299e4e2427L, + 0xf16075e889309616L, + 0xa7c4a54c85cd7273L, + 0x1f01968d92b3c042L, + 0xe297e49df3a7857aL, + 0x5a52d75ce4d9374bL, + 0x2d6226ee69189c61L, + 0x95a7152f7e662e50L, + 0x6831673f1f726b68L, + 0xd0f454fe080cd959L, + 0xc578c6770689a3a2L, + 0x7dbdf5b611f71193L, + 0x802b87a670e354abL, + 0x38eeb467679de69aL, + 0x4fde45d5ea5c4db0L, + 0xf71b7614fd22ff81L, + 0x0a8d04049c36bab9L, + 0xb24837c58b480888L, + 0xe4ece76187b5ecedL, + 0x5c29d4a090cb5edcL, + 0xa1bfa6b0f1df1be4L, + 0x197a9571e6a1a9d5L, + 0x6e4a64c36b6002ffL, + 0xd68f57027c1eb0ceL, + 0x2b1925121d0af5f6L, + 0x93dc16d30a7447c7L, + 0x38782ee75175e913L, + 0x80bd1d26460b5b22L, + 0x7d2b6f36271f1e1aL, + 0xc5ee5cf73061ac2bL, + 0xb2dead45bda00701L, + 0x0a1b9e84aadeb530L, + 0xf78dec94cbcaf008L, + 0x4f48df55dcb44239L, + 0x19ec0ff1d049a65cL, + 0xa1293c30c737146dL, + 0x5cbf4e20a6235155L, + 0xe47a7de1b15de364L, + 0x934a8c533c9c484eL, + 0x2b8fbf922be2fa7fL, + 0xd619cd824af6bf47L, + 0x6edcfe435d880d76L, + 0x7b506cca530d778dL, + 0xc3955f0b4473c5bcL, + 0x3e032d1b25678084L, + 0x86c61eda321932b5L, + 0xf1f6ef68bfd8999fL, + 0x4933dca9a8a62baeL, + 0xb4a5aeb9c9b26e96L, + 0x0c609d78deccdca7L, + 0x5ac44ddcd23138c2L, + 0xe2017e1dc54f8af3L, + 0x1f970c0da45bcfcbL, + 0xa7523fccb3257dfaL, + 0xd062ce7e3ee4d6d0L, + 0x68a7fdbf299a64e1L, + 0x95318faf488e21d9L, + 0x2df4bc6e5ff093e8L, + 0xbe28aabd5584d42fL, + 0x06ed997c42fa661eL, + 0xfb7beb6c23ee2326L, + 0x43bed8ad34909117L, + 0x348e291fb9513a3dL, + 0x8c4b1adeae2f880cL, + 0x71dd68cecf3bcd34L, + 0xc9185b0fd8457f05L, + 0x9fbc8babd4b89b60L, + 0x2779b86ac3c62951L, + 0xdaefca7aa2d26c69L, + 0x622af9bbb5acde58L, + 0x151a0809386d7572L, + 0xaddf3bc82f13c743L, + 0x504949d84e07827bL, + 0xe88c7a195979304aL, + 0xfd00e89057fc4ab1L, + 0x45c5db514082f880L, + 0xb853a9412196bdb8L, + 0x00969a8036e80f89L, + 0x77a66b32bb29a4a3L, + 0xcf6358f3ac571692L, + 0x32f52ae3cd4353aaL, + 0x8a301922da3de19bL, + 0xdc94c986d6c005feL, + 0x6451fa47c1beb7cfL, + 0x99c78857a0aaf2f7L, + 0x2102bb96b7d440c6L, + 0x56324a243a15ebecL, + 0xeef779e52d6b59ddL, + 0x13610bf54c7f1ce5L, + 0xaba438345b01aed4L, + 0x70f05dcea2ebd226L, + 0xc8356e0fb5956017L, + 0x35a31c1fd481252fL, + 0x8d662fdec3ff971eL, + 0xfa56de6c4e3e3c34L, + 0x4293edad59408e05L, + 0xbf059fbd3854cb3dL, + 0x07c0ac7c2f2a790cL, + 0x51647cd823d79d69L, + 0xe9a14f1934a92f58L, + 0x14373d0955bd6a60L, + 0xacf20ec842c3d851L, + 0xdbc2ff7acf02737bL, + 0x6307ccbbd87cc14aL, + 0x9e91beabb9688472L, + 0x26548d6aae163643L, + 0x33d81fe3a0934cb8L, + 0x8b1d2c22b7edfe89L, + 0x768b5e32d6f9bbb1L, + 0xce4e6df3c1870980L, + 0xb97e9c414c46a2aaL, + 0x01bbaf805b38109bL, + 0xfc2ddd903a2c55a3L, + 0x44e8ee512d52e792L, + 0x124c3ef521af03f7L, + 0xaa890d3436d1b1c6L, + 0x571f7f2457c5f4feL, + 0xefda4ce540bb46cfL, + 0x98eabd57cd7aede5L, + 0x202f8e96da045fd4L, + 0xddb9fc86bb101aecL, + 0x657ccf47ac6ea8ddL, + 0xf6a0d994a61aef1aL, + 0x4e65ea55b1645d2bL, + 0xb3f39845d0701813L, + 0x0b36ab84c70eaa22L, + 0x7c065a364acf0108L, + 0xc4c369f75db1b339L, + 0x39551be73ca5f601L, + 0x819028262bdb4430L, + 0xd734f8822726a055L, + 0x6ff1cb4330581264L, + 0x9267b953514c575cL, + 0x2aa28a924632e56dL, + 0x5d927b20cbf34e47L, + 0xe55748e1dc8dfc76L, + 0x18c13af1bd99b94eL, + 0xa0040930aae70b7fL, + 0xb5889bb9a4627184L, + 0x0d4da878b31cc3b5L, + 0xf0dbda68d208868dL, + 0x481ee9a9c57634bcL, + 0x3f2e181b48b79f96L, + 0x87eb2bda5fc92da7L, + 0x7a7d59ca3edd689fL, + 0xc2b86a0b29a3daaeL, + 0x941cbaaf255e3ecbL, + 0x2cd9896e32208cfaL, + 0xd14ffb7e5334c9c2L, + 0x698ac8bf444a7bf3L, + 0x1eba390dc98bd0d9L, + 0xa67f0accdef562e8L, + 0x5be978dcbfe127d0L, + 0xe32c4b1da89f95e1L, + 0x48887329f39e3b35L, + 0xf04d40e8e4e08904L, + 0x0ddb32f885f4cc3cL, + 0xb51e0139928a7e0dL, + 0xc22ef08b1f4bd527L, + 0x7aebc34a08356716L, + 0x877db15a6921222eL, + 0x3fb8829b7e5f901fL, + 0x691c523f72a2747aL, + 0xd1d961fe65dcc64bL, + 0x2c4f13ee04c88373L, + 0x948a202f13b63142L, + 0xe3bad19d9e779a68L, + 0x5b7fe25c89092859L, + 0xa6e9904ce81d6d61L, + 0x1e2ca38dff63df50L, + 0x0ba03104f1e6a5abL, + 0xb36502c5e698179aL, + 0x4ef370d5878c52a2L, + 0xf636431490f2e093L, + 0x8106b2a61d334bb9L, + 0x39c381670a4df988L, + 0xc455f3776b59bcb0L, + 0x7c90c0b67c270e81L, + 0x2a34101270daeae4L, + 0x92f123d367a458d5L, + 0x6f6751c306b01dedL, + 0xd7a2620211ceafdcL, + 0xa09293b09c0f04f6L, + 0x1857a0718b71b6c7L, + 0xe5c1d261ea65f3ffL, + 0x5d04e1a0fd1b41ceL, + 0xced8f773f76f0609L, + 0x761dc4b2e011b438L, + 0x8b8bb6a28105f100L, + 0x334e8563967b4331L, + 0x447e74d11bbae81bL, + 0xfcbb47100cc45a2aL, + 0x012d35006dd01f12L, + 0xb9e806c17aaead23L, + 0xef4cd66576534946L, + 0x5789e5a4612dfb77L, + 0xaa1f97b40039be4fL, + 0x12daa47517470c7eL, + 0x65ea55c79a86a754L, + 0xdd2f66068df81565L, + 0x20b91416ecec505dL, + 0x987c27d7fb92e26cL, + 0x8df0b55ef5179897L, + 0x3535869fe2692aa6L, + 0xc8a3f48f837d6f9eL, + 0x7066c74e9403ddafL, + 0x075636fc19c27685L, + 0xbf93053d0ebcc4b4L, + 0x4205772d6fa8818cL, + 0xfac044ec78d633bdL, + 0xac649448742bd7d8L, + 0x14a1a789635565e9L, + 0xe937d599024120d1L, + 0x51f2e658153f92e0L, + 0x26c217ea98fe39caL, + 0x9e07242b8f808bfbL, + 0x6391563bee94cec3L, + 0xdb5465faf9ea7cf2L, + + 0x0000000000000000L, + 0xf6f734b768e04748L, + 0xd9374f3d89571dfbL, + 0x2fc07b8ae1b75ab3L, + 0x86b7b8284a39a89dL, + 0x70408c9f22d9efd5L, + 0x5f80f715c36eb566L, + 0xa977c3a2ab8ef22eL, + 0x39b65603cce4c251L, + 0xcf4162b4a4048519L, + 0xe081193e45b3dfaaL, + 0x16762d892d5398e2L, + 0xbf01ee2b86dd6accL, + 0x49f6da9cee3d2d84L, + 0x6636a1160f8a7737L, + 0x90c195a1676a307fL, + 0x736cac0799c984a2L, + 0x859b98b0f129c3eaL, + 0xaa5be33a109e9959L, + 0x5cacd78d787ede11L, + 0xf5db142fd3f02c3fL, + 0x032c2098bb106b77L, + 0x2cec5b125aa731c4L, + 0xda1b6fa53247768cL, + 0x4adafa04552d46f3L, + 0xbc2dceb33dcd01bbL, + 0x93edb539dc7a5b08L, + 0x651a818eb49a1c40L, + 0xcc6d422c1f14ee6eL, + 0x3a9a769b77f4a926L, + 0x155a0d119643f395L, + 0xe3ad39a6fea3b4ddL, + 0xe6d9580f33930944L, + 0x102e6cb85b734e0cL, + 0x3fee1732bac414bfL, + 0xc9192385d22453f7L, + 0x606ee02779aaa1d9L, + 0x9699d490114ae691L, + 0xb959af1af0fdbc22L, + 0x4fae9bad981dfb6aL, + 0xdf6f0e0cff77cb15L, + 0x29983abb97978c5dL, + 0x065841317620d6eeL, + 0xf0af75861ec091a6L, + 0x59d8b624b54e6388L, + 0xaf2f8293ddae24c0L, + 0x80eff9193c197e73L, + 0x7618cdae54f9393bL, + 0x95b5f408aa5a8de6L, + 0x6342c0bfc2bacaaeL, + 0x4c82bb35230d901dL, + 0xba758f824bedd755L, + 0x13024c20e063257bL, + 0xe5f5789788836233L, + 0xca35031d69343880L, + 0x3cc237aa01d47fc8L, + 0xac03a20b66be4fb7L, + 0x5af496bc0e5e08ffL, + 0x7534ed36efe9524cL, + 0x83c3d98187091504L, + 0x2ab41a232c87e72aL, + 0xdc432e944467a062L, + 0xf383551ea5d0fad1L, + 0x057461a9cd30bd99L, + 0xf96b964d3fb181e3L, + 0x0f9ca2fa5751c6abL, + 0x205cd970b6e69c18L, + 0xd6abedc7de06db50L, + 0x7fdc2e657588297eL, + 0x892b1ad21d686e36L, + 0xa6eb6158fcdf3485L, + 0x501c55ef943f73cdL, + 0xc0ddc04ef35543b2L, + 0x362af4f99bb504faL, + 0x19ea8f737a025e49L, + 0xef1dbbc412e21901L, + 0x466a7866b96ceb2fL, + 0xb09d4cd1d18cac67L, + 0x9f5d375b303bf6d4L, + 0x69aa03ec58dbb19cL, + 0x8a073a4aa6780541L, + 0x7cf00efdce984209L, + 0x533075772f2f18baL, + 0xa5c741c047cf5ff2L, + 0x0cb08262ec41addcL, + 0xfa47b6d584a1ea94L, + 0xd587cd5f6516b027L, + 0x2370f9e80df6f76fL, + 0xb3b16c496a9cc710L, + 0x454658fe027c8058L, + 0x6a862374e3cbdaebL, + 0x9c7117c38b2b9da3L, + 0x3506d46120a56f8dL, + 0xc3f1e0d6484528c5L, + 0xec319b5ca9f27276L, + 0x1ac6afebc112353eL, + 0x1fb2ce420c2288a7L, + 0xe945faf564c2cfefL, + 0xc685817f8575955cL, + 0x3072b5c8ed95d214L, + 0x9905766a461b203aL, + 0x6ff242dd2efb6772L, + 0x40323957cf4c3dc1L, + 0xb6c50de0a7ac7a89L, + 0x26049841c0c64af6L, + 0xd0f3acf6a8260dbeL, + 0xff33d77c4991570dL, + 0x09c4e3cb21711045L, + 0xa0b320698affe26bL, + 0x564414dee21fa523L, + 0x79846f5403a8ff90L, + 0x8f735be36b48b8d8L, + 0x6cde624595eb0c05L, + 0x9a2956f2fd0b4b4dL, + 0xb5e92d781cbc11feL, + 0x431e19cf745c56b6L, + 0xea69da6ddfd2a498L, + 0x1c9eeedab732e3d0L, + 0x335e95505685b963L, + 0xc5a9a1e73e65fe2bL, + 0x55683446590fce54L, + 0xa39f00f131ef891cL, + 0x8c5f7b7bd058d3afL, + 0x7aa84fccb8b894e7L, + 0xd3df8c6e133666c9L, + 0x2528b8d97bd62181L, + 0x0ae8c3539a617b32L, + 0xfc1ff7e4f2813c7aL, + 0xc60e0ac927f490adL, + 0x30f93e7e4f14d7e5L, + 0x1f3945f4aea38d56L, + 0xe9ce7143c643ca1eL, + 0x40b9b2e16dcd3830L, + 0xb64e8656052d7f78L, + 0x998efddce49a25cbL, + 0x6f79c96b8c7a6283L, + 0xffb85ccaeb1052fcL, + 0x094f687d83f015b4L, + 0x268f13f762474f07L, + 0xd07827400aa7084fL, + 0x790fe4e2a129fa61L, + 0x8ff8d055c9c9bd29L, + 0xa038abdf287ee79aL, + 0x56cf9f68409ea0d2L, + 0xb562a6cebe3d140fL, + 0x43959279d6dd5347L, + 0x6c55e9f3376a09f4L, + 0x9aa2dd445f8a4ebcL, + 0x33d51ee6f404bc92L, + 0xc5222a519ce4fbdaL, + 0xeae251db7d53a169L, + 0x1c15656c15b3e621L, + 0x8cd4f0cd72d9d65eL, + 0x7a23c47a1a399116L, + 0x55e3bff0fb8ecba5L, + 0xa3148b47936e8cedL, + 0x0a6348e538e07ec3L, + 0xfc947c525000398bL, + 0xd35407d8b1b76338L, + 0x25a3336fd9572470L, + 0x20d752c6146799e9L, + 0xd62066717c87dea1L, + 0xf9e01dfb9d308412L, + 0x0f17294cf5d0c35aL, + 0xa660eaee5e5e3174L, + 0x5097de5936be763cL, + 0x7f57a5d3d7092c8fL, + 0x89a09164bfe96bc7L, + 0x196104c5d8835bb8L, + 0xef963072b0631cf0L, + 0xc0564bf851d44643L, + 0x36a17f4f3934010bL, + 0x9fd6bced92baf325L, + 0x6921885afa5ab46dL, + 0x46e1f3d01bedeedeL, + 0xb016c767730da996L, + 0x53bbfec18dae1d4bL, + 0xa54cca76e54e5a03L, + 0x8a8cb1fc04f900b0L, + 0x7c7b854b6c1947f8L, + 0xd50c46e9c797b5d6L, + 0x23fb725eaf77f29eL, + 0x0c3b09d44ec0a82dL, + 0xfacc3d632620ef65L, + 0x6a0da8c2414adf1aL, + 0x9cfa9c7529aa9852L, + 0xb33ae7ffc81dc2e1L, + 0x45cdd348a0fd85a9L, + 0xecba10ea0b737787L, + 0x1a4d245d639330cfL, + 0x358d5fd782246a7cL, + 0xc37a6b60eac42d34L, + 0x3f659c841845114eL, + 0xc992a83370a55606L, + 0xe652d3b991120cb5L, + 0x10a5e70ef9f24bfdL, + 0xb9d224ac527cb9d3L, + 0x4f25101b3a9cfe9bL, + 0x60e56b91db2ba428L, + 0x96125f26b3cbe360L, + 0x06d3ca87d4a1d31fL, + 0xf024fe30bc419457L, + 0xdfe485ba5df6cee4L, + 0x2913b10d351689acL, + 0x806472af9e987b82L, + 0x76934618f6783ccaL, + 0x59533d9217cf6679L, + 0xafa409257f2f2131L, + 0x4c093083818c95ecL, + 0xbafe0434e96cd2a4L, + 0x953e7fbe08db8817L, + 0x63c94b09603bcf5fL, + 0xcabe88abcbb53d71L, + 0x3c49bc1ca3557a39L, + 0x1389c79642e2208aL, + 0xe57ef3212a0267c2L, + 0x75bf66804d6857bdL, + 0x83485237258810f5L, + 0xac8829bdc43f4a46L, + 0x5a7f1d0aacdf0d0eL, + 0xf308dea80751ff20L, + 0x05ffea1f6fb1b868L, + 0x2a3f91958e06e2dbL, + 0xdcc8a522e6e6a593L, + 0xd9bcc48b2bd6180aL, + 0x2f4bf03c43365f42L, + 0x008b8bb6a28105f1L, + 0xf67cbf01ca6142b9L, + 0x5f0b7ca361efb097L, + 0xa9fc4814090ff7dfL, + 0x863c339ee8b8ad6cL, + 0x70cb07298058ea24L, + 0xe00a9288e732da5bL, + 0x16fda63f8fd29d13L, + 0x393dddb56e65c7a0L, + 0xcfcae902068580e8L, + 0x66bd2aa0ad0b72c6L, + 0x904a1e17c5eb358eL, + 0xbf8a659d245c6f3dL, + 0x497d512a4cbc2875L, + 0xaad0688cb21f9ca8L, + 0x5c275c3bdaffdbe0L, + 0x73e727b13b488153L, + 0x8510130653a8c61bL, + 0x2c67d0a4f8263435L, + 0xda90e41390c6737dL, + 0xf5509f99717129ceL, + 0x03a7ab2e19916e86L, + 0x93663e8f7efb5ef9L, + 0x65910a38161b19b1L, + 0x4a5171b2f7ac4302L, + 0xbca645059f4c044aL, + 0x15d186a734c2f664L, + 0xe326b2105c22b12cL, + 0xcce6c99abd95eb9fL, + 0x3a11fd2dd575acd7L, + + 0x0000000000000000L, + 0x71b0c13da512335dL, + 0xe361827b4a2466baL, + 0x92d14346ef3655e7L, + 0xf21a22a5ccdf5e1fL, + 0x83aae39869cd6d42L, + 0x117ba0de86fb38a5L, + 0x60cb61e323e90bf8L, + 0xd0ed6318c1292f55L, + 0xa15da225643b1c08L, + 0x338ce1638b0d49efL, + 0x423c205e2e1f7ab2L, + 0x22f741bd0df6714aL, + 0x53478080a8e44217L, + 0xc196c3c647d217f0L, + 0xb02602fbe2c024adL, + 0x9503e062dac5cdc1L, + 0xe4b3215f7fd7fe9cL, + 0x7662621990e1ab7bL, + 0x07d2a32435f39826L, + 0x6719c2c7161a93deL, + 0x16a903fab308a083L, + 0x847840bc5c3ef564L, + 0xf5c88181f92cc639L, + 0x45ee837a1bece294L, + 0x345e4247befed1c9L, + 0xa68f010151c8842eL, + 0xd73fc03cf4dab773L, + 0xb7f4a1dfd733bc8bL, + 0xc64460e272218fd6L, + 0x549523a49d17da31L, + 0x2525e2993805e96cL, + 0x1edee696ed1c08e9L, + 0x6f6e27ab480e3bb4L, + 0xfdbf64eda7386e53L, + 0x8c0fa5d0022a5d0eL, + 0xecc4c43321c356f6L, + 0x9d74050e84d165abL, + 0x0fa546486be7304cL, + 0x7e158775cef50311L, + 0xce33858e2c3527bcL, + 0xbf8344b3892714e1L, + 0x2d5207f566114106L, + 0x5ce2c6c8c303725bL, + 0x3c29a72be0ea79a3L, + 0x4d99661645f84afeL, + 0xdf482550aace1f19L, + 0xaef8e46d0fdc2c44L, + 0x8bdd06f437d9c528L, + 0xfa6dc7c992cbf675L, + 0x68bc848f7dfda392L, + 0x190c45b2d8ef90cfL, + 0x79c72451fb069b37L, + 0x0877e56c5e14a86aL, + 0x9aa6a62ab122fd8dL, + 0xeb1667171430ced0L, + 0x5b3065ecf6f0ea7dL, + 0x2a80a4d153e2d920L, + 0xb851e797bcd48cc7L, + 0xc9e126aa19c6bf9aL, + 0xa92a47493a2fb462L, + 0xd89a86749f3d873fL, + 0x4a4bc532700bd2d8L, + 0x3bfb040fd519e185L, + 0x3dbdcd2dda3811d2L, + 0x4c0d0c107f2a228fL, + 0xdedc4f56901c7768L, + 0xaf6c8e6b350e4435L, + 0xcfa7ef8816e74fcdL, + 0xbe172eb5b3f57c90L, + 0x2cc66df35cc32977L, + 0x5d76accef9d11a2aL, + 0xed50ae351b113e87L, + 0x9ce06f08be030ddaL, + 0x0e312c4e5135583dL, + 0x7f81ed73f4276b60L, + 0x1f4a8c90d7ce6098L, + 0x6efa4dad72dc53c5L, + 0xfc2b0eeb9dea0622L, + 0x8d9bcfd638f8357fL, + 0xa8be2d4f00fddc13L, + 0xd90eec72a5efef4eL, + 0x4bdfaf344ad9baa9L, + 0x3a6f6e09efcb89f4L, + 0x5aa40feacc22820cL, + 0x2b14ced76930b151L, + 0xb9c58d918606e4b6L, + 0xc8754cac2314d7ebL, + 0x78534e57c1d4f346L, + 0x09e38f6a64c6c01bL, + 0x9b32cc2c8bf095fcL, + 0xea820d112ee2a6a1L, + 0x8a496cf20d0bad59L, + 0xfbf9adcfa8199e04L, + 0x6928ee89472fcbe3L, + 0x18982fb4e23df8beL, + 0x23632bbb3724193bL, + 0x52d3ea8692362a66L, + 0xc002a9c07d007f81L, + 0xb1b268fdd8124cdcL, + 0xd179091efbfb4724L, + 0xa0c9c8235ee97479L, + 0x32188b65b1df219eL, + 0x43a84a5814cd12c3L, + 0xf38e48a3f60d366eL, + 0x823e899e531f0533L, + 0x10efcad8bc2950d4L, + 0x615f0be5193b6389L, + 0x01946a063ad26871L, + 0x7024ab3b9fc05b2cL, + 0xe2f5e87d70f60ecbL, + 0x93452940d5e43d96L, + 0xb660cbd9ede1d4faL, + 0xc7d00ae448f3e7a7L, + 0x550149a2a7c5b240L, + 0x24b1889f02d7811dL, + 0x447ae97c213e8ae5L, + 0x35ca2841842cb9b8L, + 0xa71b6b076b1aec5fL, + 0xd6abaa3ace08df02L, + 0x668da8c12cc8fbafL, + 0x173d69fc89dac8f2L, + 0x85ec2aba66ec9d15L, + 0xf45ceb87c3feae48L, + 0x94978a64e017a5b0L, + 0xe5274b59450596edL, + 0x77f6081faa33c30aL, + 0x0646c9220f21f057L, + 0x7b7b9a5bb47023a4L, + 0x0acb5b66116210f9L, + 0x981a1820fe54451eL, + 0xe9aad91d5b467643L, + 0x8961b8fe78af7dbbL, + 0xf8d179c3ddbd4ee6L, + 0x6a003a85328b1b01L, + 0x1bb0fbb89799285cL, + 0xab96f94375590cf1L, + 0xda26387ed04b3facL, + 0x48f77b383f7d6a4bL, + 0x3947ba059a6f5916L, + 0x598cdbe6b98652eeL, + 0x283c1adb1c9461b3L, + 0xbaed599df3a23454L, + 0xcb5d98a056b00709L, + 0xee787a396eb5ee65L, + 0x9fc8bb04cba7dd38L, + 0x0d19f842249188dfL, + 0x7ca9397f8183bb82L, + 0x1c62589ca26ab07aL, + 0x6dd299a107788327L, + 0xff03dae7e84ed6c0L, + 0x8eb31bda4d5ce59dL, + 0x3e951921af9cc130L, + 0x4f25d81c0a8ef26dL, + 0xddf49b5ae5b8a78aL, + 0xac445a6740aa94d7L, + 0xcc8f3b8463439f2fL, + 0xbd3ffab9c651ac72L, + 0x2feeb9ff2967f995L, + 0x5e5e78c28c75cac8L, + 0x65a57ccd596c2b4dL, + 0x1415bdf0fc7e1810L, + 0x86c4feb613484df7L, + 0xf7743f8bb65a7eaaL, + 0x97bf5e6895b37552L, + 0xe60f9f5530a1460fL, + 0x74dedc13df9713e8L, + 0x056e1d2e7a8520b5L, + 0xb5481fd598450418L, + 0xc4f8dee83d573745L, + 0x56299daed26162a2L, + 0x27995c93777351ffL, + 0x47523d70549a5a07L, + 0x36e2fc4df188695aL, + 0xa433bf0b1ebe3cbdL, + 0xd5837e36bbac0fe0L, + 0xf0a69caf83a9e68cL, + 0x81165d9226bbd5d1L, + 0x13c71ed4c98d8036L, + 0x6277dfe96c9fb36bL, + 0x02bcbe0a4f76b893L, + 0x730c7f37ea648bceL, + 0xe1dd3c710552de29L, + 0x906dfd4ca040ed74L, + 0x204bffb74280c9d9L, + 0x51fb3e8ae792fa84L, + 0xc32a7dcc08a4af63L, + 0xb29abcf1adb69c3eL, + 0xd251dd128e5f97c6L, + 0xa3e11c2f2b4da49bL, + 0x31305f69c47bf17cL, + 0x40809e546169c221L, + 0x46c657766e483276L, + 0x3776964bcb5a012bL, + 0xa5a7d50d246c54ccL, + 0xd4171430817e6791L, + 0xb4dc75d3a2976c69L, + 0xc56cb4ee07855f34L, + 0x57bdf7a8e8b30ad3L, + 0x260d36954da1398eL, + 0x962b346eaf611d23L, + 0xe79bf5530a732e7eL, + 0x754ab615e5457b99L, + 0x04fa7728405748c4L, + 0x643116cb63be433cL, + 0x1581d7f6c6ac7061L, + 0x875094b0299a2586L, + 0xf6e0558d8c8816dbL, + 0xd3c5b714b48dffb7L, + 0xa2757629119fcceaL, + 0x30a4356ffea9990dL, + 0x4114f4525bbbaa50L, + 0x21df95b17852a1a8L, + 0x506f548cdd4092f5L, + 0xc2be17ca3276c712L, + 0xb30ed6f79764f44fL, + 0x0328d40c75a4d0e2L, + 0x72981531d0b6e3bfL, + 0xe04956773f80b658L, + 0x91f9974a9a928505L, + 0xf132f6a9b97b8efdL, + 0x808237941c69bda0L, + 0x125374d2f35fe847L, + 0x63e3b5ef564ddb1aL, + 0x5818b1e083543a9fL, + 0x29a870dd264609c2L, + 0xbb79339bc9705c25L, + 0xcac9f2a66c626f78L, + 0xaa0293454f8b6480L, + 0xdbb25278ea9957ddL, + 0x4963113e05af023aL, + 0x38d3d003a0bd3167L, + 0x88f5d2f8427d15caL, + 0xf94513c5e76f2697L, + 0x6b94508308597370L, + 0x1a2491bead4b402dL, + 0x7aeff05d8ea24bd5L, + 0x0b5f31602bb07888L, + 0x998e7226c4862d6fL, + 0xe83eb31b61941e32L, + 0xcd1b51825991f75eL, + 0xbcab90bffc83c403L, + 0x2e7ad3f913b591e4L, + 0x5fca12c4b6a7a2b9L, + 0x3f017327954ea941L, + 0x4eb1b21a305c9a1cL, + 0xdc60f15cdf6acffbL, + 0xadd030617a78fca6L, + 0x1df6329a98b8d80bL, + 0x6c46f3a73daaeb56L, + 0xfe97b0e1d29cbeb1L, + 0x8f2771dc778e8decL, + 0xefec103f54678614L, + 0x9e5cd102f175b549L, + 0x0c8d92441e43e0aeL, + 0x7d3d5379bb51d3f3L, + + 0x0000000000000000L, + 0xbfdb6c480f15915eL, + 0x4b6ffec346bcb1d7L, + 0xf4b4928b49a92089L, + 0x96dffd868d7963aeL, + 0x290491ce826cf2f0L, + 0xddb00345cbc5d279L, + 0x626b6f0dc4d04327L, + 0x1966dd5e42655437L, + 0xa6bdb1164d70c569L, + 0x5209239d04d9e5e0L, + 0xedd24fd50bcc74beL, + 0x8fb920d8cf1c3799L, + 0x30624c90c009a6c7L, + 0xc4d6de1b89a0864eL, + 0x7b0db25386b51710L, + 0x32cdbabc84caa86eL, + 0x8d16d6f48bdf3930L, + 0x79a2447fc27619b9L, + 0xc6792837cd6388e7L, + 0xa412473a09b3cbc0L, + 0x1bc92b7206a65a9eL, + 0xef7db9f94f0f7a17L, + 0x50a6d5b1401aeb49L, + 0x2bab67e2c6affc59L, + 0x94700baac9ba6d07L, + 0x60c4992180134d8eL, + 0xdf1ff5698f06dcd0L, + 0xbd749a644bd69ff7L, + 0x02aff62c44c30ea9L, + 0xf61b64a70d6a2e20L, + 0x49c008ef027fbf7eL, + 0x659b7579099550dcL, + 0xda4019310680c182L, + 0x2ef48bba4f29e10bL, + 0x912fe7f2403c7055L, + 0xf34488ff84ec3372L, + 0x4c9fe4b78bf9a22cL, + 0xb82b763cc25082a5L, + 0x07f01a74cd4513fbL, + 0x7cfda8274bf004ebL, + 0xc326c46f44e595b5L, + 0x379256e40d4cb53cL, + 0x88493aac02592462L, + 0xea2255a1c6896745L, + 0x55f939e9c99cf61bL, + 0xa14dab628035d692L, + 0x1e96c72a8f2047ccL, + 0x5756cfc58d5ff8b2L, + 0xe88da38d824a69ecL, + 0x1c393106cbe34965L, + 0xa3e25d4ec4f6d83bL, + 0xc189324300269b1cL, + 0x7e525e0b0f330a42L, + 0x8ae6cc80469a2acbL, + 0x353da0c8498fbb95L, + 0x4e30129bcf3aac85L, + 0xf1eb7ed3c02f3ddbL, + 0x055fec5889861d52L, + 0xba84801086938c0cL, + 0xd8efef1d4243cf2bL, + 0x673483554d565e75L, + 0x938011de04ff7efcL, + 0x2c5b7d960beaefa2L, + 0xcb36eaf2132aa1b8L, + 0x74ed86ba1c3f30e6L, + 0x805914315596106fL, + 0x3f8278795a838131L, + 0x5de917749e53c216L, + 0xe2327b3c91465348L, + 0x1686e9b7d8ef73c1L, + 0xa95d85ffd7fae29fL, + 0xd25037ac514ff58fL, + 0x6d8b5be45e5a64d1L, + 0x993fc96f17f34458L, + 0x26e4a52718e6d506L, + 0x448fca2adc369621L, + 0xfb54a662d323077fL, + 0x0fe034e99a8a27f6L, + 0xb03b58a1959fb6a8L, + 0xf9fb504e97e009d6L, + 0x46203c0698f59888L, + 0xb294ae8dd15cb801L, + 0x0d4fc2c5de49295fL, + 0x6f24adc81a996a78L, + 0xd0ffc180158cfb26L, + 0x244b530b5c25dbafL, + 0x9b903f4353304af1L, + 0xe09d8d10d5855de1L, + 0x5f46e158da90ccbfL, + 0xabf273d39339ec36L, + 0x14291f9b9c2c7d68L, + 0x7642709658fc3e4fL, + 0xc9991cde57e9af11L, + 0x3d2d8e551e408f98L, + 0x82f6e21d11551ec6L, + 0xaead9f8b1abff164L, + 0x1176f3c315aa603aL, + 0xe5c261485c0340b3L, + 0x5a190d005316d1edL, + 0x3872620d97c692caL, + 0x87a90e4598d30394L, + 0x731d9cced17a231dL, + 0xccc6f086de6fb243L, + 0xb7cb42d558daa553L, + 0x08102e9d57cf340dL, + 0xfca4bc161e661484L, + 0x437fd05e117385daL, + 0x2114bf53d5a3c6fdL, + 0x9ecfd31bdab657a3L, + 0x6a7b4190931f772aL, + 0xd5a02dd89c0ae674L, + 0x9c6025379e75590aL, + 0x23bb497f9160c854L, + 0xd70fdbf4d8c9e8ddL, + 0x68d4b7bcd7dc7983L, + 0x0abfd8b1130c3aa4L, + 0xb564b4f91c19abfaL, + 0x41d0267255b08b73L, + 0xfe0b4a3a5aa51a2dL, + 0x8506f869dc100d3dL, + 0x3add9421d3059c63L, + 0xce6906aa9aacbceaL, + 0x71b26ae295b92db4L, + 0x13d905ef51696e93L, + 0xac0269a75e7cffcdL, + 0x58b6fb2c17d5df44L, + 0xe76d976418c04e1aL, + 0xa2b4f3b77ec2d01bL, + 0x1d6f9fff71d74145L, + 0xe9db0d74387e61ccL, + 0x5600613c376bf092L, + 0x346b0e31f3bbb3b5L, + 0x8bb06279fcae22ebL, + 0x7f04f0f2b5070262L, + 0xc0df9cbaba12933cL, + 0xbbd22ee93ca7842cL, + 0x040942a133b21572L, + 0xf0bdd02a7a1b35fbL, + 0x4f66bc62750ea4a5L, + 0x2d0dd36fb1dee782L, + 0x92d6bf27becb76dcL, + 0x66622dacf7625655L, + 0xd9b941e4f877c70bL, + 0x9079490bfa087875L, + 0x2fa22543f51de92bL, + 0xdb16b7c8bcb4c9a2L, + 0x64cddb80b3a158fcL, + 0x06a6b48d77711bdbL, + 0xb97dd8c578648a85L, + 0x4dc94a4e31cdaa0cL, + 0xf21226063ed83b52L, + 0x891f9455b86d2c42L, + 0x36c4f81db778bd1cL, + 0xc2706a96fed19d95L, + 0x7dab06def1c40ccbL, + 0x1fc069d335144fecL, + 0xa01b059b3a01deb2L, + 0x54af971073a8fe3bL, + 0xeb74fb587cbd6f65L, + 0xc72f86ce775780c7L, + 0x78f4ea8678421199L, + 0x8c40780d31eb3110L, + 0x339b14453efea04eL, + 0x51f07b48fa2ee369L, + 0xee2b1700f53b7237L, + 0x1a9f858bbc9252beL, + 0xa544e9c3b387c3e0L, + 0xde495b903532d4f0L, + 0x619237d83a2745aeL, + 0x9526a553738e6527L, + 0x2afdc91b7c9bf479L, + 0x4896a616b84bb75eL, + 0xf74dca5eb75e2600L, + 0x03f958d5fef70689L, + 0xbc22349df1e297d7L, + 0xf5e23c72f39d28a9L, + 0x4a39503afc88b9f7L, + 0xbe8dc2b1b521997eL, + 0x0156aef9ba340820L, + 0x633dc1f47ee44b07L, + 0xdce6adbc71f1da59L, + 0x28523f373858fad0L, + 0x9789537f374d6b8eL, + 0xec84e12cb1f87c9eL, + 0x535f8d64beededc0L, + 0xa7eb1feff744cd49L, + 0x183073a7f8515c17L, + 0x7a5b1caa3c811f30L, + 0xc58070e233948e6eL, + 0x3134e2697a3daee7L, + 0x8eef8e2175283fb9L, + 0x698219456de871a3L, + 0xd659750d62fde0fdL, + 0x22ede7862b54c074L, + 0x9d368bce2441512aL, + 0xff5de4c3e091120dL, + 0x4086888bef848353L, + 0xb4321a00a62da3daL, + 0x0be97648a9383284L, + 0x70e4c41b2f8d2594L, + 0xcf3fa8532098b4caL, + 0x3b8b3ad869319443L, + 0x845056906624051dL, + 0xe63b399da2f4463aL, + 0x59e055d5ade1d764L, + 0xad54c75ee448f7edL, + 0x128fab16eb5d66b3L, + 0x5b4fa3f9e922d9cdL, + 0xe494cfb1e6374893L, + 0x10205d3aaf9e681aL, + 0xaffb3172a08bf944L, + 0xcd905e7f645bba63L, + 0x724b32376b4e2b3dL, + 0x86ffa0bc22e70bb4L, + 0x3924ccf42df29aeaL, + 0x42297ea7ab478dfaL, + 0xfdf212efa4521ca4L, + 0x09468064edfb3c2dL, + 0xb69dec2ce2eead73L, + 0xd4f68321263eee54L, + 0x6b2def69292b7f0aL, + 0x9f997de260825f83L, + 0x204211aa6f97ceddL, + 0x0c196c3c647d217fL, + 0xb3c200746b68b021L, + 0x477692ff22c190a8L, + 0xf8adfeb72dd401f6L, + 0x9ac691bae90442d1L, + 0x251dfdf2e611d38fL, + 0xd1a96f79afb8f306L, + 0x6e720331a0ad6258L, + 0x157fb16226187548L, + 0xaaa4dd2a290de416L, + 0x5e104fa160a4c49fL, + 0xe1cb23e96fb155c1L, + 0x83a04ce4ab6116e6L, + 0x3c7b20aca47487b8L, + 0xc8cfb227eddda731L, + 0x7714de6fe2c8366fL, + 0x3ed4d680e0b78911L, + 0x810fbac8efa2184fL, + 0x75bb2843a60b38c6L, + 0xca60440ba91ea998L, + 0xa80b2b066dceeabfL, + 0x17d0474e62db7be1L, + 0xe364d5c52b725b68L, + 0x5cbfb98d2467ca36L, + 0x27b20bdea2d2dd26L, + 0x98696796adc74c78L, + 0x6cddf51de46e6cf1L, + 0xd3069955eb7bfdafL, + 0xb16df6582fabbe88L, + 0x0eb69a1020be2fd6L, + 0xfa02089b69170f5fL, + 0x45d964d366029e01L, + + 0x0000000000000000L, + 0x3ea616bd2ae10d77L, + 0x7d4c2d7a55c21aeeL, + 0x43ea3bc77f231799L, + 0xfa985af4ab8435dcL, + 0xc43e4c49816538abL, + 0x87d4778efe462f32L, + 0xb9726133d4a72245L, + 0xc1e993ba0f9ff8d3L, + 0xff4f8507257ef5a4L, + 0xbca5bec05a5de23dL, + 0x8203a87d70bcef4aL, + 0x3b71c94ea41bcd0fL, + 0x05d7dff38efac078L, + 0x463de434f1d9d7e1L, + 0x789bf289db38da96L, + 0xb70a012747a862cdL, + 0x89ac179a6d496fbaL, + 0xca462c5d126a7823L, + 0xf4e03ae0388b7554L, + 0x4d925bd3ec2c5711L, + 0x73344d6ec6cd5a66L, + 0x30de76a9b9ee4dffL, + 0x0e786014930f4088L, + 0x76e3929d48379a1eL, + 0x4845842062d69769L, + 0x0bafbfe71df580f0L, + 0x3509a95a37148d87L, + 0x8c7bc869e3b3afc2L, + 0xb2ddded4c952a2b5L, + 0xf137e513b671b52cL, + 0xcf91f3ae9c90b85bL, + 0x5acd241dd7c756f1L, + 0x646b32a0fd265b86L, + 0x2781096782054c1fL, + 0x19271fdaa8e44168L, + 0xa0557ee97c43632dL, + 0x9ef3685456a26e5aL, + 0xdd195393298179c3L, + 0xe3bf452e036074b4L, + 0x9b24b7a7d858ae22L, + 0xa582a11af2b9a355L, + 0xe6689add8d9ab4ccL, + 0xd8ce8c60a77bb9bbL, + 0x61bced5373dc9bfeL, + 0x5f1afbee593d9689L, + 0x1cf0c029261e8110L, + 0x2256d6940cff8c67L, + 0xedc7253a906f343cL, + 0xd3613387ba8e394bL, + 0x908b0840c5ad2ed2L, + 0xae2d1efdef4c23a5L, + 0x175f7fce3beb01e0L, + 0x29f96973110a0c97L, + 0x6a1352b46e291b0eL, + 0x54b5440944c81679L, + 0x2c2eb6809ff0ccefL, + 0x1288a03db511c198L, + 0x51629bfaca32d601L, + 0x6fc48d47e0d3db76L, + 0xd6b6ec743474f933L, + 0xe810fac91e95f444L, + 0xabfac10e61b6e3ddL, + 0x955cd7b34b57eeaaL, + 0xb59a483baf8eade2L, + 0x8b3c5e86856fa095L, + 0xc8d66541fa4cb70cL, + 0xf67073fcd0adba7bL, + 0x4f0212cf040a983eL, + 0x71a404722eeb9549L, + 0x324e3fb551c882d0L, + 0x0ce829087b298fa7L, + 0x7473db81a0115531L, + 0x4ad5cd3c8af05846L, + 0x093ff6fbf5d34fdfL, + 0x3799e046df3242a8L, + 0x8eeb81750b9560edL, + 0xb04d97c821746d9aL, + 0xf3a7ac0f5e577a03L, + 0xcd01bab274b67774L, + 0x0290491ce826cf2fL, + 0x3c365fa1c2c7c258L, + 0x7fdc6466bde4d5c1L, + 0x417a72db9705d8b6L, + 0xf80813e843a2faf3L, + 0xc6ae05556943f784L, + 0x85443e921660e01dL, + 0xbbe2282f3c81ed6aL, + 0xc379daa6e7b937fcL, + 0xfddfcc1bcd583a8bL, + 0xbe35f7dcb27b2d12L, + 0x8093e161989a2065L, + 0x39e180524c3d0220L, + 0x074796ef66dc0f57L, + 0x44adad2819ff18ceL, + 0x7a0bbb95331e15b9L, + 0xef576c267849fb13L, + 0xd1f17a9b52a8f664L, + 0x921b415c2d8be1fdL, + 0xacbd57e1076aec8aL, + 0x15cf36d2d3cdcecfL, + 0x2b69206ff92cc3b8L, + 0x68831ba8860fd421L, + 0x56250d15aceed956L, + 0x2ebeff9c77d603c0L, + 0x1018e9215d370eb7L, + 0x53f2d2e62214192eL, + 0x6d54c45b08f51459L, + 0xd426a568dc52361cL, + 0xea80b3d5f6b33b6bL, + 0xa96a881289902cf2L, + 0x97cc9eafa3712185L, + 0x585d6d013fe199deL, + 0x66fb7bbc150094a9L, + 0x2511407b6a238330L, + 0x1bb756c640c28e47L, + 0xa2c537f59465ac02L, + 0x9c632148be84a175L, + 0xdf891a8fc1a7b6ecL, + 0xe12f0c32eb46bb9bL, + 0x99b4febb307e610dL, + 0xa712e8061a9f6c7aL, + 0xe4f8d3c165bc7be3L, + 0xda5ec57c4f5d7694L, + 0x632ca44f9bfa54d1L, + 0x5d8ab2f2b11b59a6L, + 0x1e608935ce384e3fL, + 0x20c69f88e4d94348L, + 0x5fedb624078ac8afL, + 0x614ba0992d6bc5d8L, + 0x22a19b5e5248d241L, + 0x1c078de378a9df36L, + 0xa575ecd0ac0efd73L, + 0x9bd3fa6d86eff004L, + 0xd839c1aaf9cce79dL, + 0xe69fd717d32deaeaL, + 0x9e04259e0815307cL, + 0xa0a2332322f43d0bL, + 0xe34808e45dd72a92L, + 0xddee1e59773627e5L, + 0x649c7f6aa39105a0L, + 0x5a3a69d7897008d7L, + 0x19d05210f6531f4eL, + 0x277644addcb21239L, + 0xe8e7b7034022aa62L, + 0xd641a1be6ac3a715L, + 0x95ab9a7915e0b08cL, + 0xab0d8cc43f01bdfbL, + 0x127fedf7eba69fbeL, + 0x2cd9fb4ac14792c9L, + 0x6f33c08dbe648550L, + 0x5195d63094858827L, + 0x290e24b94fbd52b1L, + 0x17a83204655c5fc6L, + 0x544209c31a7f485fL, + 0x6ae41f7e309e4528L, + 0xd3967e4de439676dL, + 0xed3068f0ced86a1aL, + 0xaeda5337b1fb7d83L, + 0x907c458a9b1a70f4L, + 0x05209239d04d9e5eL, + 0x3b868484faac9329L, + 0x786cbf43858f84b0L, + 0x46caa9feaf6e89c7L, + 0xffb8c8cd7bc9ab82L, + 0xc11ede705128a6f5L, + 0x82f4e5b72e0bb16cL, + 0xbc52f30a04eabc1bL, + 0xc4c90183dfd2668dL, + 0xfa6f173ef5336bfaL, + 0xb9852cf98a107c63L, + 0x87233a44a0f17114L, + 0x3e515b7774565351L, + 0x00f74dca5eb75e26L, + 0x431d760d219449bfL, + 0x7dbb60b00b7544c8L, + 0xb22a931e97e5fc93L, + 0x8c8c85a3bd04f1e4L, + 0xcf66be64c227e67dL, + 0xf1c0a8d9e8c6eb0aL, + 0x48b2c9ea3c61c94fL, + 0x7614df571680c438L, + 0x35fee49069a3d3a1L, + 0x0b58f22d4342ded6L, + 0x73c300a4987a0440L, + 0x4d651619b29b0937L, + 0x0e8f2ddecdb81eaeL, + 0x30293b63e75913d9L, + 0x895b5a5033fe319cL, + 0xb7fd4ced191f3cebL, + 0xf417772a663c2b72L, + 0xcab161974cdd2605L, + 0xea77fe1fa804654dL, + 0xd4d1e8a282e5683aL, + 0x973bd365fdc67fa3L, + 0xa99dc5d8d72772d4L, + 0x10efa4eb03805091L, + 0x2e49b25629615de6L, + 0x6da3899156424a7fL, + 0x53059f2c7ca34708L, + 0x2b9e6da5a79b9d9eL, + 0x15387b188d7a90e9L, + 0x56d240dff2598770L, + 0x68745662d8b88a07L, + 0xd10637510c1fa842L, + 0xefa021ec26fea535L, + 0xac4a1a2b59ddb2acL, + 0x92ec0c96733cbfdbL, + 0x5d7dff38efac0780L, + 0x63dbe985c54d0af7L, + 0x2031d242ba6e1d6eL, + 0x1e97c4ff908f1019L, + 0xa7e5a5cc4428325cL, + 0x9943b3716ec93f2bL, + 0xdaa988b611ea28b2L, + 0xe40f9e0b3b0b25c5L, + 0x9c946c82e033ff53L, + 0xa2327a3fcad2f224L, + 0xe1d841f8b5f1e5bdL, + 0xdf7e57459f10e8caL, + 0x660c36764bb7ca8fL, + 0x58aa20cb6156c7f8L, + 0x1b401b0c1e75d061L, + 0x25e60db13494dd16L, + 0xb0bada027fc333bcL, + 0x8e1cccbf55223ecbL, + 0xcdf6f7782a012952L, + 0xf350e1c500e02425L, + 0x4a2280f6d4470660L, + 0x7484964bfea60b17L, + 0x376ead8c81851c8eL, + 0x09c8bb31ab6411f9L, + 0x715349b8705ccb6fL, + 0x4ff55f055abdc618L, + 0x0c1f64c2259ed181L, + 0x32b9727f0f7fdcf6L, + 0x8bcb134cdbd8feb3L, + 0xb56d05f1f139f3c4L, + 0xf6873e368e1ae45dL, + 0xc821288ba4fbe92aL, + 0x07b0db25386b5171L, + 0x3916cd98128a5c06L, + 0x7afcf65f6da94b9fL, + 0x445ae0e2474846e8L, + 0xfd2881d193ef64adL, + 0xc38e976cb90e69daL, + 0x8064acabc62d7e43L, + 0xbec2ba16eccc7334L, + 0xc659489f37f4a9a2L, + 0xf8ff5e221d15a4d5L, + 0xbb1565e56236b34cL, + 0x85b3735848d7be3bL, + 0x3cc1126b9c709c7eL, + 0x026704d6b6919109L, + 0x418d3f11c9b28690L, + 0x7f2b29ace3538be7L, + + 0x0000000000000000L, + 0x169489cc969951e5L, + 0x2d2913992d32a3caL, + 0x3bbd9a55bbabf22fL, + 0x5a5227325a654794L, + 0x4cc6aefeccfc1671L, + 0x777b34ab7757e45eL, + 0x61efbd67e1ceb5bbL, + 0xb4a44e64b4ca8f28L, + 0xa230c7a82253decdL, + 0x998d5dfd99f82ce2L, + 0x8f19d4310f617d07L, + 0xeef66956eeafc8bcL, + 0xf862e09a78369959L, + 0xc3df7acfc39d6b76L, + 0xd54bf30355043a93L, + 0x5d91ba9a31028d3bL, + 0x4b053356a79bdcdeL, + 0x70b8a9031c302ef1L, + 0x662c20cf8aa97f14L, + 0x07c39da86b67caafL, + 0x11571464fdfe9b4aL, + 0x2aea8e3146556965L, + 0x3c7e07fdd0cc3880L, + 0xe935f4fe85c80213L, + 0xffa17d32135153f6L, + 0xc41ce767a8faa1d9L, + 0xd2886eab3e63f03cL, + 0xb367d3ccdfad4587L, + 0xa5f35a0049341462L, + 0x9e4ec055f29fe64dL, + 0x88da49996406b7a8L, + 0xbb23753462051a76L, + 0xadb7fcf8f49c4b93L, + 0x960a66ad4f37b9bcL, + 0x809eef61d9aee859L, + 0xe171520638605de2L, + 0xf7e5dbcaaef90c07L, + 0xcc58419f1552fe28L, + 0xdaccc85383cbafcdL, + 0x0f873b50d6cf955eL, + 0x1913b29c4056c4bbL, + 0x22ae28c9fbfd3694L, + 0x343aa1056d646771L, + 0x55d51c628caad2caL, + 0x434195ae1a33832fL, + 0x78fc0ffba1987100L, + 0x6e688637370120e5L, + 0xe6b2cfae5307974dL, + 0xf0264662c59ec6a8L, + 0xcb9bdc377e353487L, + 0xdd0f55fbe8ac6562L, + 0xbce0e89c0962d0d9L, + 0xaa7461509ffb813cL, + 0x91c9fb0524507313L, + 0x875d72c9b2c922f6L, + 0x521681cae7cd1865L, + 0x4482080671544980L, + 0x7f3f9253caffbbafL, + 0x69ab1b9f5c66ea4aL, + 0x0844a6f8bda85ff1L, + 0x1ed02f342b310e14L, + 0x256db561909afc3bL, + 0x33f93cad0603addeL, + 0x429fcc3b9c9da787L, + 0x540b45f70a04f662L, + 0x6fb6dfa2b1af044dL, + 0x7922566e273655a8L, + 0x18cdeb09c6f8e013L, + 0x0e5962c55061b1f6L, + 0x35e4f890ebca43d9L, + 0x2370715c7d53123cL, + 0xf63b825f285728afL, + 0xe0af0b93bece794aL, + 0xdb1291c605658b65L, + 0xcd86180a93fcda80L, + 0xac69a56d72326f3bL, + 0xbafd2ca1e4ab3edeL, + 0x8140b6f45f00ccf1L, + 0x97d43f38c9999d14L, + 0x1f0e76a1ad9f2abcL, + 0x099aff6d3b067b59L, + 0x3227653880ad8976L, + 0x24b3ecf41634d893L, + 0x455c5193f7fa6d28L, + 0x53c8d85f61633ccdL, + 0x6875420adac8cee2L, + 0x7ee1cbc64c519f07L, + 0xabaa38c51955a594L, + 0xbd3eb1098fccf471L, + 0x86832b5c3467065eL, + 0x9017a290a2fe57bbL, + 0xf1f81ff74330e200L, + 0xe76c963bd5a9b3e5L, + 0xdcd10c6e6e0241caL, + 0xca4585a2f89b102fL, + 0xf9bcb90ffe98bdf1L, + 0xef2830c36801ec14L, + 0xd495aa96d3aa1e3bL, + 0xc201235a45334fdeL, + 0xa3ee9e3da4fdfa65L, + 0xb57a17f13264ab80L, + 0x8ec78da489cf59afL, + 0x985304681f56084aL, + 0x4d18f76b4a5232d9L, + 0x5b8c7ea7dccb633cL, + 0x6031e4f267609113L, + 0x76a56d3ef1f9c0f6L, + 0x174ad0591037754dL, + 0x01de599586ae24a8L, + 0x3a63c3c03d05d687L, + 0x2cf74a0cab9c8762L, + 0xa42d0395cf9a30caL, + 0xb2b98a595903612fL, + 0x8904100ce2a89300L, + 0x9f9099c07431c2e5L, + 0xfe7f24a795ff775eL, + 0xe8ebad6b036626bbL, + 0xd356373eb8cdd494L, + 0xc5c2bef22e548571L, + 0x10894df17b50bfe2L, + 0x061dc43dedc9ee07L, + 0x3da05e6856621c28L, + 0x2b34d7a4c0fb4dcdL, + 0x4adb6ac32135f876L, + 0x5c4fe30fb7aca993L, + 0x67f2795a0c075bbcL, + 0x7166f0969a9e0a59L, + 0x853f9877393b4f0eL, + 0x93ab11bbafa21eebL, + 0xa8168bee1409ecc4L, + 0xbe8202228290bd21L, + 0xdf6dbf45635e089aL, + 0xc9f93689f5c7597fL, + 0xf244acdc4e6cab50L, + 0xe4d02510d8f5fab5L, + 0x319bd6138df1c026L, + 0x270f5fdf1b6891c3L, + 0x1cb2c58aa0c363ecL, + 0x0a264c46365a3209L, + 0x6bc9f121d79487b2L, + 0x7d5d78ed410dd657L, + 0x46e0e2b8faa62478L, + 0x50746b746c3f759dL, + 0xd8ae22ed0839c235L, + 0xce3aab219ea093d0L, + 0xf5873174250b61ffL, + 0xe313b8b8b392301aL, + 0x82fc05df525c85a1L, + 0x94688c13c4c5d444L, + 0xafd516467f6e266bL, + 0xb9419f8ae9f7778eL, + 0x6c0a6c89bcf34d1dL, + 0x7a9ee5452a6a1cf8L, + 0x41237f1091c1eed7L, + 0x57b7f6dc0758bf32L, + 0x36584bbbe6960a89L, + 0x20ccc277700f5b6cL, + 0x1b715822cba4a943L, + 0x0de5d1ee5d3df8a6L, + 0x3e1ced435b3e5578L, + 0x2888648fcda7049dL, + 0x1335feda760cf6b2L, + 0x05a17716e095a757L, + 0x644eca71015b12ecL, + 0x72da43bd97c24309L, + 0x4967d9e82c69b126L, + 0x5ff35024baf0e0c3L, + 0x8ab8a327eff4da50L, + 0x9c2c2aeb796d8bb5L, + 0xa791b0bec2c6799aL, + 0xb1053972545f287fL, + 0xd0ea8415b5919dc4L, + 0xc67e0dd92308cc21L, + 0xfdc3978c98a33e0eL, + 0xeb571e400e3a6febL, + 0x638d57d96a3cd843L, + 0x7519de15fca589a6L, + 0x4ea44440470e7b89L, + 0x5830cd8cd1972a6cL, + 0x39df70eb30599fd7L, + 0x2f4bf927a6c0ce32L, + 0x14f663721d6b3c1dL, + 0x0262eabe8bf26df8L, + 0xd72919bddef6576bL, + 0xc1bd9071486f068eL, + 0xfa000a24f3c4f4a1L, + 0xec9483e8655da544L, + 0x8d7b3e8f849310ffL, + 0x9befb743120a411aL, + 0xa0522d16a9a1b335L, + 0xb6c6a4da3f38e2d0L, + 0xc7a0544ca5a6e889L, + 0xd134dd80333fb96cL, + 0xea8947d588944b43L, + 0xfc1dce191e0d1aa6L, + 0x9df2737effc3af1dL, + 0x8b66fab2695afef8L, + 0xb0db60e7d2f10cd7L, + 0xa64fe92b44685d32L, + 0x73041a28116c67a1L, + 0x659093e487f53644L, + 0x5e2d09b13c5ec46bL, + 0x48b9807daac7958eL, + 0x29563d1a4b092035L, + 0x3fc2b4d6dd9071d0L, + 0x047f2e83663b83ffL, + 0x12eba74ff0a2d21aL, + 0x9a31eed694a465b2L, + 0x8ca5671a023d3457L, + 0xb718fd4fb996c678L, + 0xa18c74832f0f979dL, + 0xc063c9e4cec12226L, + 0xd6f74028585873c3L, + 0xed4ada7de3f381ecL, + 0xfbde53b1756ad009L, + 0x2e95a0b2206eea9aL, + 0x3801297eb6f7bb7fL, + 0x03bcb32b0d5c4950L, + 0x15283ae79bc518b5L, + 0x74c787807a0bad0eL, + 0x62530e4cec92fcebL, + 0x59ee941957390ec4L, + 0x4f7a1dd5c1a05f21L, + 0x7c832178c7a3f2ffL, + 0x6a17a8b4513aa31aL, + 0x51aa32e1ea915135L, + 0x473ebb2d7c0800d0L, + 0x26d1064a9dc6b56bL, + 0x30458f860b5fe48eL, + 0x0bf815d3b0f416a1L, + 0x1d6c9c1f266d4744L, + 0xc8276f1c73697dd7L, + 0xdeb3e6d0e5f02c32L, + 0xe50e7c855e5bde1dL, + 0xf39af549c8c28ff8L, + 0x9275482e290c3a43L, + 0x84e1c1e2bf956ba6L, + 0xbf5c5bb7043e9989L, + 0xa9c8d27b92a7c86cL, + 0x21129be2f6a17fc4L, + 0x3786122e60382e21L, + 0x0c3b887bdb93dc0eL, + 0x1aaf01b74d0a8debL, + 0x7b40bcd0acc43850L, + 0x6dd4351c3a5d69b5L, + 0x5669af4981f69b9aL, + 0x40fd2685176fca7fL, + 0x95b6d586426bf0ecL, + 0x83225c4ad4f2a109L, + 0xb89fc61f6f595326L, + 0xae0b4fd3f9c002c3L, + 0xcfe4f2b4180eb778L, + 0xd9707b788e97e69dL, + 0xe2cde12d353c14b2L, + 0xf45968e1a3a54557L, + + 0x0000000000000000L, + 0x0aed36d1a3bb9d7fL, + 0x15da6da347773afeL, + 0x1f375b72e4cca781L, + 0x2bb4db468eee75fcL, + 0x2159ed972d55e883L, + 0x3e6eb6e5c9994f02L, + 0x348380346a22d27dL, + 0x5769b68d1ddcebf8L, + 0x5d84805cbe677687L, + 0x42b3db2e5aabd106L, + 0x485eedfff9104c79L, + 0x7cdd6dcb93329e04L, + 0x76305b1a3089037bL, + 0x69070068d445a4faL, + 0x63ea36b977fe3985L, + 0xaed36d1a3bb9d7f0L, + 0xa43e5bcb98024a8fL, + 0xbb0900b97cceed0eL, + 0xb1e43668df757071L, + 0x8567b65cb557a20cL, + 0x8f8a808d16ec3f73L, + 0x90bddbfff22098f2L, + 0x9a50ed2e519b058dL, + 0xf9badb9726653c08L, + 0xf357ed4685dea177L, + 0xec60b634611206f6L, + 0xe68d80e5c2a99b89L, + 0xd20e00d1a88b49f4L, + 0xd8e336000b30d48bL, + 0xc7d46d72effc730aL, + 0xcd395ba34c47ee75L, + 0x697ffc672fe43c8bL, + 0x6392cab68c5fa1f4L, + 0x7ca591c468930675L, + 0x7648a715cb289b0aL, + 0x42cb2721a10a4977L, + 0x482611f002b1d408L, + 0x57114a82e67d7389L, + 0x5dfc7c5345c6eef6L, + 0x3e164aea3238d773L, + 0x34fb7c3b91834a0cL, + 0x2bcc2749754fed8dL, + 0x21211198d6f470f2L, + 0x15a291acbcd6a28fL, + 0x1f4fa77d1f6d3ff0L, + 0x0078fc0ffba19871L, + 0x0a95cade581a050eL, + 0xc7ac917d145deb7bL, + 0xcd41a7acb7e67604L, + 0xd276fcde532ad185L, + 0xd89bca0ff0914cfaL, + 0xec184a3b9ab39e87L, + 0xe6f57cea390803f8L, + 0xf9c22798ddc4a479L, + 0xf32f11497e7f3906L, + 0x90c527f009810083L, + 0x9a281121aa3a9dfcL, + 0x851f4a534ef63a7dL, + 0x8ff27c82ed4da702L, + 0xbb71fcb6876f757fL, + 0xb19cca6724d4e800L, + 0xaeab9115c0184f81L, + 0xa446a7c463a3d2feL, + 0xd2fff8ce5fc87916L, + 0xd812ce1ffc73e469L, + 0xc725956d18bf43e8L, + 0xcdc8a3bcbb04de97L, + 0xf94b2388d1260ceaL, + 0xf3a61559729d9195L, + 0xec914e2b96513614L, + 0xe67c78fa35eaab6bL, + 0x85964e43421492eeL, + 0x8f7b7892e1af0f91L, + 0x904c23e00563a810L, + 0x9aa11531a6d8356fL, + 0xae229505ccfae712L, + 0xa4cfa3d46f417a6dL, + 0xbbf8f8a68b8dddecL, + 0xb115ce7728364093L, + 0x7c2c95d46471aee6L, + 0x76c1a305c7ca3399L, + 0x69f6f87723069418L, + 0x631bcea680bd0967L, + 0x57984e92ea9fdb1aL, + 0x5d75784349244665L, + 0x42422331ade8e1e4L, + 0x48af15e00e537c9bL, + 0x2b45235979ad451eL, + 0x21a81588da16d861L, + 0x3e9f4efa3eda7fe0L, + 0x3472782b9d61e29fL, + 0x00f1f81ff74330e2L, + 0x0a1ccece54f8ad9dL, + 0x152b95bcb0340a1cL, + 0x1fc6a36d138f9763L, + 0xbb8004a9702c459dL, + 0xb16d3278d397d8e2L, + 0xae5a690a375b7f63L, + 0xa4b75fdb94e0e21cL, + 0x9034dfeffec23061L, + 0x9ad9e93e5d79ad1eL, + 0x85eeb24cb9b50a9fL, + 0x8f03849d1a0e97e0L, + 0xece9b2246df0ae65L, + 0xe60484f5ce4b331aL, + 0xf933df872a87949bL, + 0xf3dee956893c09e4L, + 0xc75d6962e31edb99L, + 0xcdb05fb340a546e6L, + 0xd28704c1a469e167L, + 0xd86a321007d27c18L, + 0x155369b34b95926dL, + 0x1fbe5f62e82e0f12L, + 0x008904100ce2a893L, + 0x0a6432c1af5935ecL, + 0x3ee7b2f5c57be791L, + 0x340a842466c07aeeL, + 0x2b3ddf56820cdd6fL, + 0x21d0e98721b74010L, + 0x423adf3e56497995L, + 0x48d7e9eff5f2e4eaL, + 0x57e0b29d113e436bL, + 0x5d0d844cb285de14L, + 0x698e0478d8a70c69L, + 0x636332a97b1c9116L, + 0x7c5469db9fd03697L, + 0x76b95f0a3c6babe8L, + 0x9126d7cfe7076147L, + 0x9bcbe11e44bcfc38L, + 0x84fcba6ca0705bb9L, + 0x8e118cbd03cbc6c6L, + 0xba920c8969e914bbL, + 0xb07f3a58ca5289c4L, + 0xaf48612a2e9e2e45L, + 0xa5a557fb8d25b33aL, + 0xc64f6142fadb8abfL, + 0xcca25793596017c0L, + 0xd3950ce1bdacb041L, + 0xd9783a301e172d3eL, + 0xedfbba047435ff43L, + 0xe7168cd5d78e623cL, + 0xf821d7a73342c5bdL, + 0xf2cce17690f958c2L, + 0x3ff5bad5dcbeb6b7L, + 0x35188c047f052bc8L, + 0x2a2fd7769bc98c49L, + 0x20c2e1a738721136L, + 0x144161935250c34bL, + 0x1eac5742f1eb5e34L, + 0x019b0c301527f9b5L, + 0x0b763ae1b69c64caL, + 0x689c0c58c1625d4fL, + 0x62713a8962d9c030L, + 0x7d4661fb861567b1L, + 0x77ab572a25aefaceL, + 0x4328d71e4f8c28b3L, + 0x49c5e1cfec37b5ccL, + 0x56f2babd08fb124dL, + 0x5c1f8c6cab408f32L, + 0xf8592ba8c8e35dccL, + 0xf2b41d796b58c0b3L, + 0xed83460b8f946732L, + 0xe76e70da2c2ffa4dL, + 0xd3edf0ee460d2830L, + 0xd900c63fe5b6b54fL, + 0xc6379d4d017a12ceL, + 0xccdaab9ca2c18fb1L, + 0xaf309d25d53fb634L, + 0xa5ddabf476842b4bL, + 0xbaeaf08692488ccaL, + 0xb007c65731f311b5L, + 0x848446635bd1c3c8L, + 0x8e6970b2f86a5eb7L, + 0x915e2bc01ca6f936L, + 0x9bb31d11bf1d6449L, + 0x568a46b2f35a8a3cL, + 0x5c67706350e11743L, + 0x43502b11b42db0c2L, + 0x49bd1dc017962dbdL, + 0x7d3e9df47db4ffc0L, + 0x77d3ab25de0f62bfL, + 0x68e4f0573ac3c53eL, + 0x6209c68699785841L, + 0x01e3f03fee8661c4L, + 0x0b0ec6ee4d3dfcbbL, + 0x14399d9ca9f15b3aL, + 0x1ed4ab4d0a4ac645L, + 0x2a572b7960681438L, + 0x20ba1da8c3d38947L, + 0x3f8d46da271f2ec6L, + 0x3560700b84a4b3b9L, + 0x43d92f01b8cf1851L, + 0x493419d01b74852eL, + 0x560342a2ffb822afL, + 0x5cee74735c03bfd0L, + 0x686df44736216dadL, + 0x6280c296959af0d2L, + 0x7db799e471565753L, + 0x775aaf35d2edca2cL, + 0x14b0998ca513f3a9L, + 0x1e5daf5d06a86ed6L, + 0x016af42fe264c957L, + 0x0b87c2fe41df5428L, + 0x3f0442ca2bfd8655L, + 0x35e9741b88461b2aL, + 0x2ade2f696c8abcabL, + 0x203319b8cf3121d4L, + 0xed0a421b8376cfa1L, + 0xe7e774ca20cd52deL, + 0xf8d02fb8c401f55fL, + 0xf23d196967ba6820L, + 0xc6be995d0d98ba5dL, + 0xcc53af8cae232722L, + 0xd364f4fe4aef80a3L, + 0xd989c22fe9541ddcL, + 0xba63f4969eaa2459L, + 0xb08ec2473d11b926L, + 0xafb99935d9dd1ea7L, + 0xa554afe47a6683d8L, + 0x91d72fd0104451a5L, + 0x9b3a1901b3ffccdaL, + 0x840d427357336b5bL, + 0x8ee074a2f488f624L, + 0x2aa6d366972b24daL, + 0x204be5b73490b9a5L, + 0x3f7cbec5d05c1e24L, + 0x3591881473e7835bL, + 0x0112082019c55126L, + 0x0bff3ef1ba7ecc59L, + 0x14c865835eb26bd8L, + 0x1e255352fd09f6a7L, + 0x7dcf65eb8af7cf22L, + 0x7722533a294c525dL, + 0x68150848cd80f5dcL, + 0x62f83e996e3b68a3L, + 0x567bbead0419badeL, + 0x5c96887ca7a227a1L, + 0x43a1d30e436e8020L, + 0x494ce5dfe0d51d5fL, + 0x8475be7cac92f32aL, + 0x8e9888ad0f296e55L, + 0x91afd3dfebe5c9d4L, + 0x9b42e50e485e54abL, + 0xafc1653a227c86d6L, + 0xa52c53eb81c71ba9L, + 0xba1b0899650bbc28L, + 0xb0f63e48c6b02157L, + 0xd31c08f1b14e18d2L, + 0xd9f13e2012f585adL, + 0xc6c66552f639222cL, + 0xcc2b53835582bf53L, + 0xf8a8d3b73fa06d2eL, + 0xf245e5669c1bf051L, + 0xed72be1478d757d0L, + 0xe79f88c5db6ccaafL, + 0x0000000000000000L, + 0xb0bc2e589204f500L, + 0x55a17ae27c9e796bL, + 0xe51d54baee9a8c6bL, + 0xab42f5c4f93cf2d6L, + 0x1bfedb9c6b3807d6L, + 0xfee38f2685a28bbdL, + 0x4e5fa17e17a67ebdL, + 0x625ccddaaaee76c7L, + 0xd2e0e38238ea83c7L, + 0x37fdb738d6700facL, + 0x874199604474faacL, + 0xc91e381e53d28411L, + 0x79a21646c1d67111L, + 0x9cbf42fc2f4cfd7aL, + 0x2c036ca4bd48087aL, + 0xc4b99bb555dced8eL, + 0x7405b5edc7d8188eL, + 0x9118e157294294e5L, + 0x21a4cf0fbb4661e5L, + 0x6ffb6e71ace01f58L, + 0xdf4740293ee4ea58L, + 0x3a5a1493d07e6633L, + 0x8ae63acb427a9333L, + 0xa6e5566fff329b49L, + 0x165978376d366e49L, + 0xf3442c8d83ace222L, + 0x43f802d511a81722L, + 0x0da7a3ab060e699fL, + 0xbd1b8df3940a9c9fL, + 0x5806d9497a9010f4L, + 0xe8baf711e894e5f4L, + 0xbdaa1139f32e4877L, + 0x0d163f61612abd77L, + 0xe80b6bdb8fb0311cL, + 0x58b745831db4c41cL, + 0x16e8e4fd0a12baa1L, + 0xa654caa598164fa1L, + 0x43499e1f768cc3caL, + 0xf3f5b047e48836caL, + 0xdff6dce359c03eb0L, + 0x6f4af2bbcbc4cbb0L, + 0x8a57a601255e47dbL, + 0x3aeb8859b75ab2dbL, + 0x74b42927a0fccc66L, + 0xc408077f32f83966L, + 0x211553c5dc62b50dL, + 0x91a97d9d4e66400dL, + 0x79138a8ca6f2a5f9L, + 0xc9afa4d434f650f9L, + 0x2cb2f06eda6cdc92L, + 0x9c0ede3648682992L, + 0xd2517f485fce572fL, + 0x62ed5110cdcaa22fL, + 0x87f005aa23502e44L, + 0x374c2bf2b154db44L, + 0x1b4f47560c1cd33eL, + 0xabf3690e9e18263eL, + 0x4eee3db47082aa55L, + 0xfe5213ece2865f55L, + 0xb00db292f52021e8L, + 0x00b19cca6724d4e8L, + 0xe5acc87089be5883L, + 0x5510e6281bbaad83L, + 0x4f8d0420becb0385L, + 0xff312a782ccff685L, + 0x1a2c7ec2c2557aeeL, + 0xaa90509a50518feeL, + 0xe4cff1e447f7f153L, + 0x5473dfbcd5f30453L, + 0xb16e8b063b698838L, + 0x01d2a55ea96d7d38L, + 0x2dd1c9fa14257542L, + 0x9d6de7a286218042L, + 0x7870b31868bb0c29L, + 0xc8cc9d40fabff929L, + 0x86933c3eed198794L, + 0x362f12667f1d7294L, + 0xd33246dc9187feffL, + 0x638e688403830bffL, + 0x8b349f95eb17ee0bL, + 0x3b88b1cd79131b0bL, + 0xde95e57797899760L, + 0x6e29cb2f058d6260L, + 0x20766a51122b1cddL, + 0x90ca4409802fe9ddL, + 0x75d710b36eb565b6L, + 0xc56b3eebfcb190b6L, + 0xe968524f41f998ccL, + 0x59d47c17d3fd6dccL, + 0xbcc928ad3d67e1a7L, + 0x0c7506f5af6314a7L, + 0x422aa78bb8c56a1aL, + 0xf29689d32ac19f1aL, + 0x178bdd69c45b1371L, + 0xa737f331565fe671L, + 0xf22715194de54bf2L, + 0x429b3b41dfe1bef2L, + 0xa7866ffb317b3299L, + 0x173a41a3a37fc799L, + 0x5965e0ddb4d9b924L, + 0xe9d9ce8526dd4c24L, + 0x0cc49a3fc847c04fL, + 0xbc78b4675a43354fL, + 0x907bd8c3e70b3d35L, + 0x20c7f69b750fc835L, + 0xc5daa2219b95445eL, + 0x75668c790991b15eL, + 0x3b392d071e37cfe3L, + 0x8b85035f8c333ae3L, + 0x6e9857e562a9b688L, + 0xde2479bdf0ad4388L, + 0x369e8eac1839a67cL, + 0x8622a0f48a3d537cL, + 0x633ff44e64a7df17L, + 0xd383da16f6a32a17L, + 0x9ddc7b68e10554aaL, + 0x2d6055307301a1aaL, + 0xc87d018a9d9b2dc1L, + 0x78c12fd20f9fd8c1L, + 0x54c24376b2d7d0bbL, + 0xe47e6d2e20d325bbL, + 0x01633994ce49a9d0L, + 0xb1df17cc5c4d5cd0L, + 0xff80b6b24beb226dL, + 0x4f3c98ead9efd76dL, + 0xaa21cc5037755b06L, + 0x1a9de208a571ae06L, + 0x9f1a08417d96070aL, + 0x2fa62619ef92f20aL, + 0xcabb72a301087e61L, + 0x7a075cfb930c8b61L, + 0x3458fd8584aaf5dcL, + 0x84e4d3dd16ae00dcL, + 0x61f98767f8348cb7L, + 0xd145a93f6a3079b7L, + 0xfd46c59bd77871cdL, + 0x4dfaebc3457c84cdL, + 0xa8e7bf79abe608a6L, + 0x185b912139e2fda6L, + 0x5604305f2e44831bL, + 0xe6b81e07bc40761bL, + 0x03a54abd52dafa70L, + 0xb31964e5c0de0f70L, + 0x5ba393f4284aea84L, + 0xeb1fbdacba4e1f84L, + 0x0e02e91654d493efL, + 0xbebec74ec6d066efL, + 0xf0e16630d1761852L, + 0x405d48684372ed52L, + 0xa5401cd2ade86139L, + 0x15fc328a3fec9439L, + 0x39ff5e2e82a49c43L, + 0x8943707610a06943L, + 0x6c5e24ccfe3ae528L, + 0xdce20a946c3e1028L, + 0x92bdabea7b986e95L, + 0x220185b2e99c9b95L, + 0xc71cd108070617feL, + 0x77a0ff509502e2feL, + 0x22b019788eb84f7dL, + 0x920c37201cbcba7dL, + 0x7711639af2263616L, + 0xc7ad4dc26022c316L, + 0x89f2ecbc7784bdabL, + 0x394ec2e4e58048abL, + 0xdc53965e0b1ac4c0L, + 0x6cefb806991e31c0L, + 0x40ecd4a2245639baL, + 0xf050fafab652ccbaL, + 0x154dae4058c840d1L, + 0xa5f18018caccb5d1L, + 0xebae2166dd6acb6cL, + 0x5b120f3e4f6e3e6cL, + 0xbe0f5b84a1f4b207L, + 0x0eb375dc33f04707L, + 0xe60982cddb64a2f3L, + 0x56b5ac95496057f3L, + 0xb3a8f82fa7fadb98L, + 0x0314d67735fe2e98L, + 0x4d4b770922585025L, + 0xfdf75951b05ca525L, + 0x18ea0deb5ec6294eL, + 0xa85623b3ccc2dc4eL, + 0x84554f17718ad434L, + 0x34e9614fe38e2134L, + 0xd1f435f50d14ad5fL, + 0x61481bad9f10585fL, + 0x2f17bad388b626e2L, + 0x9fab948b1ab2d3e2L, + 0x7ab6c031f4285f89L, + 0xca0aee69662caa89L, + 0xd0970c61c35d048fL, + 0x602b22395159f18fL, + 0x85367683bfc37de4L, + 0x358a58db2dc788e4L, + 0x7bd5f9a53a61f659L, + 0xcb69d7fda8650359L, + 0x2e74834746ff8f32L, + 0x9ec8ad1fd4fb7a32L, + 0xb2cbc1bb69b37248L, + 0x0277efe3fbb78748L, + 0xe76abb59152d0b23L, + 0x57d695018729fe23L, + 0x1989347f908f809eL, + 0xa9351a27028b759eL, + 0x4c284e9dec11f9f5L, + 0xfc9460c57e150cf5L, + 0x142e97d49681e901L, + 0xa492b98c04851c01L, + 0x418fed36ea1f906aL, + 0xf133c36e781b656aL, + 0xbf6c62106fbd1bd7L, + 0x0fd04c48fdb9eed7L, + 0xeacd18f2132362bcL, + 0x5a7136aa812797bcL, + 0x76725a0e3c6f9fc6L, + 0xc6ce7456ae6b6ac6L, + 0x23d320ec40f1e6adL, + 0x936f0eb4d2f513adL, + 0xdd30afcac5536d10L, + 0x6d8c819257579810L, + 0x8891d528b9cd147bL, + 0x382dfb702bc9e17bL, + 0x6d3d1d5830734cf8L, + 0xdd813300a277b9f8L, + 0x389c67ba4ced3593L, + 0x882049e2dee9c093L, + 0xc67fe89cc94fbe2eL, + 0x76c3c6c45b4b4b2eL, + 0x93de927eb5d1c745L, + 0x2362bc2627d53245L, + 0x0f61d0829a9d3a3fL, + 0xbfddfeda0899cf3fL, + 0x5ac0aa60e6034354L, + 0xea7c84387407b654L, + 0xa423254663a1c8e9L, + 0x149f0b1ef1a53de9L, + 0xf1825fa41f3fb182L, + 0x413e71fc8d3b4482L, + 0xa98486ed65afa176L, + 0x1938a8b5f7ab5476L, + 0xfc25fc0f1931d81dL, + 0x4c99d2578b352d1dL, + 0x02c673299c9353a0L, + 0xb27a5d710e97a6a0L, + 0x576709cbe00d2acbL, + 0xe7db27937209dfcbL, + 0xcbd84b37cf41d7b1L, + 0x7b64656f5d4522b1L, + 0x9e7931d5b3dfaedaL, + 0x2ec51f8d21db5bdaL, + 0x609abef3367d2567L, + 0xd02690aba479d067L, + 0x353bc4114ae35c0cL, + 0x8587ea49d8e7a90cL, }; + + private static final long[] M_UX2N = { + 0x0080000000000000L, + 0x0000800000000000L, + 0x0000000080000000L, + 0x9a6c9329ac4bc9b5L, + 0x10f4bb0f129310d6L, + 0x70f05dcea2ebd226L, + 0x311211205672822dL, + 0x2fc297db0f46c96eL, + 0xca4d536fabf7da84L, + 0xfb4cdc3b379ee6edL, + 0xea261148df25140aL, + 0x59ccb2c07aa6c9b4L, + 0x20b3674a839af27aL, + 0x2d8e1986da94d583L, + 0x42cdf4c20337635dL, + 0x1d78724bf0f26839L, + 0xb96c84e0afb34bd5L, + 0x5d2e1fcd2df0a3eaL, + 0xcd9506572332be42L, + 0x23bda2427f7d690fL, + 0x347a953232374f07L, + 0x1c2a807ac2a8ceeaL, + 0x9b92ad0e14fe1460L, + 0x2574114889f670b2L, + 0x4a84a6c45e3bf520L, + 0x915bbac21cd1c7ffL, + 0xb0290ec579f291f5L, + 0xcf2548505c624e6eL, + 0xb154f27bf08a8207L, + 0xce4e92344baf7d35L, + 0x51da8d7e057c5eb3L, + 0x9fb10823f5be15dfL, + 0x73b825b3ff1f71cfL, + 0x5db436c5406ebb74L, + 0xfa7ed8f3ec3f2bcaL, + 0xc4d58efdc61b9ef6L, + 0xa7e39e61e855bd45L, + 0x97ad46f9dd1bf2f1L, + 0x1a0abb01f853ee6bL, + 0x3f0827c3348f8215L, + 0x4eb68c4506134607L, + 0x4a46f6de5df34e0aL, + 0x2d855d6a1c57a8ddL, + 0x8688da58e1115812L, + 0x5232f417fc7c7300L, + 0xa4080fb2e767d8daL, + 0xd515a7e17693e562L, + 0x1181f7c862e94226L, + 0x9e23cd058204ca91L, + 0x9b8992c57a0aed82L, + 0xb2c0afb84609b6ffL, + 0x2f7160553a5ea018L, + 0x3cd378b5c99f2722L, + 0x814054ad61a3b058L, + 0xbf766189fce806d8L, + 0x85a5e898ac49f86fL, + 0x34830d11bc84f346L, + 0x9644d95b173c8c1cL, + 0x150401ac9ac759b1L, + 0xebe1f7f46fb00ebaL, + 0x8ee4ce0c2e2bd662L, + 0x4000000000000000L, + 0x2000000000000000L, + 0x0800000000000000L, }; + + /** + * Computes the CRC64 checksum for the given byte array using the Azure Storage CRC64 polynomial. + * This method processes the input data in chunks of 32 bytes for efficiency and uses lookup tables + * to update the CRC values. + * + * @param src the byte array for which the CRC64 checksum is to be computed. + * @param uCrc the initial CRC value. + * @return the computed CRC64 checksum. + */ + public static long compute(byte[] src, long uCrc) { + return compute(src, 0, src.length, uCrc); + } + + /** + * Computes the CRC64 checksum for a region of a byte array. Avoids copying when the caller has + * an array view (e.g. {@link ByteBuffer#array()} with array offset and position) or when combined + * with {@link #compute(byte[], long)} for whole-array input. + * + * @param src the byte array. + * @param offset start index (inclusive). + * @param length number of bytes to process. + * @param uCrc the initial CRC value. + * @return the computed CRC64 checksum. + */ + public static long compute(byte[] src, int offset, int length, long uCrc) { + int pData = offset; + long uSize = length; + long uBytes, uStop; + + uCrc = ~uCrc; // Flip all bits of uCrc + + uStop = uSize - (uSize % 32); + if (uStop >= 2 * 32) { + long uCrc0 = 0L; + long uCrc1 = 0L; + long uCrc2 = 0L; + long uCrc3 = 0L; + + int pLast = pData + (int) uStop - 32; + uSize -= uStop; + uCrc0 = uCrc; + + ByteBuffer buffer = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN); + + for (; pData < pLast; pData += 32) { + long b0, b1, b2, b3; + + // Load and XOR data with CRC + b0 = buffer.getLong(pData) ^ uCrc0; + b1 = buffer.getLong(pData + 8) ^ uCrc1; + b2 = buffer.getLong(pData + 16) ^ uCrc2; + b3 = buffer.getLong(pData + 24) ^ uCrc3; + + // Unsigned updates using tables and masking + uCrc0 = M_U32[7 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 = M_U32[7 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 = M_U32[7 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 = M_U32[7 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[6 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[6 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[6 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[6 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[5 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[5 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[5 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[5 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[4 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[4 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[4 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[4 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[3 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[3 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[3 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[3 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[2 * 256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[2 * 256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[2 * 256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[2 * 256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[256 + ((int) (b0 & 0xFF))]; + b0 >>>= 8; + uCrc1 ^= M_U32[256 + ((int) (b1 & 0xFF))]; + b1 >>>= 8; + uCrc2 ^= M_U32[256 + ((int) (b2 & 0xFF))]; + b2 >>>= 8; + uCrc3 ^= M_U32[256 + ((int) (b3 & 0xFF))]; + b3 >>>= 8; + + uCrc0 ^= M_U32[((int) (b0 & 0xFF))]; + uCrc1 ^= M_U32[((int) (b1 & 0xFF))]; + uCrc2 ^= M_U32[((int) (b2 & 0xFF))]; + uCrc3 ^= M_U32[((int) (b3 & 0xFF))]; + } + + // Combine CRC values + uCrc = 0; + uCrc ^= ByteBuffer.wrap(src, pData, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc0; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + uCrc ^= ByteBuffer.wrap(src, pData + 8, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc1; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + uCrc ^= ByteBuffer.wrap(src, pData + 16, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc2; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + uCrc ^= ByteBuffer.wrap(src, pData + 24, 8).order(ByteOrder.LITTLE_ENDIAN).getLong() ^ uCrc3; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + uCrc = (uCrc >>> 8) ^ M_U1[(int) (uCrc & 0xFF)]; + + pData += 32; + } + + // Process remaining bytes + for (uBytes = 0; uBytes < uSize; ++uBytes, ++pData) { + uCrc = (uCrc >>> 8) ^ M_U1[(int) ((uCrc ^ src[pData]) & 0xFF)]; + } + + return ~uCrc; // Flip all bits of uCrc and return as long + } + + /** + * Concatenates two CRC64 values by combining their initial and final CRC values and sizes. + * This method ensures unsigned behavior and uses the `mulX_N` method to perform necessary + * multiplications in GF(2^64). + * + * @param uInitialCrcAB The initial CRC value for the concatenated data. + * @param uInitialCrcA The initial CRC value for the first data segment. + * @param uFinalCrcA The final CRC value for the first data segment. + * @param uSizeA The size of the first data segment. + * @param uInitialCrcB The initial CRC value for the second data segment. + * @param uFinalCrcB The final CRC value for the second data segment. + * @param uSizeB The size of the second data segment. + * @return The concatenated CRC64 value. + */ + public static long concat(long uInitialCrcAB, long uInitialCrcA, long uFinalCrcA, long uSizeA, long uInitialCrcB, + long uFinalCrcB, long uSizeB) { + long uFinalCrcAB = ~uFinalCrcA; + + // Ensure unsigned behavior when comparing uInitialCrcA and uInitialCrcAB + if ((uInitialCrcA) != (uInitialCrcAB)) { + // Apply mulX_N with proper unsigned masking + uFinalCrcAB ^= multiplyCrcByPowerOfX((uInitialCrcA ^ uInitialCrcAB), uSizeA); + } + + uFinalCrcAB ^= ~uInitialCrcB; // Ensure unsigned XOR logic + uFinalCrcAB = multiplyCrcByPowerOfX(uFinalCrcAB, uSizeB); + uFinalCrcAB ^= uFinalCrcB; + + return uFinalCrcAB; // Ensure the result is treated as unsigned + } + + /** + * Multiplies a CRC value by x^n in GF(2^64). + * This method uses a lookup table to perform the multiplication efficiently. + * + * @param a The CRC value to be multiplied. + * @param uSize The power of x by which the CRC value is to be multiplied. + * @return The resulting CRC64 value after multiplication. + */ + private static long multiplyCrcByPowerOfX(long a, long uSize) { + long i = 0; + long r = a; + + while (uSize != 0) { + if ((uSize & 1) == 1) { + r = mulPolyUnrolled(r, M_UX2N[(int) i]); // Ensure result is treated as unsigned + } + uSize >>>= 1; // Unsigned right shift + i += 1; + } + + return r; + } + + /** + * Multiplies two CRC values using the polynomial in GF(2^64). + * This method performs the multiplication using an unrolled loop for efficiency. + * + * @param a The first CRC value to be multiplied. + * @param b The second CRC value to be multiplied. + * @return The resulting CRC64 value after multiplication. + */ + private static long mulPolyUnrolled(long a, long b) { + final long p = POLY; + final long p2 = (p >>> 1) ^ (p); // Use unsigned shift + final long bw = Long.SIZE; + + final long[] vt = { 0, p2, p, p ^ p2 }; + final long[] vs = { bw - 2, bw - 1 }; + long[] vb = { (b >>> 1) ^ vt[(int) ((b & 1) << 1)], b }; // Unsigned right shift + long[] vr = { 0, 0 }; + + for (long i = 0; i < bw; i += 2) { + for (int j = 0; j < 2; ++j) { + vr[j] ^= vb[j] * ((a >>> vs[j]) & 1); // Unsigned right shift + vb[j] = (vb[j] >>> 2) ^ vt[(int) (vb[j] & 3)]; // Unsigned right shift + } + a <<= 2; // Shift remains the same as a is multiplied by 4 + } + + return vr[0] ^ vr[1]; + } + +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java new file mode 100644 index 000000000000..8e0ed1ff6e86 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageConstants.java @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +/** + * Constants used in the structured message encoding and decoding process. + */ +public final class StructuredMessageConstants { + /** + * The default version of the structured message. + */ + public static final int DEFAULT_MESSAGE_VERSION = 1; + + /** + * The length of the header for version 1 of the structured message. + */ + public static final int V1_HEADER_LENGTH = 13; + + /** + * The length of the segment header for version 1 of the structured message. + */ + public static final int V1_SEGMENT_HEADER_LENGTH = 10; + + /** + * The length of the CRC64 checksum. + */ + public static final int CRC64_LENGTH = 8; + + /** + * The default length of segments for version 1. + */ + public static final int V1_DEFAULT_SEGMENT_CONTENT_LENGTH = 4 * 1024 * 1024; // 4 MiB + + /** + * The maximum amount of data to encode at once. + */ + public static final int STATIC_MAXIMUM_ENCODED_DATA_LENGTH = 4 * 1024 * 1024; // 4 MiB + + /** + * The maximum single part upload size to use CRC64 header. + */ + public static final int MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER = 4 * 1024 * 1024; // 4 MiB + + /** + * The structured body type header value indicating version 1 with CRC64 properties. + */ + public static final String STRUCTURED_BODY_TYPE_VALUE = "XSM/1.0; properties=crc64"; + + public static final String CONTENT_VALIDATION_MODE_KEY = "contentValidationMode"; + + public static final String USE_CRC64_CHECKSUM_HEADER_CONTEXT = "crc64ChecksumHeaderContext"; + + public static final String USE_STRUCTURED_MESSAGE_CONTEXT = "structuredMessageChecksumAlgorithm"; + + public static final String STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY = "azure-storage-structured-message-decoding"; +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java new file mode 100644 index 000000000000..3a6570fe07bc --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoder.java @@ -0,0 +1,536 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.logging.ClientLogger; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.DEFAULT_MESSAGE_VERSION; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; + +/** + * Streaming decoder for the storage structured message format used to validate downloaded blob/file/datalake + * content with CRC64 checksums. + * + *

This class owns the actual parsing and CRC validation. The pipeline policy hands it raw {@link ByteBuffer}s as + * they arrive on the wire (via {@link #decodeChunk(ByteBuffer)}); the decoder returns only the payload bytes that + * have already been CRC-validated and tells the policy when the entire message has been consumed + * (via {@link #isComplete()}). Any malformed input or CRC mismatch surfaces as an + * {@link IllegalArgumentException} thrown from {@code decodeChunk} so the policy can translate it into a stream + * error.

+ * + *

Wire format (V1)

+ * + *

The encoded body has the following layout (all integers little-endian):

+ * 
+ *   |-- message header (13 B) ----------------------------------------|
+ *   |  version (1)  |  total message length (8)  |  flags (2)  |  numSegments (2)  |
+ *
+ *   for each segment in 1..numSegments:
+ *     |-- segment header (10 B) -|
+ *     |  segNum (2)  |  segContentLen (8)  |
+ *     |-- segment payload (segContentLen B) --|
+ *     |-- segment CRC64 footer (8 B; only if STORAGE_CRC64) --|
+ *
+ *   |-- message CRC64 footer (8 B; only if STORAGE_CRC64) --|
+ *
+ * + *

Emission guarantee

+ * + * Payload bytes for a segment are never emitted to the caller until that segment's CRC64 footer + * has been validated. This matches the emission semantics used by {@code BlobDecryptionPolicy}/{@code DecryptorV2} + * (which only emits a decrypted region after its GCM tag is verified) and ensures that no unvalidated bytes are + * exposed to consumers, even if the connection is later torn down or the download is retried. + * + *

Thread-safety

+ * + *

This class is not thread-safe. A new instance is created for every HTTP response, and the + * reactive operators in the policy ({@code concatMap}) serialize access to the single instance. Retries produce new + * HTTP responses and therefore new decoder instances, so a CRC failure on one attempt cannot pollute another.

+ */ +public class StructuredMessageDecoder { + private static final ClientLogger LOGGER = new ClientLogger(StructuredMessageDecoder.class); + + private long messageLength = -1; + private StructuredMessageFlags flags; + private int numSegments = -1; + private final long expectedEncodedMessageLength; + // Number of encoded bytes consumed so far (headers + payloads + footers). + private long messageOffset = 0; + private int currentSegmentNumber = 0; + private long currentSegmentContentLength = 0; + private long currentSegmentContentOffset = 0; + private boolean segmentHeaderRead = false; + // Running CRC64 over all payload bytes seen so far (across every segment). + private long messageCrc64 = 0; + // Running CRC64 over only the current segment's payload bytes. + private long segmentCrc64 = 0; + // Holds bytes left over from a previous decodeChunk() call when the current chunk did not contain a full + // header or footer. + private final ByteArrayOutputStream pendingBytes = new ByteArrayOutputStream(); + // Payload for the current segment, one per inbound wire buffer. Not pre-sized to the full segment length, + // avoiding unnecessary heap spikes. + // These bytes are intentionally NOT emitted until the segment's CRC footer has been validated. + private List currentSegmentPayload; + // Validated payload buffers to emit from the current decodeChunk() invocation. + private final List validatedOutput = new ArrayList<>(); + + /** + * Constructs a new StructuredMessageDecoder. + * + * @param expectedEncodedMessageLength The expected encoded structured-message length (typically HTTP + * {@code Content-Length}). + */ + public StructuredMessageDecoder(long expectedEncodedMessageLength) { + this.expectedEncodedMessageLength = expectedEncodedMessageLength; + } + + /** + * Reads the 13-byte message header (version + total length + flags + numSegments) the first time the decoder + * sees enough bytes, and validates each field. Subsequent calls are no-ops. + * + * @param buffer The buffer to read from. + * @return true if the header was successfully read (or had already been read on a previous pass); false if more + * bytes are still needed. + */ + private boolean tryReadMessageHeader(ByteBuffer buffer) { + if (messageLength != -1) { + // Header already parsed on a previous chunk; nothing to do. + return true; + } + + if (getAvailableBytes(buffer) < V1_HEADER_LENGTH) { + // Not enough bytes for the full header yet; carry over what we have. + appendToPending(buffer); + return false; + } + + ByteBuffer combined = getCombinedBuffer(buffer); + + // Byte 0: protocol version. + int messageVersion = Byte.toUnsignedInt(combined.get()); + if (messageVersion != DEFAULT_MESSAGE_VERSION) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + enrichExceptionMessage("Unsupported structured message version: " + messageVersion))); + } + + // Bytes 1-8: total encoded message length. Must be at least the header itself, and must agree with what the + // HTTP layer told us via Content-Length – any disagreement implies a truncated/extended response. + long msgLen = combined.getLong(); + if (msgLen < V1_HEADER_LENGTH) { + throw LOGGER.logExceptionAsError( + new IllegalArgumentException(enrichExceptionMessage("Message length too small: " + msgLen))); + } + if (msgLen != expectedEncodedMessageLength) { + throw LOGGER + .logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage("Structured message length " + + msgLen + " did not match content length " + expectedEncodedMessageLength))); + } + + // Bytes 9-10: flags (NONE or STORAGE_CRC64). Bytes 11-12: number of segments. + flags = StructuredMessageFlags.fromValue(Short.toUnsignedInt(combined.getShort())); + numSegments = Short.toUnsignedInt(combined.getShort()); + if (numSegments < 1) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException( + enrichExceptionMessage("Structured message must have at least one segment, got: " + numSegments))); + } + + // Commit: drop the 13 bytes we just parsed from pending/buffer and record the message length. + consumeBytes(V1_HEADER_LENGTH, buffer); + messageOffset += V1_HEADER_LENGTH; + messageLength = msgLen; + + return true; + } + + /** + * Reads the 10-byte header for the next segment (segment number + segment payload length) and resets + * per-segment state so {@link #tryReadSegmentContent(ByteBuffer)} can begin filling + * {@link #currentSegmentPayload}. + * + *

Validates that segments arrive in order and that the declared segment size leaves enough room in the + * remaining message for any subsequent segment headers, payloads, footers, and the trailing message footer – + * this catches malformed/oversized segment lengths up front instead of waiting until we run off the end of the + * stream.

+ * + * @param buffer The buffer to read from. + * @return true if the segment header was read; false if more bytes are needed. + */ + private boolean tryReadSegmentHeader(ByteBuffer buffer) { + if (getAvailableBytes(buffer) < V1_SEGMENT_HEADER_LENGTH) { + appendToPending(buffer); + return false; + } + + ByteBuffer combined = getCombinedBuffer(buffer); + + // Bytes 0-1: segment number. Bytes 2-9: declared payload length of this segment. + int segmentNum = Short.toUnsignedInt(combined.getShort()); + long segmentSize = combined.getLong(); + + // Segments must arrive strictly in order so the running CRC and "segment N follows segment N-1" assumption + // hold. Anything else implies a malformed/reordered response. + if (segmentNum != currentSegmentNumber + 1) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "Unexpected segment number. Expected: " + (currentSegmentNumber + 1) + ", got: " + segmentNum))); + } + + // Compute an upper bound on the legal segment size: whatever is left in the message, minus the bytes that + // MUST still appear after this segment's payload (this segment's footer, the headers/payloads/footers of all + // remaining segments, and the trailing message footer). + long footerSize = flags == StructuredMessageFlags.STORAGE_CRC64 ? CRC64_LENGTH : 0; + long remainingSegmentsAfterThis = (long) numSegments - segmentNum; + long reservedBytes + = footerSize + remainingSegmentsAfterThis * (V1_SEGMENT_HEADER_LENGTH + footerSize) + footerSize; + long maxSegmentSize = messageLength - messageOffset - V1_SEGMENT_HEADER_LENGTH - reservedBytes; + if (segmentSize < 0 || segmentSize > maxSegmentSize) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "Invalid segment size detected: " + segmentSize + " (max=" + maxSegmentSize + ")"))); + } + + // Commit: drop the 10 header bytes and set up per-segment state so payload accumulation can start fresh. + consumeBytes(V1_SEGMENT_HEADER_LENGTH, buffer); + messageOffset += V1_SEGMENT_HEADER_LENGTH; + currentSegmentNumber = segmentNum; + currentSegmentContentLength = segmentSize; + currentSegmentContentOffset = 0; + currentSegmentPayload = new ArrayList<>(); + + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + // Reset only the per-segment running CRC; the message-wide running CRC keeps accumulating across all + // segments so the final message footer covers the entire payload. + segmentCrc64 = 0; + } + + return true; + } + + /** + * Pulls as many payload bytes as possible (bounded by what is still owed for the current segment) from the + * pending+buffer view into {@link #currentSegmentPayload}, updating the running per-segment and per-message + * CRC64 values along the way. + * + *

Bytes accumulated here are not yet emitted to the caller. They are released only after + * {@link #tryReadSegmentFooter(ByteBuffer)} validates this segment's CRC. This is the mechanism that enforces + * "no unvalidated bytes ever leave the decoder".

+ * + * @param buffer The buffer to read from. + * @return The number of payload bytes read in this call (0 means we either had no bytes available or the + * current segment's payload was already complete). + */ + private int tryReadSegmentContent(ByteBuffer buffer) { + long remaining = currentSegmentContentLength - currentSegmentContentOffset; + if (remaining == 0) { + // Segment payload is already complete; nothing to do here. The caller will move on to read the footer. + return 0; + } + + int available = getAvailableBytes(buffer); + if (available == 0) { + return 0; + } + + // Read the minimum of "what's available right now" and "what's still owed for this segment" so we never + // accidentally consume the segment footer here. + int toRead = (int) Math.min(available, remaining); + ByteBuffer combined = getCombinedBuffer(buffer); + + // Materialize only this chunk so retained memory grows with bytes received, not the full declared segment size. + byte[] content = new byte[toRead]; + combined.get(content); + + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + // Update both CRCs incrementally: the segment CRC will be checked at the segment footer, and the + // message CRC accumulates across every segment to be checked at the message footer. + segmentCrc64 = StorageCrc64Calculator.compute(content, 0, toRead, segmentCrc64); + messageCrc64 = StorageCrc64Calculator.compute(content, 0, toRead, messageCrc64); + } + + consumeBytes(toRead, buffer); + currentSegmentPayload.add(content); + + messageOffset += toRead; + currentSegmentContentOffset += toRead; + + return toRead; + } + + /** + * Validates the 8-byte segment CRC64 footer for the segment that has just finished accumulating. Pre-condition: + * {@code currentSegmentContentOffset == currentSegmentContentLength}. + * + *

This step is intentionally separate from reading the message footer: when the CRC matches, we want to be + * able to flush the buffered segment payload to the caller right away – even if the trailing message footer is + * not yet available in the current chunk.

+ * + * @param buffer The buffer to read from. + * @return true if the footer was successfully read (or no footer is required for this message); false if more + * bytes are still needed. + */ + private boolean tryReadSegmentFooter(ByteBuffer buffer) { + if (currentSegmentContentOffset != currentSegmentContentLength) { + // Segment payload is not complete yet; wait for more content. + return true; + } + + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + return tryConsumeCrc64Footer(buffer, segmentCrc64, " in segment " + currentSegmentNumber); + } + + // No CRC was negotiated, so there is no footer to read; the caller can release the buffered payload. + return true; + } + + /** + * Validates the 8-byte message CRC64 footer that follows the last segment. + * + * @param buffer The buffer to read from. + * @return true if the footer was successfully read (or none is required); false if more bytes are still needed. + */ + private boolean tryReadMessageFooter(ByteBuffer buffer) { + if (flags == StructuredMessageFlags.STORAGE_CRC64) { + return tryConsumeCrc64Footer(buffer, messageCrc64, " in message footer"); + } + return true; + } + + /** + * Decodes as much as possible from the given buffer and returns any fully validated + * payload bytes that are now safe to emit downstream. + * + *

The returned buffers will only ever contain bytes from segments whose CRC (when + * enabled) has already been verified. If no segments have been fully validated by + * this invocation the method returns an empty list. Callers distinguish "more bytes + * needed" from "stream complete" via {@link #isComplete()}.

+ * + * @param buffer The buffer containing encoded data. + * @return Validated payload bytes ready to emit downstream; empty when none are ready yet. + * @throws IllegalArgumentException if the input is malformed or a CRC64 check fails. + */ + public List decodeChunk(ByteBuffer buffer) { + // Decoder always reads little-endian; force the order on the caller's buffer so all our get() calls match + // the wire format regardless of how the buffer was constructed. + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // Output collected during this single invocation. Each segment whose CRC validates in this call is appended + // here and ultimately returned to the policy as one or more ByteBuffers. + validatedOutput.clear(); + + // Step 1: parse the message header on the first chunk that has enough bytes for it. If this chunk doesn't, + // bail out early. + if (!tryReadMessageHeader(buffer)) { + return finishDecodeChunk(); + } + + // Step 2: walk forward through the message until we either hit the end (messageOffset == messageLength) or + // we run out of bytes for the current structural element and have to wait for the next chunk. + while (messageOffset < messageLength) { + if (!segmentHeaderRead) { + // We are between segments. If every segment has been processed, only the trailing message footer + // can still appear in the stream – read it (or wait for it) and exit. + if (currentSegmentNumber == numSegments) { + if (!tryReadMessageFooter(buffer)) { + break; + } + break; + } + // Otherwise, parse the next segment's header. May return false if it is split across chunks. + if (!tryReadSegmentHeader(buffer)) { + break; + } + segmentHeaderRead = true; + } + + // Drain as many payload bytes as are available into the per-segment buffer. + int payloadRead = tryReadSegmentContent(buffer); + + if (currentSegmentContentOffset == currentSegmentContentLength) { + // Segment payload fully buffered. Validate the CRC footer (if any). When the footer isn't fully + // available yet, break and resume on the next chunk – currentSegmentPayload keeps its contents so + // we can still emit them on the call where the footer arrives. + if (!tryReadSegmentFooter(buffer)) { + break; + } + // Segment passed validation: it is now safe to release the buffered payload to the caller. + releaseValidatedSegmentPayload(); + segmentHeaderRead = false; + // Loop continues: either consume the next segment's header or the message footer. + } else if (payloadRead == 0 && getAvailableBytes(buffer) == 0) { + // Nothing left to read this pass and the segment is not complete – wait for the next chunk. + break; + } + } + + return finishDecodeChunk(); + } + + /** + * Hands off validated segment bytes for emission as {@link ByteBuffer} views over the accumulated payload copies + * without consolidating them into a single array. + */ + private void releaseValidatedSegmentPayload() { + List validatedCopies = currentSegmentPayload; + currentSegmentPayload = null; + for (byte[] validatedCopy : validatedCopies) { + validatedOutput.add(ByteBuffer.wrap(validatedCopy)); + } + } + + private List finishDecodeChunk() { + if (validatedOutput.isEmpty()) { + return Collections.emptyList(); + } + List result; + if (validatedOutput.size() == 1) { + result = Collections.singletonList(validatedOutput.get(0)); + } else { + result = new ArrayList<>(validatedOutput); + } + validatedOutput.clear(); + return result; + } + + /** + * @return the total number of bytes the decoder can currently see across the carry-over {@link #pendingBytes} + * plus the unread tail of the supplied buffer. Used to decide whether a structural element (header / + * footer) can be parsed in this pass or whether we must defer to the next chunk. + */ + private int getAvailableBytes(ByteBuffer buffer) { + return pendingBytes.size() + buffer.remaining(); + } + + /** + * Returns a single read-only view that logically concatenates {@link #pendingBytes} with the unread tail of + * a buffer. + * + *

The position of the supplied buffer is intentionally not advanced here – reads happen on the + * combined view, and the original buffer's position is moved later by {@link #consumeBytes(int, ByteBuffer)} + * once we know the parse succeeded.

+ * + *

When pendingBytes is empty we avoid the allocation and just return a duplicate of the buffer; + * otherwise we materialize a fresh array of size {@code pending + buffer.remaining()}.

+ */ + private ByteBuffer getCombinedBuffer(ByteBuffer buffer) { + if (pendingBytes.size() == 0) { + ByteBuffer dup = buffer.duplicate(); + dup.order(ByteOrder.LITTLE_ENDIAN); + return dup; + } + + byte[] pending = pendingBytes.toByteArray(); + ByteBuffer combined = ByteBuffer.allocate(pending.length + buffer.remaining()); + combined.order(ByteOrder.LITTLE_ENDIAN); + combined.put(pending); + combined.put(buffer.duplicate()); + combined.flip(); + return combined; + } + + /** + * Consumes the next 8 bytes as a little-endian CRC64 footer, validates it against expectedCrc64, and + * advances {@link #messageOffset}. Used for both segment and message footers. + * + *

If fewer than 8 bytes are available, the remaining buffer bytes are stashed in {@link #pendingBytes} and + * the method returns false so the caller can break out of the decode loop and wait for the next + * chunk. On a CRC mismatch, an {@link IllegalArgumentException} is thrown (the decoder is then discarded by + * the enclosing policy).

+ */ + private boolean tryConsumeCrc64Footer(ByteBuffer buffer, long expectedCrc64, String mismatchDetail) { + if (getAvailableBytes(buffer) < CRC64_LENGTH) { + // Not enough bytes yet for the footer; carry whatever we have over to the next call. + appendToPending(buffer); + return false; + } + ByteBuffer combined = getCombinedBuffer(buffer); + long reportedCrc = combined.getLong(); + if (expectedCrc64 != reportedCrc) { + throw LOGGER.logExceptionAsError(new IllegalArgumentException(enrichExceptionMessage( + "CRC64 mismatch" + mismatchDetail + ". Expected: " + expectedCrc64 + ", got: " + reportedCrc))); + } + consumeBytes(CRC64_LENGTH, buffer); + messageOffset += CRC64_LENGTH; + return true; + } + + /** + * Drains {@code bytesToConsume} bytes from the logical pending+buffer stream that + * {@link #getCombinedBuffer(ByteBuffer)} produced. + * + *

Bytes are taken from {@link #pendingBytes} first, then from the live buffer. The pending stream is + * reset whenever it is fully drained, and any leftover (when {@code bytesToConsume} was less than what was in + * pending) is rewritten so the carry-over stays compact.

+ */ + private void consumeBytes(int bytesToConsume, ByteBuffer buffer) { + int pendingSize = pendingBytes.size(); + if (bytesToConsume <= pendingSize) { + // The entire consume fits in pending: rewrite whatever survives back into pending after a reset. + byte[] remaining = pendingBytes.toByteArray(); + pendingBytes.reset(); + if (bytesToConsume < pendingSize) { + pendingBytes.write(remaining, bytesToConsume, pendingSize - bytesToConsume); + } + } else { + // Pending is fully drained and the remainder comes from the live buffer; advance its position directly. + int bytesFromBuffer = bytesToConsume - pendingSize; + pendingBytes.reset(); + buffer.position(buffer.position() + bytesFromBuffer); + } + } + + /** + * Stashes everything still unread in the buffer into {@link #pendingBytes} so it can be combined with the + * next chunk on the next call to {@link #decodeChunk(ByteBuffer)}. + * + *

This is only called when the current chunk does not contain enough bytes for the next structural element, + * so the carry-over is always small (bounded by the largest header size, currently 13 bytes).

+ */ + private void appendToPending(ByteBuffer buffer) { + while (buffer.hasRemaining()) { + pendingBytes.write(buffer.get()); + } + } + + /** + * Reports whether the decoder has finished consuming the entire structured message and validated everything it + * was supposed to validate. Used by the pipeline policy to distinguish "stream ended cleanly" from "stream was + * truncated". + * + *

The check requires all of:

+ *
    + *
  • The message header has been parsed ({@code messageLength != -1}).
    • + *
  • Every byte of the declared message has been consumed.
    • + *
  • No carry-over bytes remain in pending.
    • + *
  • No segment is currently in progress (no segment header without a matching footer).
    • + *
  • The current segment's payload accumulation is itself complete.
    • + *
+ * + * @return true if all expected bytes have been decoded and validated; false otherwise. + */ + public boolean isComplete() { + return messageLength != -1 + && messageOffset == messageLength + && pendingBytes.size() == 0 + && !segmentHeaderRead + && currentSegmentContentOffset == currentSegmentContentLength; + } + + /** + * Appends the current decoder offset to an exception message so failures can be traced back to a specific + * point in the encoded stream. + * + * @param message The original exception message. + * @return The original message with {@code [decoderOffset=N]} appended. + */ + private String enrichExceptionMessage(String message) { + return String.format("%s [decoderOffset=%d]", message, messageOffset); + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoder.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoder.java new file mode 100644 index 000000000000..35a559799e78 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageEncoder.java @@ -0,0 +1,275 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.implementation.StorageImplUtils; +import reactor.core.publisher.Flux; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.DEFAULT_MESSAGE_VERSION; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.HashMap; +import java.util.Map; + +/** + * Encoder for structured messages with support for segmenting and CRC64 checksums. + */ +public class StructuredMessageEncoder { + private static final ClientLogger LOGGER = new ClientLogger(StructuredMessageEncoder.class); + private static final int CRC64_SCRATCH_BUFFER_SIZE = 64 * 1024; + + private final int messageVersion; + private final int contentLength; + private final int messageLength; + private final StructuredMessageFlags structuredMessageFlags; + private final int segmentSize; + private final int numSegments; + + private int currentContentOffset; + private int currentSegmentNumber; + private int currentSegmentOffset; + private long messageCRC64; + private final Map segmentCRC64s; + private final byte[] crc64ScratchBuffer; + + /** + * Constructs a new StructuredMessageEncoder. + * @param contentLength The length of the content to be encoded. + * @param segmentSize The size of each segment. + * @param structuredMessageFlags The structuredMessageFlags to be set. + * @throws IllegalArgumentException If the segment size is less than 1, the content length is less than 1, or the + * number of segments is greater than {@link java.lang.Short#MAX_VALUE}. + */ + public StructuredMessageEncoder(int contentLength, int segmentSize, StructuredMessageFlags structuredMessageFlags) { + if (segmentSize < 1) { + StorageImplUtils.assertInBounds("segmentSize", segmentSize, 1, Long.MAX_VALUE); + } + if (contentLength < 1) { + StorageImplUtils.assertInBounds("contentLength", contentLength, 1, Long.MAX_VALUE); + } + + this.messageVersion = DEFAULT_MESSAGE_VERSION; + this.contentLength = contentLength; + this.structuredMessageFlags = structuredMessageFlags; + this.segmentSize = segmentSize; + this.numSegments = Math.max(1, (int) Math.ceil((double) this.contentLength / this.segmentSize)); + this.messageLength = calculateMessageLength(); + this.currentContentOffset = 0; + this.currentSegmentNumber = 0; + this.currentSegmentOffset = 0; + this.messageCRC64 = 0; + this.segmentCRC64s = new HashMap<>(); + this.crc64ScratchBuffer = new byte[CRC64_SCRATCH_BUFFER_SIZE]; + + if (numSegments > Short.MAX_VALUE) { + StorageImplUtils.assertInBounds("numSegments", numSegments, 1, Short.MAX_VALUE); + } + } + + private int getMessageHeaderLength() { + return V1_HEADER_LENGTH; + } + + private int getSegmentHeaderLength() { + return V1_SEGMENT_HEADER_LENGTH; + } + + private int getSegmentFooterLength() { + return (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) ? CRC64_LENGTH : 0; + } + + private int getMessageFooterLength() { + return (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) ? CRC64_LENGTH : 0; + } + + private int getSegmentContentLength() { + // last segment size is remaining content + if (currentSegmentNumber == numSegments) { + return contentLength - ((currentSegmentNumber - 1) * segmentSize); + } else { + return segmentSize; + } + } + + private byte[] generateMessageHeader() { + // 1 byte version, 8 byte size, 2 byte structuredMessageFlags, 2 byte numSegments + ByteBuffer buffer = ByteBuffer.allocate(getMessageHeaderLength()).order(ByteOrder.LITTLE_ENDIAN); + buffer.put((byte) messageVersion); + buffer.putLong(messageLength); + buffer.putShort((short) structuredMessageFlags.getValue()); + buffer.putShort((short) numSegments); + + return buffer.array(); + } + + private byte[] generateSegmentHeader() { + int segmentContentSize = Math.min(segmentSize, contentLength - currentContentOffset); + // 2 byte number, 8 byte size + ByteBuffer buffer = ByteBuffer.allocate(getSegmentHeaderLength()).order(ByteOrder.LITTLE_ENDIAN); + buffer.putShort((short) currentSegmentNumber); + buffer.putLong(segmentContentSize); + + return buffer.array(); + } + + /** + * Encodes the given buffer into a structured message format as a stream of ByteBuffers. + * + * @param unencodedBuffer The buffer to be encoded. + * @return A Flux of encoded ByteBuffers. + * @throws IllegalArgumentException If the buffer length exceeds the content length, or the content has already been + * encoded. + */ + public Flux encode(ByteBuffer unencodedBuffer) { + StorageImplUtils.assertNotNull("unencodedBuffer", unencodedBuffer); + + return Flux.defer(() -> { + if (currentContentOffset == contentLength) { + // Already encoded; return empty (e.g. extra aggregator from staging/flush, or retry re-subscription). + return Flux.empty(); + } + + if ((unencodedBuffer.remaining() + currentContentOffset) > contentLength) { + return Flux.error( + LOGGER.logExceptionAsError(new IllegalArgumentException("Buffer length exceeds content length."))); + } + + if (!unencodedBuffer.hasRemaining()) { + return Flux.empty(); + } + + // Emit buffers lazily to avoid materializing full encoded output in memory + // Could be swapped to Flux.generate if performance is impacted and we're eagerly pushing a lot of data. + return Flux.create(sink -> { + // if we are at the beginning of the message, encode message header and emit it + if (currentContentOffset == 0) { + sink.next(ByteBuffer.wrap(generateMessageHeader())); + } + + // while there are remaining bytes in the unencoded buffer, encode the segment content + while (unencodedBuffer.hasRemaining()) { + // if we are at the beginning of a segment's content, encode segment header and emit it + if (currentSegmentOffset == 0) { + incrementCurrentSegment(); + sink.next(ByteBuffer.wrap(generateSegmentHeader())); + } + + // encode the segment content and emit it + sink.next(encodeSegmentContent(unencodedBuffer)); + + // if we are at the end of a segment's content, encode segment footer + if (currentSegmentOffset == getSegmentContentLength()) { + byte[] footer = generateSegmentFooter(); + if (footer.length > 0) { + sink.next(ByteBuffer.wrap(footer)); + } + currentSegmentOffset = 0; + } + } + + // if all content has been encoded, encode message footer and emit it + if (currentContentOffset == contentLength) { + byte[] footer = generateMessageFooter(); + if (footer.length > 0) { + sink.next(ByteBuffer.wrap(footer)); + } + } + + sink.complete(); + }); + }); + } + + private byte[] generateSegmentFooter() { + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + return ByteBuffer.allocate(CRC64_LENGTH) + .order(ByteOrder.LITTLE_ENDIAN) + .putLong(segmentCRC64s.get(currentSegmentNumber)) + .array(); + } + return new byte[0]; + } + + private byte[] generateMessageFooter() { + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + return ByteBuffer.allocate(CRC64_LENGTH).order(ByteOrder.LITTLE_ENDIAN).putLong(messageCRC64).array(); + } + return new byte[0]; + } + + private ByteBuffer encodeSegmentContent(ByteBuffer unencodedBuffer) { + // get the number of bytes to read from the unencoded buffer based on the segment content length and the current segment offset + int readSize = Math.min(unencodedBuffer.remaining(), getSegmentContentLength() - currentSegmentOffset); + + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + if (unencodedBuffer.hasArray()) { + // if the unencoded buffer has an array, compute the CRC64 checksum of the segment content + // this is more efficient than copying the array to a new byte array and computing the checksum + int pos = unencodedBuffer.arrayOffset() + unencodedBuffer.position(); + segmentCRC64s.put(currentSegmentNumber, StorageCrc64Calculator.compute(unencodedBuffer.array(), pos, + readSize, segmentCRC64s.get(currentSegmentNumber))); + messageCRC64 = StorageCrc64Calculator.compute(unencodedBuffer.array(), pos, readSize, messageCRC64); + } else { + updateCrc64sWithoutAccessibleArray(unencodedBuffer, readSize); + } + } + + currentContentOffset += readSize; + currentSegmentOffset += readSize; + + // Return a view (slice) to avoid allocating 4MB per segment; caller must consume before next segment. + ByteBuffer slice = unencodedBuffer.slice(); + slice.limit(readSize); + unencodedBuffer.position(unencodedBuffer.position() + readSize); + return slice.asReadOnlyBuffer(); + } + + private void updateCrc64sWithoutAccessibleArray(ByteBuffer unencodedBuffer, int readSize) { + ByteBuffer duplicate = unencodedBuffer.duplicate(); + duplicate.limit(duplicate.position() + readSize); + + long segmentCrc64 = segmentCRC64s.get(currentSegmentNumber); + long currentMessageCrc64 = messageCRC64; + + while (duplicate.hasRemaining()) { + int chunkSize = Math.min(duplicate.remaining(), crc64ScratchBuffer.length); + duplicate.get(crc64ScratchBuffer, 0, chunkSize); + segmentCrc64 = StorageCrc64Calculator.compute(crc64ScratchBuffer, 0, chunkSize, segmentCrc64); + currentMessageCrc64 = StorageCrc64Calculator.compute(crc64ScratchBuffer, 0, chunkSize, currentMessageCrc64); + } + + segmentCRC64s.put(currentSegmentNumber, segmentCrc64); + messageCRC64 = currentMessageCrc64; + } + + private int calculateMessageLength() { + int length = getMessageHeaderLength(); + + length += (getSegmentHeaderLength() + getSegmentFooterLength()) * numSegments; + length += contentLength; + length += getMessageFooterLength(); + return length; + } + + private void incrementCurrentSegment() { + currentSegmentNumber++; + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + segmentCRC64s.putIfAbsent(currentSegmentNumber, 0L); + } + } + + /** + * Returns the length of the message. + * + * @return The length of the message. + */ + public long getEncodedMessageLength() { + return messageLength; + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlags.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlags.java new file mode 100644 index 000000000000..a5ccb2974a6e --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlags.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +/** + * Defines values for StructuredMessageFlags. + */ +public enum StructuredMessageFlags { + /** + * No flags set. + */ + NONE(0), + + /** + * StructuredMessageFlag indicating the use of CRC64. + */ + STORAGE_CRC64(1); + + /** + * The actual serialized value for a StructuredMessageFlags instance. + */ + private final int value; + + StructuredMessageFlags(int value) { + this.value = value; + } + + /** + * Parses a serialized value to a StructuredMessageFlags instance. + * + * @param value the serialized value to parse. + * @return the parsed StructuredMessageFlags object, or null if unable to parse. + */ + public static StructuredMessageFlags fromString(String value) { + if (value == null) { + return null; + } + StructuredMessageFlags[] items = StructuredMessageFlags.values(); + for (StructuredMessageFlags item : items) { + if (item.getValue() == Integer.parseInt(value)) { + return item; + } + } + return null; + } + + /** + * Parses a serialized value to a StructuredMessageFlags instance. + * @param value the serialized value to parse. + * @return the parsed StructuredMessageFlags object. + * @throws IllegalArgumentException if unable to parse. + */ + public static StructuredMessageFlags fromValue(int value) { + for (StructuredMessageFlags flag : StructuredMessageFlags.values()) { + if (flag.getValue() == value) { + return flag; + } + } + throw new IllegalArgumentException("Invalid value for StructuredMessageFlags: " + value); + } + + /** + * Returns the value for a StructuredMessageFlags instance. + * + * @return the integer value of the StructuredMessageFlags object. + */ + public int getValue() { + return value; + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/package-info.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/package-info.java new file mode 100644 index 000000000000..74c1334e3a7a --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/implementation/contentvalidation/package-info.java @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** + * Package containing classes for structured message encoding and decoding. + */ +package com.azure.storage.common.implementation.contentvalidation; diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java new file mode 100644 index 000000000000..7e88de8cd37e --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/DecodedResponse.java @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpResponse; +import com.azure.core.util.CoreUtils; +import com.azure.core.util.FluxUtil; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +/** + * {@link HttpResponse} wrapper that exposes a decoded body stream while preserving the request and status code of + * the original response. + * + *

The policy hands this class a Flux that already represents validated, framing-stripped bytes (produced by the + * decoder pipeline). This class's only job is to make that Flux look like the body of the original + * {@link HttpResponse}. {@code Content-Length} is overridden to the decoded payload size so it matches what callers + * will actually read; all other headers are forwarded verbatim. The validation is transparent to callers.

+ */ +class DecodedResponse extends HttpResponse { + private final HttpResponse originalResponse; + private final Flux decodedBody; + private final HttpHeaders adjustedHeaders; + + /** + * Wraps {@code httpResponse} with a body backed by {@code decodedBody}. + * + *

{@code Content-Length} is overridden to {@code decodedContentLength} so callers see the size of the bytes + * they will actually read from the decoded payload, not the larger wire size of the structured message.

+ * + * @param httpResponse The original response from the storage service. + * @param decodedBody The Flux of CRC-validated, framing-stripped payload bytes produced by the decoder pipeline. + * @param decodedContentLength The size of the decoded payload that callers will consume. + */ + DecodedResponse(HttpResponse httpResponse, Flux decodedBody, long decodedContentLength) { + super(httpResponse.getRequest()); + this.originalResponse = httpResponse; + this.decodedBody = decodedBody; + HttpHeaders headers = new HttpHeaders(httpResponse.getHeaders()); + headers.set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(decodedContentLength)); + this.adjustedHeaders = headers; + } + + @Override + public int getStatusCode() { + return originalResponse.getStatusCode(); + } + + @Override + @SuppressWarnings("deprecation") + public String getHeaderValue(String name) { + return adjustedHeaders.getValue(name); + } + + @Override + public HttpHeaders getHeaders() { + return adjustedHeaders; + } + + @Override + public Flux getBody() { + return decodedBody; + } + + @Override + public Mono getBodyAsByteArray() { + return FluxUtil.collectBytesInByteBufferStream(decodedBody); + } + + @Override + public Mono getBodyAsString() { + return getBodyAsByteArray() + .map(b -> CoreUtils.bomAwareToString(b, adjustedHeaders.getValue(HttpHeaderName.CONTENT_TYPE))); + } + + @Override + public Mono getBodyAsString(Charset charset) { + return FluxUtil.collectBytesInByteBufferStream(decodedBody).map(b -> new String(b, charset)); + } +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java new file mode 100644 index 000000000000..21150174a03a --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicy.java @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpPipelineCallContext; +import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpResponse; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageDecoder; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * HTTP pipeline policy that decodes the storage structured message body returned for downloads when CRC64 + * content validation is active. + * + *

The policy decides when to opt in (via the context key), tells the service to + * encode the response (via the request header), constructs the decoder and the wrapper response, and + * translates decoder-level failures (malformed framing, CRC mismatch, premature end-of-stream) into reactive + * {@link IOException} errors.

+ * + *

This policy uses {@link com.azure.core.http.HttpPipelinePosition#PER_RETRY PER_RETRY} semantics by default, so + * each retry produces a fresh response that this policy wraps with a fresh decoder. A CRC failure on one attempt + * cannot pollute another, and the storage download retry logic ({@code BlobAsyncClientBase.downloadStream...}) can + * resume by reissuing range requests; each new range response is validated end-to-end on its own.

+ * + *

Because the wrapped {@link StructuredMessageDecoder} only releases payload bytes after the corresponding + * segment's CRC has been verified, the {@link DecodedResponse}'s body Flux is guaranteed to contain only validated + * bytes – callers never see a byte that could later fail validation, even when retries are involved.

+ */ +public class StorageContentValidationDecoderPolicy implements HttpPipelinePolicy { + private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationDecoderPolicy.class); + + /** + * Creates a new instance of {@link StorageContentValidationDecoderPolicy}. + */ + public StorageContentValidationDecoderPolicy() { + } + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + // Check if the decoding should be applied. + if (!shouldApplyDecoding(context)) { + return next.process(); + } + + // Tell the service we want a structured-message body. + context.getHttpRequest() + .getHeaders() + .set(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME, + StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE); + + return next.process().map(httpResponse -> { + // The HTTP Content-Length is the size of the encoded structured message body. We hand it to the + // decoder which cross-checks it against the message header. + Long contentLength = getContentLength(httpResponse.getHeaders()); + + // Only 2xx GET responses with a positive content length carry a body that we can decode. + if (!isEligibleDownload(httpResponse, contentLength)) { + return httpResponse; + } + + // Confirm the service honored our structured-body request and parse the decoded length in one step, + // failing fast with a consistent error if either header is absent or not parseable as a long. + long decodedContentLength = validateAndGetDecodedContentLength(httpResponse); + + // Fresh decoder per response so retries each get a clean state machine. + StructuredMessageDecoder decoder = new StructuredMessageDecoder(contentLength); + + Flux decodedStream = decodeStream(httpResponse.getBody(), decoder); + return new DecodedResponse(httpResponse, decodedStream, decodedContentLength); + }); + } + + /** + * @return true when the request carries the boolean opt-in flag set + * by {@code ContentValidationModeResolver.addStructuredMessageDecodingToContext}. + */ + private boolean shouldApplyDecoding(HttpPipelineCallContext context) { + return context.getData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY) + .map(value -> value instanceof Boolean && (Boolean) value) + .orElse(false); + } + + /** + * Verifies the response acknowledges the structured-body request ({@code x-ms-structured-body} present) and + * parses {@code x-ms-structured-content-length} in one step. Throws with a consistent error message if either + * header is absent or the length value is not parseable as a {@code long}, matching the fail-fast behaviour + * used for other validation failures. + * + * @return the decoded payload size declared by the service. + */ + private long validateAndGetDecodedContentLength(HttpResponse httpResponse) { + String structuredBody + = httpResponse.getHeaders().getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME); + String structuredContentLength + = httpResponse.getHeaders().getValue(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME); + if (structuredBody == null || structuredContentLength == null) { + throw LOGGER.logExceptionAsError( + new IllegalStateException("Structured message was requested but the response did not acknowledge it.")); + } + try { + return Long.parseLong(structuredContentLength); + } catch (NumberFormatException e) { + throw LOGGER.logExceptionAsError(new IllegalStateException("x-ms-structured-content-length header value '" + + structuredContentLength + "' could not be parsed as a long.", e)); + } + } + + /** + * Reads {@code Content-Length} as a {@code long}, returning {@code null} when the header is missing or + * not parseable so callers can simply skip decoding for non-bodied responses. + */ + private static Long getContentLength(HttpHeaders headers) { + String value = headers.getValue(HttpHeaderName.CONTENT_LENGTH); + if (value != null) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + // Header invalid; treat as not eligible. + } + } + return null; + } + + /** + * @return true for a 2xx response to a GET request, the only response shape that carries a body we + * can decode. 206 (Partial Content) on retried range downloads is included. + */ + private static boolean isDownloadResponse(HttpResponse response) { + return response.getRequest().getHttpMethod() == HttpMethod.GET && response.getStatusCode() / 100 == 2; + } + + /** + * @return true when the response is one we should decode: a 2xx GET with a positive, parseable + * {@code Content-Length}. + */ + private static boolean isEligibleDownload(HttpResponse response, Long contentLength) { + return isDownloadResponse(response) && contentLength != null && contentLength > 0; + } + + /** + * Builds the body-decoding Flux: each upstream {@link ByteBuffer} is fed to the decoder in order + * ({@code concatMap} preserves order and serializes access), and a deferred stream-completion check is + * appended so a truncated body raises an error instead of completing silently. + */ + private Flux decodeStream(Flux encodedFlux, StructuredMessageDecoder decoder) { + // limitRate(1) mirrors StorageContentValidationPolicy's upload path: process one wire buffer at a time so + // the decoder can copy only the current chunk into owned storage and release the inbound buffer before the + // next segment payload bytes arrive. + return encodedFlux.limitRate(1) + .concatMap(buffer -> decodeBuffer(buffer, decoder)) + .concatWith(Mono.defer(() -> handleStreamCompletion(decoder))); + } + + /** + * Feeds a single inbound chunk to the decoder and translates its outputs into reactive emissions: + * If the decoder reports validated bytes, emit them downstream. + * If the decoder threw because the input is malformed or a CRC mismatch was detected, surface that as + * an {@link IOException}. + * If the decoder is already complete (e.g., extra trailing bytes after the message footer), drop the + * chunk silently. + */ + private Flux decodeBuffer(ByteBuffer buffer, StructuredMessageDecoder decoder) { + if (decoder.isComplete()) { + // Decoding finished on a previous chunk; ignore any trailing bytes the transport might still emit. + return Flux.empty(); + } + + if (buffer == null || !buffer.hasRemaining()) { + return Flux.empty(); + } + + try { + return Flux.fromIterable(decoder.decodeChunk(buffer)); + } catch (IllegalArgumentException e) { + return Flux.error(new IOException("Failed to decode structured message: " + e.getMessage(), e)); + } catch (Exception e) { + // Anything not foreseen by the decoder, log it. + LOGGER.error("Failed to decode structured message chunk: " + e.getMessage(), e); + return Flux.error(new IOException("Failed to decode structured message chunk: " + e.getMessage(), e)); + } + } + + /** + * Run after the upstream Flux completes. If the decoder is not in a complete state, the response body ended + * before all expected bytes arrived – surface this as an {@link IOException} so callers don't accept a + * truncated payload. + */ + private Mono handleStreamCompletion(StructuredMessageDecoder decoder) { + if (!decoder.isComplete()) { + return Mono.error(new IOException("Stream ended prematurely before structured message decoding completed")); + } + return Mono.empty(); + } + +} diff --git a/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationPolicy.java b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationPolicy.java new file mode 100644 index 000000000000..a449e37a8c2f --- /dev/null +++ b/sdk/storage/azure-storage-common/src/main/java/com/azure/storage/common/policy/StorageContentValidationPolicy.java @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpPipelineCallContext; +import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpResponse; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.util.FluxUtil; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.common.implementation.contentvalidation.StorageCrc64Calculator; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageEncoder; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageFlags; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.Optional; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CONTENT_VALIDATION_MODE_KEY; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_CRC64_CHECKSUM_HEADER_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_STRUCTURED_MESSAGE_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_DEFAULT_SEGMENT_CONTENT_LENGTH; +import static com.azure.storage.common.implementation.Constants.HeaderConstants.CONTENT_CRC64_HEADER_NAME; +import static com.azure.storage.common.implementation.Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME; +import static com.azure.storage.common.implementation.Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE; + +/** + * StorageContentValidationPolicy is a policy that applies content validation to the request body. + */ +public class StorageContentValidationPolicy implements HttpPipelinePolicy { + private static final ClientLogger LOGGER = new ClientLogger(StorageContentValidationPolicy.class); + + /** + * Creates a new instance of {@link StorageContentValidationPolicy}. + */ + public StorageContentValidationPolicy() { + } + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + // Defer creating the next policy Mono until validation (and any required header mutations) has completed. + // Some downstream policies may compute auth/signatures eagerly in their `process()` method. + return applyContentValidation(context).then(Mono.defer(next::process)); + } + + /** + * Applies content validation to the request body. + * + * @param context the HTTP pipeline call context + * @return a {@link Mono} that completes when content validation has been applied to the request body. + */ + private Mono applyContentValidation(HttpPipelineCallContext context) { + Optional behaviorOptional = context.getContext().getData(CONTENT_VALIDATION_MODE_KEY); + if (!behaviorOptional.isPresent()) { + return Mono.empty(); + } + + String contentValidationBehavior = behaviorOptional.get().toString(); + if (contentValidationBehavior.isEmpty()) { + return Mono.empty(); + } + + Mono validation = Mono.empty(); + + if (contentValidationBehavior.contains(USE_CRC64_CHECKSUM_HEADER_CONTEXT)) { + validation = validation.then(applyCRC64Header(context)); + } + if (contentValidationBehavior.contains(USE_STRUCTURED_MESSAGE_CONTEXT)) { + validation = validation.then(applyStructuredMessage(context)); + } + + return validation; + } + + /** + * Applies the crc64 header to the request body. + * + * @param context the HTTP pipeline call context + * @return a {@link Mono} that completes when the crc64 header has been applied to the request body. + */ + private Mono applyCRC64Header(HttpPipelineCallContext context) { + if (context.getHttpRequest().getBody() == null) { + return Mono.empty(); + } + + // Collect request body bytes once, compute CRC64 off the reactive thread, and then restore the body + // as a replayable Flux so downstream processing / sending still sees the expected content. + Flux originalBody = context.getHttpRequest().getBody(); + return FluxUtil.collectBytesInByteBufferStream(originalBody) + .flatMap(originalBytes -> Mono.fromCallable(() -> StorageCrc64Calculator.compute(originalBytes, 0)) + // Run CRC64 on boundedElastic so synchronous work over the full body does not block the reactive + // I/O thread (e.g. Netty event loop) that drives the pipeline. + .subscribeOn(Schedulers.boundedElastic()) + .flatMap(contentCRC64 -> { + // Restore body for downstream consumers. + context.getHttpRequest().setBody(Flux.just(ByteBuffer.wrap(originalBytes))); + + // Convert the 64-bit CRC value to 8 bytes in little-endian format. + byte[] crc64Bytes = new byte[8]; + for (int i = 0; i < 8; i++) { + crc64Bytes[i] = (byte) (contentCRC64 >>> (i * 8)); + } + + // Base64 encode the binary representation. + String encodedCRC64 = Base64.getEncoder().encodeToString(crc64Bytes); + context.getHttpRequest().setHeader(CONTENT_CRC64_HEADER_NAME, encodedCRC64); + return Mono.empty(); + })); + } + + /** + * Applies the structured message to the request body. + * + * @param context the HTTP pipeline call context + * @return a {@link Mono} that completes when the structured message has been applied, or fails if the request is + * not valid for structured messaging + */ + private Mono applyStructuredMessage(HttpPipelineCallContext context) { + String contentLengthValue = context.getHttpRequest().getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH); + if (contentLengthValue == null || contentLengthValue.isEmpty()) { + return FluxUtil.monoError(LOGGER, + LOGGER.logExceptionAsError( + new IllegalArgumentException("Content-Length header is required to apply structured message " + + "and CRC64 encoding, but it was not present on the request."))); + } + + long parsedContentLength; + try { + parsedContentLength = Long.parseLong(contentLengthValue); + } catch (NumberFormatException ex) { + return FluxUtil.monoError(LOGGER, LOGGER.logExceptionAsError(new IllegalArgumentException( + "Content-Length header value '" + contentLengthValue + + "' is not a valid non-negative integer value required for structured message and CRC64 encoding.", + ex))); + } + + int unencodedContentLength = (int) parsedContentLength; + + Flux originalBody = context.getHttpRequest().getBody(); + + /* + * Replace the request body with a structured message: raw content wrapped with headers, segment + * boundaries, and CRC64 checksums so the service can validate integrity as it receives the stream. + * + * A fresh encoder is created on each subscribe (via defer) so retries re-encode correctly from the + * original replayable body. The encoded buffers are slices of the original data, produced lazily and + * consumed by the HTTP client without materialization. + * + * limitRate(1) keeps the encoder's segment boundaries aligned with buffer boundaries. + */ + Flux encodedBody = Flux.defer(() -> { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(unencodedContentLength, + V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64); + return Flux.from(originalBody).limitRate(1).concatMap(encoder::encode); + }); + + context.getHttpRequest().setBody(encodedBody); + + long encodedLength = new StructuredMessageEncoder(unencodedContentLength, V1_DEFAULT_SEGMENT_CONTENT_LENGTH, + StructuredMessageFlags.STORAGE_CRC64).getEncodedMessageLength(); + context.getHttpRequest().setHeader(HttpHeaderName.CONTENT_LENGTH, String.valueOf(encodedLength)); + context.getHttpRequest().setHeader(STRUCTURED_BODY_TYPE_HEADER_NAME, STRUCTURED_BODY_TYPE_VALUE); + context.getHttpRequest() + .setHeader(STRUCTURED_CONTENT_LENGTH_HEADER_NAME, String.valueOf(unencodedContentLength)); + + return Mono.empty(); + } + +} diff --git a/sdk/storage/azure-storage-common/src/main/java/module-info.java b/sdk/storage/azure-storage-common/src/main/java/module-info.java index f5f1cfa99b2b..412a11fe41d6 100644 --- a/sdk/storage/azure-storage-common/src/main/java/module-info.java +++ b/sdk/storage/azure-storage-common/src/main/java/module-info.java @@ -25,4 +25,7 @@ exports com.azure.storage.common.implementation.connectionstring to // FIXME this should not be a long-term solution com.azure.data.tables, com.azure.storage.blob, com.azure.storage.blob.cryptography, com.azure.storage.file.share, com.azure.storage.file.datalake, com.azure.storage.queue; + + exports com.azure.storage.common.implementation.contentvalidation to // FIXME this should not be a long-term solution + com.azure.storage.blob, com.azure.storage.file.share, com.azure.storage.file.datalake; } diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java index 5e84dd31947d..ddd488ff0887 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockDownloadHttpResponse.java @@ -6,10 +6,10 @@ import com.azure.core.http.HttpHeaderName; import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpResponse; +import com.azure.core.util.FluxUtil; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -19,14 +19,21 @@ with than was worth it. Because this type is just for BlobDownload, we don't need to accept a header type. */ public class MockDownloadHttpResponse extends HttpResponse { + private final HttpResponse originalResponse; private final int statusCode; private final HttpHeaders headers; private final Flux body; public MockDownloadHttpResponse(HttpResponse response, int statusCode, Flux body) { + this(response, statusCode, response.getHeaders(), body); + } + + public MockDownloadHttpResponse(HttpResponse response, int statusCode, HttpHeaders headers, + Flux body) { super(response.getRequest()); + this.originalResponse = response; this.statusCode = statusCode; - this.headers = response.getHeaders(); + this.headers = headers; this.body = body; } @@ -52,21 +59,26 @@ public HttpHeaders getHeaders() { @Override public Flux getBody() { - return body; + return Flux.using(() -> originalResponse, ignored -> body, HttpResponse::close); } @Override public Mono getBodyAsByteArray() { - return Mono.error(new IOException()); + return FluxUtil.collectBytesInByteBufferStream(getBody()); } @Override public Mono getBodyAsString() { - return Mono.error(new IOException()); + return getBodyAsByteArray().map(bytes -> new String(bytes, Charset.defaultCharset())); } @Override public Mono getBodyAsString(Charset charset) { - return Mono.error(new IOException()); + return getBodyAsByteArray().map(bytes -> new String(bytes, charset)); + } + + @Override + public void close() { + originalResponse.close(); } } diff --git a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java index 1f1109d8b38b..347d3ac11a59 100644 --- a/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java +++ b/sdk/storage/azure-storage-common/src/test-shared/java/com/azure/storage/common/test/shared/policy/MockPartialResponsePolicy.java @@ -8,6 +8,7 @@ import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpPipelineCallContext; import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpPipelinePosition; import com.azure.core.http.HttpResponse; import com.azure.core.http.policy.HttpPipelinePolicy; import reactor.core.publisher.Flux; @@ -16,50 +17,137 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; public class MockPartialResponsePolicy implements HttpPipelinePolicy { - static final HttpHeaderName RANGE_HEADER = HttpHeaderName.fromString("x-ms-range"); - private int tries; - private final List rangeHeaders = new ArrayList<>(); + static final HttpHeaderName X_MS_RANGE_HEADER = HttpHeaderName.fromString("x-ms-range"); + static final HttpHeaderName RANGE_HEADER = HttpHeaderName.RANGE; + private final AtomicInteger tries; + private final List rangeHeaders = Collections.synchronizedList(new ArrayList<>()); + private final int maxBytesPerResponse; + private final AtomicInteger hits = new AtomicInteger(); + private final String targetUrlPrefix; + /** + * Creates a MockPartialResponsePolicy that simulates network interruptions. + * + * @param tries Number of times to simulate interruptions (0 = no interruptions) + */ public MockPartialResponsePolicy(int tries) { - this.tries = tries; + this(tries, 200, null); + } + + /** + * Creates a MockPartialResponsePolicy with configurable interruption behavior. + * + * @param tries Number of times to simulate interruptions (0 = no interruptions) + * @param maxBytesPerResponse Maximum bytes to return in each interrupted response + */ + public MockPartialResponsePolicy(int tries, int maxBytesPerResponse) { + this(tries, maxBytesPerResponse, null); + } + + /** + * Creates a MockPartialResponsePolicy with configurable interruption behavior and an optional URL filter. + * + * @param tries Number of times to simulate interruptions (0 = no interruptions) + * @param maxBytesPerResponse Maximum bytes to return in each interrupted response + * @param targetUrlPrefix If non-null, only requests whose URL starts with this prefix will be interrupted. + */ + public MockPartialResponsePolicy(int tries, int maxBytesPerResponse, String targetUrlPrefix) { + this.tries = new AtomicInteger(tries); + this.maxBytesPerResponse = maxBytesPerResponse; + this.targetUrlPrefix = targetUrlPrefix; + } + + @Override + public HttpPipelinePosition getPipelinePosition() { + return HttpPipelinePosition.PER_RETRY; } @Override public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { return next.process().flatMap(response -> { HttpHeader rangeHttpHeader = response.getRequest().getHeaders().get(RANGE_HEADER); - String rangeHeader = rangeHttpHeader == null ? null : rangeHttpHeader.getValue(); + HttpHeader xMsRangeHttpHeader = response.getRequest().getHeaders().get(X_MS_RANGE_HEADER); - if (rangeHeader != null && rangeHeader.startsWith("bytes=")) { - rangeHeaders.add(rangeHeader); + if (response.getRequest().getHttpMethod() == HttpMethod.GET) { + String recordedRange = null; + if (rangeHttpHeader != null && rangeHttpHeader.getValue().startsWith("bytes=")) { + recordedRange = rangeHttpHeader.getValue(); + } else if (xMsRangeHttpHeader != null && xMsRangeHttpHeader.getValue().startsWith("bytes=")) { + recordedRange = xMsRangeHttpHeader.getValue(); + } + rangeHeaders.add(recordedRange == null ? "" : recordedRange); } - if ((response.getRequest().getHttpMethod() != HttpMethod.GET) || this.tries == 0) { + boolean urlMatches = targetUrlPrefix == null + || response.getRequest().getUrl().toString().startsWith(targetUrlPrefix); + + if ((response.getRequest().getHttpMethod() != HttpMethod.GET) || !urlMatches) { return Mono.just(response); } else { - this.tries -= 1; - return response.getBody().collectList().flatMap(bodyBuffers -> { - ByteBuffer firstBuffer = bodyBuffers.get(0); - byte firstByte = firstBuffer.get(); - - // Simulate partial response by returning the first byte only from the requested range and timeout - return Mono.just(new MockDownloadHttpResponse(response, 206, - Flux.just(ByteBuffer.wrap(new byte[] { firstByte })) - .concatWith(Flux.error(new IOException("Simulated timeout"))) - )); - }); + int remainingTries = this.tries.getAndUpdate(value -> value > 0 ? value - 1 : value); + if (remainingTries <= 0) { + return Mono.just(response); + } + hits.incrementAndGet(); + + Flux limitedBody = limitStreamToBytes(response.getBody(), maxBytesPerResponse); + return Mono.just( + new MockDownloadHttpResponse(response, response.getStatusCode(), response.getHeaders(), + limitedBody)); } }); } + private Flux limitStreamToBytes(Flux body, int maxBytes) { + return Flux.defer(() -> { + final long[] bytesEmitted = new long[] { 0 }; + return body.concatMap(buffer -> { + if (buffer == null || !buffer.hasRemaining()) { + return Flux.just(buffer); + } + + long remaining = maxBytes - bytesEmitted[0]; + if (remaining <= 0) { + return Flux.error(new IOException("Simulated timeout")); + } + + int bufferSize = buffer.remaining(); + if (bufferSize <= remaining) { + bytesEmitted[0] += bufferSize; + if (bytesEmitted[0] >= maxBytes) { + return Flux.just(buffer).concatWith(Flux.error(new IOException("Simulated timeout"))); + } + return Flux.just(buffer); + } else { + int bytesToEmit = (int) remaining; + ByteBuffer slice = buffer.duplicate(); + slice.limit(slice.position() + bytesToEmit); + + ByteBuffer limited = ByteBuffer.allocate(bytesToEmit); + limited.put(slice); + limited.flip(); + + bytesEmitted[0] += bytesToEmit; + return Flux.just(limited).concatWith(Flux.error(new IOException("Simulated timeout"))); + } + }); + }); + } + public int getTriesRemaining() { - return tries; + return tries.get(); } public List getRangeHeaders() { return rangeHeaders; } + + public int getHits() { + return hits.get(); + } } diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolverTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolverTests.java new file mode 100644 index 000000000000..14408a4163bb --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/ContentValidationModeResolverTests.java @@ -0,0 +1,236 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.Context; +import com.azure.core.util.ProgressListener; +import com.azure.storage.common.ContentValidationAlgorithm; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import reactor.core.publisher.Mono; +import reactor.test.StepVerifier; + +import java.util.stream.Stream; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CONTENT_VALIDATION_MODE_KEY; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_CRC64_CHECKSUM_HEADER_CONTEXT; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.USE_STRUCTURED_MESSAGE_CONTEXT; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ContentValidationModeResolverTests { + + private static String modeOnContext(Context context, ContentValidationAlgorithm algorithm, long contentLength, + boolean chunkedUpload) { + return ContentValidationModeResolver.addContentValidationMode(context, algorithm, contentLength, chunkedUpload) + .getData(CONTENT_VALIDATION_MODE_KEY) + .map(Object::toString) + .orElse(null); + } + + // =========================================================================================== + // addContentValidationMode (Context) — single-part + // =========================================================================================== + + static Stream singlePartDoesNotSetModeSupplier() { + return Stream.of(Arguments.of(null, 1024), Arguments.of(ContentValidationAlgorithm.NONE, 1024), + Arguments.of(null, 8 * 1024 * 1024), Arguments.of(ContentValidationAlgorithm.NONE, 8 * 1024 * 1024)); + } + + @ParameterizedTest + @MethodSource("singlePartDoesNotSetModeSupplier") + public void singlePartDoesNotSetModeForNullOrNone(ContentValidationAlgorithm algorithm, long length) { + assertEquals(null, modeOnContext(Context.NONE, algorithm, length, false)); + } + + @Test + public void singlePartSmallUploadUsesCrc64Header() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + assertEquals(USE_CRC64_CHECKSUM_HEADER_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.CRC64, underThreshold, false)); + } + + @Test + public void singlePartAtExact4MBBoundaryUsesStructuredMessage() { + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, modeOnContext(Context.NONE, ContentValidationAlgorithm.CRC64, + MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER, false)); + } + + @Test + public void singlePartLargeUploadUsesStructuredMessage() { + long overThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER + 1; + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.CRC64, overThreshold, false)); + } + + @Test + public void singlePartAutoSmallUploadUsesCrc64Header() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + assertEquals(USE_CRC64_CHECKSUM_HEADER_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.AUTO, underThreshold, false)); + } + + @Test + public void singlePartAutoAtExact4MBBoundaryUsesStructuredMessage() { + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, modeOnContext(Context.NONE, ContentValidationAlgorithm.AUTO, + MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER, false)); + } + + @Test + public void singlePartAutoLargeUploadUsesStructuredMessage() { + long overThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER + 1; + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.AUTO, overThreshold, false)); + } + + @Test + public void addContentValidationModeNullContextUsesNone() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + assertEquals(USE_CRC64_CHECKSUM_HEADER_CONTEXT, + modeOnContext(null, ContentValidationAlgorithm.CRC64, underThreshold, false)); + } + + // =========================================================================================== + // addContentValidationMode (Context) — chunked + // =========================================================================================== + + static Stream chunkedDoesNotSetModeSupplier() { + return Stream.of(Arguments.of((ContentValidationAlgorithm) null), + Arguments.of(ContentValidationAlgorithm.NONE)); + } + + @ParameterizedTest + @MethodSource("chunkedDoesNotSetModeSupplier") + public void chunkedDoesNotSetModeForNonCrc64Algorithms(ContentValidationAlgorithm algorithm) { + assertEquals(null, modeOnContext(Context.NONE, algorithm, 1024, true)); + } + + @Test + public void chunkedCrc64AlwaysUsesStructuredMessage() { + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.CRC64, 1024, true)); + } + + @Test + public void chunkedAutoAlwaysUsesStructuredMessage() { + assertEquals(USE_STRUCTURED_MESSAGE_CONTEXT, + modeOnContext(Context.NONE, ContentValidationAlgorithm.AUTO, 1024, true)); + } + + // =========================================================================================== + // addContentValidationMode (Mono) + // =========================================================================================== + + @Test + public void addContentValidationModeMonoWritesReactorContextForCrc64() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + Mono source = Mono.deferContextual(ctx -> Mono.just(ctx.get(CONTENT_VALIDATION_MODE_KEY))); + Mono augmented = ContentValidationModeResolver.addContentValidationMode(source, + ContentValidationAlgorithm.CRC64, underThreshold, false); + StepVerifier.create(augmented).expectNext(USE_CRC64_CHECKSUM_HEADER_CONTEXT).verifyComplete(); + } + + @Test + public void addContentValidationModeMonoWritesReactorContextForAuto() { + long underThreshold = MAXIMUM_SINGLE_SHOT_UPLOAD_SIZE_TO_USE_CRC64_HEADER - 1; + Mono source = Mono.deferContextual(ctx -> Mono.just(ctx.get(CONTENT_VALIDATION_MODE_KEY))); + Mono augmented = ContentValidationModeResolver.addContentValidationMode(source, + ContentValidationAlgorithm.AUTO, underThreshold, false); + StepVerifier.create(augmented).expectNext(USE_CRC64_CHECKSUM_HEADER_CONTEXT).verifyComplete(); + } + + @Test + public void addContentValidationModeMonoLeavesChainUnchangedWhenNoMode() { + Mono source = Mono.deferContextual(ctx -> Mono.just("ok")); + Mono augmented = ContentValidationModeResolver.addContentValidationMode(source, + ContentValidationAlgorithm.NONE, 1024, false); + StepVerifier.create(augmented).expectNext("ok").verifyComplete(); + } + + // =========================================================================================== + // validateTransactionalChecksumOptions (boolean computeMd5) + // =========================================================================================== + + @Test + public void validateComputeMd5PassesForCompatibleOptions() { + assertDoesNotThrow(() -> ContentValidationModeResolver.validateTransactionalChecksumOptions(true, null)); + assertDoesNotThrow(() -> ContentValidationModeResolver.validateTransactionalChecksumOptions(false, + ContentValidationAlgorithm.AUTO)); + assertDoesNotThrow(() -> ContentValidationModeResolver.validateTransactionalChecksumOptions(false, + ContentValidationAlgorithm.NONE)); + assertDoesNotThrow(() -> ContentValidationModeResolver.validateTransactionalChecksumOptions(false, + ContentValidationAlgorithm.CRC64)); + } + + @Test + public void validateComputeMd5ThrowsWhenAlgorithmNone() { + assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateTransactionalChecksumOptions(true, ContentValidationAlgorithm.NONE)); + } + + @Test + public void validateComputeMd5ThrowsForCrc64() { + assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateTransactionalChecksumOptions(true, ContentValidationAlgorithm.CRC64)); + } + + @Test + public void validateComputeMd5ThrowsForAuto() { + assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateTransactionalChecksumOptions(true, ContentValidationAlgorithm.AUTO)); + } + + // =========================================================================================== + // validateProgressWithContentValidation + // =========================================================================================== + + @Test + public void validateProgressWithContentValidationPassesWhenNoProgress() { + assertDoesNotThrow(() -> ContentValidationModeResolver + .validateProgressWithContentValidation((ProgressListener) null, ContentValidationAlgorithm.CRC64)); + assertDoesNotThrow(() -> ContentValidationModeResolver + .validateProgressWithContentValidation((ProgressListener) null, ContentValidationAlgorithm.AUTO)); + } + + @Test + public void validateProgressWithContentValidationPassesWhenNoneOrNullAlgorithm() { + ProgressListener listener = l -> { + }; + assertDoesNotThrow(() -> ContentValidationModeResolver.validateProgressWithContentValidation(listener, null)); + assertDoesNotThrow(() -> ContentValidationModeResolver.validateProgressWithContentValidation(listener, + ContentValidationAlgorithm.NONE)); + } + + @Test + public void validateProgressWithContentValidationThrowsForCrc64() { + ProgressListener listener = l -> { + }; + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateProgressWithContentValidation(listener, ContentValidationAlgorithm.CRC64)); + assertEquals(ContentValidationModeResolver.PROGRESS_CONFLICTS_TRANSFER_CONTENT_VALIDATION_MESSAGE, + ex.getMessage()); + } + + @Test + public void validateProgressWithContentValidationThrowsForAuto() { + ProgressListener listener = l -> { + }; + assertThrows(IllegalArgumentException.class, () -> ContentValidationModeResolver + .validateProgressWithContentValidation(listener, ContentValidationAlgorithm.AUTO)); + } + + @Test + public void isCrc64OrAutoReflectsCrc64AndAutoOnly() { + assertTrue(ContentValidationModeResolver.isCrc64OrAuto(ContentValidationAlgorithm.CRC64)); + assertTrue(ContentValidationModeResolver.isCrc64OrAuto(ContentValidationAlgorithm.AUTO)); + assertFalse(ContentValidationModeResolver.isCrc64OrAuto(ContentValidationAlgorithm.NONE)); + assertFalse(ContentValidationModeResolver.isCrc64OrAuto(null)); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/MessageEncoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/MessageEncoderTests.java new file mode 100644 index 000000000000..899a2b5f5425 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/MessageEncoderTests.java @@ -0,0 +1,330 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.FluxUtil; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.Disabled; +import reactor.core.publisher.Flux; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Stream; + +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.CRC64_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_DEFAULT_SEGMENT_CONTENT_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_HEADER_LENGTH; +import static com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class MessageEncoderTests { + + private static byte[] getRandomData(int size) { + byte[] result = new byte[size]; + ThreadLocalRandom.current().nextBytes(result); + return result; + } + + private static void writeSegment(int number, byte[] data, long dataCrc, ByteArrayOutputStream stream) + throws IOException { + writeSegment(number, data, stream); // Call the method without CRC + ByteBuffer segFooter = ByteBuffer.allocate(CRC64_LENGTH).order(ByteOrder.LITTLE_ENDIAN); + segFooter.putLong(dataCrc); + stream.write(segFooter.array()); // Write segment footer + } + + private static void writeSegment(int number, byte[] data, ByteArrayOutputStream stream) throws IOException { + ByteBuffer segHeader = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); // 2 + 8 + segHeader.putShort((short) number); + segHeader.putLong(data.length); + + stream.write(segHeader.array()); // Write segment header + stream.write(data); // Write segment content + } + + // TODO (isbr): Add tests with static inputs and expected outputs for the encoder. + // Avoid reimplementing the encoder in tests to prevent potential errors in both implementation and tests. + // Consider reusing outputs from existing scripts. This approach can also benefit future decoder tests. + + private static ByteBuffer buildStructuredMessage(ByteBuffer data, int segmentSize, + StructuredMessageFlags structuredMessageFlags) throws IOException { + int segmentCount = Math.max(1, (int) Math.ceil((double) data.capacity() / segmentSize)); + int segmentFooterLength = structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64 ? CRC64_LENGTH : 0; + + int messageLength = V1_HEADER_LENGTH + ((V1_SEGMENT_HEADER_LENGTH + segmentFooterLength) * segmentCount) + + data.capacity() + (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64 ? CRC64_LENGTH : 0); + + long messageCRC = 0; + + // Message Header + ByteBuffer buffer = ByteBuffer.allocate(13).order(ByteOrder.LITTLE_ENDIAN); //1 + 8 + 2 + 2 + buffer.put((byte) 0x01); + buffer.putLong(messageLength); + buffer.putShort((short) structuredMessageFlags.getValue()); + buffer.putShort((short) segmentCount); + + ByteArrayOutputStream message = new ByteArrayOutputStream(); + message.write(buffer.array()); + + if (data.capacity() == 0) { + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + writeSegment(1, data.array(), 0, message); + } else { + writeSegment(1, data.array(), message); + } + } else { + // Segments + int[] segmentSizes = new int[segmentCount]; + Arrays.fill(segmentSizes, segmentSize); + + int offset = 0; + for (int i = 1; i <= segmentCount; i++) { + int size = segmentSizes[i - 1]; + byte[] segmentData = customCopyOfRange(data, offset, size); + offset += size; + + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + long segmentCrc = StorageCrc64Calculator.compute(segmentData, 0); + writeSegment(i, segmentData, segmentCrc, message); + messageCRC = StorageCrc64Calculator.compute(segmentData, messageCRC); + } else { + writeSegment(i, segmentData, message); + } + } + } + + // Message footer + if (structuredMessageFlags == StructuredMessageFlags.STORAGE_CRC64) { + byte[] crcBytes + = ByteBuffer.allocate(CRC64_LENGTH).order(ByteOrder.LITTLE_ENDIAN).putLong(messageCRC).array(); + message.write(crcBytes); + } + + return ByteBuffer.wrap(message.toByteArray()); + } + + public static byte[] customCopyOfRange(ByteBuffer original, int from, int size) { + int end = Math.min(from + size, original.capacity()); + return Arrays.copyOfRange(original.array(), from, end); + } + + private static Stream readAllSupplier() { + return Stream.of(Arguments.of(10, 1, StructuredMessageFlags.NONE), + Arguments.of(10, 1, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 1024, StructuredMessageFlags.NONE), + Arguments.of(1024, 1024, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 512, StructuredMessageFlags.NONE), + Arguments.of(1024, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 200, StructuredMessageFlags.NONE), + Arguments.of(1024, 200, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 10, 2, StructuredMessageFlags.NONE), + Arguments.of(1024 * 10, 2, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 50, 512, StructuredMessageFlags.NONE), + Arguments.of(1024 * 50, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 1024, 512, StructuredMessageFlags.NONE), + Arguments.of(1024 * 1024, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 1024, 1024, StructuredMessageFlags.NONE), + Arguments.of(1024 * 1024, 1024, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 1024 * 4, 1024 * 1024, StructuredMessageFlags.NONE), + Arguments.of(1024 * 1024 * 4, 1024 * 1024, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024 * 1024 * 8, 1024 * 1024, StructuredMessageFlags.NONE), + Arguments.of(1024 * 1024 * 8, 1024 * 1024, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234, 123, StructuredMessageFlags.NONE), + Arguments.of(1234, 123, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234 * 10, 12, StructuredMessageFlags.NONE), + Arguments.of(1234 * 10, 12, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234 * 1234, 567, StructuredMessageFlags.NONE), + Arguments.of(1234 * 1234, 567, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234 * 1234 * 8, 1234 * 1234, StructuredMessageFlags.NONE), + Arguments.of(1234 * 1234 * 8, 1234 * 1234, StructuredMessageFlags.STORAGE_CRC64)); + } + + @ParameterizedTest + @MethodSource("readAllSupplier") + public void readAll(int size, int segmentSize, StructuredMessageFlags flags) throws IOException { + byte[] data = getRandomData(size); + + ByteBuffer unencodedBuffer = ByteBuffer.wrap(data); + + StructuredMessageEncoder structuredMessageEncoder = new StructuredMessageEncoder(size, segmentSize, flags); + + byte[] actual + = FluxUtil.collectBytesInByteBufferStream(structuredMessageEncoder.encode(unencodedBuffer)).block(); + byte[] expected = buildStructuredMessage(unencodedBuffer, segmentSize, flags).array(); + + assertArrayEquals(expected, actual); + } + + private static Stream readMultipleSupplier() { + return Stream.of(Arguments.of(30, StructuredMessageFlags.NONE), + Arguments.of(30, StructuredMessageFlags.STORAGE_CRC64), Arguments.of(15, StructuredMessageFlags.NONE), + Arguments.of(15, StructuredMessageFlags.STORAGE_CRC64), Arguments.of(11, StructuredMessageFlags.NONE), + Arguments.of(11, StructuredMessageFlags.STORAGE_CRC64), Arguments.of(8, StructuredMessageFlags.NONE), + Arguments.of(8, StructuredMessageFlags.STORAGE_CRC64)); + } + + @ParameterizedTest + @MethodSource("readMultipleSupplier") + public void readMultiple(int segmentSize, StructuredMessageFlags flags) throws IOException { + byte[] data1 = getRandomData(10); + byte[] data2 = getRandomData(10); + byte[] data3 = getRandomData(10); + + ByteBuffer wrappedData1 = ByteBuffer.wrap(data1); + ByteBuffer wrappedData2 = ByteBuffer.wrap(data2); + ByteBuffer wrappedData3 = ByteBuffer.wrap(data3); + + ByteBuffer allWrappedData = ByteBuffer.allocate(30); + allWrappedData.put(data1); + allWrappedData.put(data2); + allWrappedData.put(data3); + + StructuredMessageEncoder structuredMessageEncoder = new StructuredMessageEncoder(30, segmentSize, flags); + + byte[] expected = buildStructuredMessage(allWrappedData, segmentSize, flags).array(); + + Flux allActualFlux = structuredMessageEncoder.encode(wrappedData1) + .concatWith(structuredMessageEncoder.encode(wrappedData2)) + .concatWith(structuredMessageEncoder.encode(wrappedData3)); + + byte[] actual = FluxUtil.collectBytesInByteBufferStream(allActualFlux).block(); + + assertArrayEquals(expected, actual); + } + + @Test + public void emptyBuffer() { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(10, 5, StructuredMessageFlags.NONE); + ByteBuffer emptyBuffer = ByteBuffer.allocate(0); + byte[] result = FluxUtil.collectBytesInByteBufferStream(encoder.encode(emptyBuffer)).block(); + assertNotNull(result); + assertEquals(0, result.length); + } + + @Test + public void contentAlreadyEncoded() { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(4, 2, StructuredMessageFlags.NONE); + FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(new byte[] { 1, 2, 3, 4 }))).block(); + // After encoding is complete, further encode calls return empty (no error) to support retries and extra buffers. + byte[] result + = FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(new byte[] { 1, 2 }))).block(); + assertNotNull(result); + assertEquals(0, result.length); + } + + @Test + public void bufferLengthExceedsContentLength() { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(4, 2, StructuredMessageFlags.NONE); + FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(new byte[] { 1, 2, 3 }))).block(); + assertThrows(IllegalArgumentException.class, + () -> FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(new byte[] { 1, 2 }))) + .block()); + } + + @Test + public void segmentSizeLessThanOne() { + assertThrows(IllegalArgumentException.class, + () -> new StructuredMessageEncoder(10, 0, StructuredMessageFlags.NONE)); + } + + @Test + public void contentLengthLessThanOne() { + assertThrows(IllegalArgumentException.class, + () -> new StructuredMessageEncoder(0, 10, StructuredMessageFlags.NONE)); + } + + @Test + public void testNumSegmentsExceedsMaxValue() { + assertThrows(IllegalArgumentException.class, + () -> new StructuredMessageEncoder(Integer.MAX_VALUE, 1, StructuredMessageFlags.NONE)); + } + + // =========================================================================================== + // getEncodedMessageLength accuracy + // =========================================================================================== + + private static Stream encodedLengthSupplier() { + return Stream.of(Arguments.of(1, 1, StructuredMessageFlags.NONE), + Arguments.of(1, 1, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(100, 100, StructuredMessageFlags.NONE), + Arguments.of(100, 100, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1024, 512, StructuredMessageFlags.NONE), + Arguments.of(1024, 512, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(4 * 1024 * 1024, V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(10 * 1024 * 1024, V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64), + Arguments.of(1234, 123, StructuredMessageFlags.STORAGE_CRC64)); + } + + @ParameterizedTest + @MethodSource("encodedLengthSupplier") + public void encodedLengthMatchesActualOutput(int size, int segmentSize, StructuredMessageFlags flags) { + byte[] data = getRandomData(size); + StructuredMessageEncoder encoder = new StructuredMessageEncoder(size, segmentSize, flags); + long predictedLength = encoder.getEncodedMessageLength(); + + byte[] actual = FluxUtil.collectBytesInByteBufferStream(encoder.encode(ByteBuffer.wrap(data))).block(); + assertNotNull(actual); + assertEquals(predictedLength, actual.length, "getEncodedMessageLength() must match actual encoded output size"); + } + + // =========================================================================================== + // Direct (non-array-backed) ByteBuffer + // =========================================================================================== + + @ParameterizedTest + @MethodSource("readAllSupplier") + public void readAllDirectByteBuffer(int size, int segmentSize, StructuredMessageFlags flags) throws IOException { + byte[] data = getRandomData(size); + ByteBuffer directBuffer = ByteBuffer.allocateDirect(size); + directBuffer.put(data); + directBuffer.flip(); + + ByteBuffer arrayBuffer = ByteBuffer.wrap(data); + + StructuredMessageEncoder encoder = new StructuredMessageEncoder(size, segmentSize, flags); + byte[] actual = FluxUtil.collectBytesInByteBufferStream(encoder.encode(directBuffer)).block(); + byte[] expected = buildStructuredMessage(arrayBuffer, segmentSize, flags).array(); + + assertArrayEquals(expected, actual, "Direct ByteBuffer encoding must match array-backed encoding"); + } + + // =========================================================================================== + // Null buffer + // =========================================================================================== + + @Test + public void encodeNullBufferThrows() { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(10, 5, StructuredMessageFlags.NONE); + assertThrows(NullPointerException.class, + () -> FluxUtil.collectBytesInByteBufferStream(encoder.encode(null)).block()); + } + + @Test + @Disabled("For local testing only") + public void bigEncode() throws IOException { + byte[] data = getRandomData(262144000); + + ByteBuffer unencodedBuffer = ByteBuffer.wrap(data); + + StructuredMessageEncoder structuredMessageEncoder = new StructuredMessageEncoder(262144000, + V1_DEFAULT_SEGMENT_CONTENT_LENGTH, StructuredMessageFlags.STORAGE_CRC64); + + byte[] actual + = FluxUtil.collectBytesInByteBufferStream(structuredMessageEncoder.encode(unencodedBuffer)).block(); + byte[] expected = buildStructuredMessage(unencodedBuffer, V1_DEFAULT_SEGMENT_CONTENT_LENGTH, + StructuredMessageFlags.STORAGE_CRC64).array(); + assertArrayEquals(expected, actual); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java new file mode 100644 index 000000000000..a07cef1f2013 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StorageCrc64CalculatorTests.java @@ -0,0 +1,222 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; + +import com.azure.storage.common.implementation.Constants; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.Random; +import java.util.stream.IntStream; +import java.math.BigInteger; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class StorageCrc64CalculatorTests { + + @ParameterizedTest + @MethodSource("testComputeSupplier") + public void testCompute(String data, long initial, long expected) { + byte[] bytes = data.getBytes(); + long actual = StorageCrc64Calculator.compute(bytes, initial); + assertEquals(expected, actual); + } + + private static Stream testComputeSupplier() { + return Stream.of(Arguments.of("", 0, 0), Arguments.of("Hello World!", 0, 208604604655264165L), + Arguments.of("123456789!@#$%^&*()", 0, 2153758901452455624L), + Arguments.of( + "This is a test where the data is longer than 64 characters so that we can test that code path.", 0, + 2736107658526394369L)); + } + + @ParameterizedTest + @MethodSource("testComputeWithBinaryDataSupplier") + void testComputeWithBinaryData(long initial, String hexData, String expectedStr) { + byte[] hexBytes = hexStringToByteArray(hexData); + long expected = unsignedLongFromString(expectedStr); + long actual = StorageCrc64Calculator.compute(hexBytes, initial); + assertEquals(expected, actual); + } + + private static Stream testComputeWithBinaryDataSupplier() { + return Stream.of(Arguments.of(0L, "C8E11B40D793D1526018", "3386042136331673945"), + Arguments.of(208604604655264165L, "C8E11B40D793D1526018", "4570059697646401418"), + Arguments.of(2153758901452455624L, "C8E11B40D793D1526018", "13366433516720813220"), + Arguments.of(12345L, "C8E11B40D793D1526018", "5139183895903464380"), + Arguments.of(0L, "AABBCCDDEEFF", "13410969003359324163"), + Arguments.of(208604604655264165L, "AABBCCDDEEFF", "7077292804802198544"), + Arguments.of(2153758901452455624L, "AABBCCDDEEFF", "7845171950889742163"), + Arguments.of(12345L, "AABBCCDDEEFF", "1250888331951523569")); + } + + private static byte[] hexStringToByteArray(String hexData) { + int length = hexData.length(); + byte[] data = new byte[length / 2]; + for (int i = 0; i < length; i += 2) { + data[i / 2] + = (byte) ((Character.digit(hexData.charAt(i), 16) << 4) + Character.digit(hexData.charAt(i + 1), 16)); + } + return data; + } + + @ParameterizedTest + @MethodSource("testComposeSupplier") + void testCompose(int numSegments, int minBlockSize, int maxBlockSize) { + Random random = new Random(); + + List blockLengths = new ArrayList<>(); + IntStream.range(0, numSegments).forEach(i -> { + int blockSize = maxBlockSize > minBlockSize + ? random.nextInt(maxBlockSize - minBlockSize) + minBlockSize + : minBlockSize; + blockLengths.add(blockSize); + }); + + byte[] data = new byte[blockLengths.stream().mapToInt(Integer::intValue).sum()]; + random.nextBytes(data); + + long wholeCrc = StorageCrc64Calculator.compute(data, 0); + Queue blockCrcs = new LinkedList<>(); + int offset = 0; + for (int length : blockLengths) { + blockCrcs.add(StorageCrc64Calculator.compute(Arrays.copyOfRange(data, offset, offset + length), 0)); + offset += length; + } + + long composedCrc = blockCrcs.poll(); + int lengthIndex = 1; + while (!blockCrcs.isEmpty()) { + long nextBlockCrc = blockCrcs.poll(); + composedCrc = StorageCrc64Calculator.concat(0, 0, composedCrc, blockLengths.get(lengthIndex - 1), 0, + nextBlockCrc, blockLengths.get(lengthIndex)); + lengthIndex++; + } + + assertEquals(wholeCrc, composedCrc); + } + + private static Stream testComposeSupplier() { + return Stream.of(Arguments.of(2, Constants.KB, Constants.KB), Arguments.of(3, Constants.KB, Constants.KB), + Arguments.of(10, Constants.KB, Constants.KB), Arguments.of(2, Constants.KB, 4 * Constants.KB), + Arguments.of(3, Constants.KB, 4 * Constants.KB), Arguments.of(10, Constants.KB, 4 * Constants.KB), + Arguments.of(2, Constants.KB, Constants.MB), Arguments.of(3, Constants.KB, Constants.MB), + Arguments.of(2, Constants.KB, 512 * Constants.MB), Arguments.of(3, Constants.KB, 512 * Constants.MB)); + } + + @Test + void testComputeInChunks() { + int maxBlockSize = Constants.GB; + int chunkSize = 8 * Constants.KB; + String baseString = "Hello World! This is testing chunked crc."; + byte[] baseBytes = baseString.getBytes(); + int baseLength = baseBytes.length; + + long wholeCrc = 8100535992282268188L; + + long chunkedCrc = 0; + int totalLength = 0; + + while (totalLength < maxBlockSize) { + int length = Math.min(chunkSize, maxBlockSize - totalLength); + byte[] chunk = new byte[length]; + for (int i = 0; i < length; i += baseLength) { + System.arraycopy(baseBytes, 0, chunk, i, Math.min(baseLength, length - i)); + } + long chunkCrc = StorageCrc64Calculator.compute(chunk, 0); + chunkedCrc = StorageCrc64Calculator.concat(chunkedCrc, 0, chunkCrc, length, 0, chunkCrc, length); + totalLength += length; + } + + assertEquals(wholeCrc, chunkedCrc); + } + + @ParameterizedTest + @CsvSource({ + "0, 0, 0, 0, 0", + "17360427831495520774, 949533, 13068224794440996385, 99043, 2942932174470096852", + "11788770130477425887, 505156, 11825964890373840515, 543420, 11679439596881108042", + "3295333047304801182, 732633, 15304759627474960884, 315943, 31840984168952505", + "16590039424904606984, 550299, 6063316096266934453, 498277, 4430932446378441680", + "4505532069077416052, 852279, 4910763717047934640, 196297, 15119506491662968913", + "390777554329396866, 834642, 2639871931800812330, 213934, 11705441749781302341", + "3373070654000205532, 282713, 998330282635826126, 765863, 9830625244855600085", + "2124306943908111903, 737293, 11017351202683543503, 311283, 8163771928713973931", + "15994440356403990005, 85861, 13803536430055425947, 962715, 1941624554903785510", + "3705122932895036835, 444701, 17573219681510991482, 603875, 4421337306620606216" }) + void testConcat(String crc1Str, long size1, String crc2Str, long size2, String expectedStr) { + // Convert the large numbers from String to BigInteger for unsigned support + long crc1 = unsignedLongFromString(crc1Str); + long crc2 = unsignedLongFromString(crc2Str); + long expected = unsignedLongFromString(expectedStr); + + long actual = StorageCrc64Calculator.concat(0, 0, crc1, size1, 0, crc2, size2); + assertEquals(expected, actual); + } + + private long unsignedLongFromString(String value) { + // Convert a string to a signed long interpreting it as unsigned 64-bit integer + return new BigInteger(value).longValue(); + } + + @ParameterizedTest + @MethodSource("testConcatWithInitialsSupplier") + void testConcatWithInitials(String initialStr, String initial1Str, String crc1Str, String size1Str, + String initial2Str, String crc2Str, String size2Str, String expectedStr) { + // Convert large unsigned values to signed long + long initial = unsignedLongFromString(initialStr); + long initial1 = unsignedLongFromString(initial1Str); + long crc1 = unsignedLongFromString(crc1Str); + long size1 = Long.parseLong(size1Str); + long initial2 = unsignedLongFromString(initial2Str); + long crc2 = unsignedLongFromString(crc2Str); + long size2 = Long.parseLong(size2Str); + long expected = unsignedLongFromString(expectedStr); + + long actual = StorageCrc64Calculator.concat(initial, initial1, crc1, size1, initial2, crc2, size2); + assertEquals(expected, actual); + } + + private static Stream testConcatWithInitialsSupplier() { + return Stream.of(Arguments.of("0", "0", "0", "0", "0", "0", "0", "0"), + Arguments.of("556425425686929588", "346224202686926702", "16342296696377857982", "332915", + "1153230192133190692", "12153371329672466699", "715661", "1441822370130745021"), + Arguments.of("6707243468313456313", "572263087298867634", "16994544883182326144", "75745", + "9131338361339398429", "10182915179976307502", "972831", "14966971284513070994"), + Arguments.of("5753644013440131291", "5049702011265556767", "17549647897932809624", "255140", + "7204171574261853450", "1993084328138883374", "793436", "6041621697050742380"), + Arguments.of("6856094926385348025", "5380840211500611709", "9696539459657763690", "537777", + "4787042077805010903", "13660128687379374948", "510799", "17784586126519415898"), + Arguments.of("7768574238870932405", "97145001356670685", "607054043350981298", "706788", + "667444555190985522", "10677778047180339455", "341788", "5763961866513573791"), + Arguments.of("3302120679354661969", "7763531798276712053", "8827557196489825944", "490442", + "8582969104890206846", "6702182603435500761", "558134", "4787302867829109706"), + Arguments.of("7553023568245626261", "9093436341919279996", "10815569438302788871", "785480", + "8305342016037017917", "6633140726058569127", "263096", "14625483825524673467"), + Arguments.of("8894905328920043035", "9101951045389247372", "10098427678135105249", "782758", + "8101576936188464286", "8318237935995533450", "265818", "5983082645903611588"), + Arguments.of("1084935736425738155", "5378644106529179816", "13762475631325587388", "1014816", + "8473418370223760471", "10401355811619715622", "33760", "3692020299859515126"), + Arguments.of("889000539881195835", "2971048229276949174", "5346315327374690144", "307387", + "1407121768110541356", "10535852615249992663", "741189", "3634018251978804152")); + } + + @Test + void testComputeSliceMatchesFullArray() { + byte[] data = "Hello World!".getBytes(); + long expected = StorageCrc64Calculator.compute(data, 0); + long actual = StorageCrc64Calculator.compute(data, 0, data.length, 0); + assertEquals(expected, actual); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java new file mode 100644 index 000000000000..f9f460bb57fc --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageDecoderTests.java @@ -0,0 +1,631 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import com.azure.core.util.FluxUtil; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import reactor.core.publisher.Flux; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for StructuredMessageDecoder. + */ +public class StructuredMessageDecoderTests { + private static final int MESSAGE_HEADER_LENGTH = StructuredMessageConstants.V1_HEADER_LENGTH; + private static final int SEGMENT_HEADER_LENGTH = StructuredMessageConstants.V1_SEGMENT_HEADER_LENGTH; + private static final int CRC64_LENGTH = 8; + + @Test + public void readsCompleteMessageInSingleChunk() { + byte[] originalData = getRandomData(1024); + ByteBuffer encodedData = encodeToByteBuffer(originalData, 512, StructuredMessageFlags.STORAGE_CRC64); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List decodedPayload = decoder.decodeChunk(encodedData); + + assertTrue(decoder.isComplete()); + assertFalse(decodedPayload.isEmpty()); + assertArrayEquals(originalData, collectDecodedBytes(decodedPayload)); + } + + @Test + public void readsTopLevelMessageHeaderSplitAcrossChunks() { + byte[] originalData = getRandomData(256); + byte[] encodedBytes = encodeToBytes(originalData, 128); + int encodedLength = encodedBytes.length; + + // Split before the 13-byte message header is complete. + int messageHeaderSplitPoint = 7; + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, messageHeaderSplitPoint); + ByteBuffer chunk2 + = ByteBuffer.wrap(encodedBytes, messageHeaderSplitPoint, encodedLength - messageHeaderSplitPoint); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List firstDecodedPayload = decoder.decodeChunk(chunk1); + assertTrue(firstDecodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + + List secondDecodedPayload = decoder.decodeChunk(chunk2); + assertFalse(secondDecodedPayload.isEmpty()); + assertTrue(decoder.isComplete()); + } + + @Test + public void readsPerSegmentHeaderSplitAcrossChunks() { + byte[] originalData = getRandomData(512); + byte[] encodedBytes = encodeToBytes(originalData, 256); + int encodedLength = encodedBytes.length; + + // Split after the full message header but before the 10-byte segment header is complete. + int segmentHeaderSplitPoint = MESSAGE_HEADER_LENGTH + 5; + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, segmentHeaderSplitPoint); + ByteBuffer chunk2 + = ByteBuffer.wrap(encodedBytes, segmentHeaderSplitPoint, encodedLength - segmentHeaderSplitPoint); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List firstDecodedPayload = decoder.decodeChunk(chunk1); + // Segment header is incomplete, so nothing is emitted yet. + assertTrue(firstDecodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + + List secondDecodedPayload = decoder.decodeChunk(chunk2); + assertFalse(secondDecodedPayload.isEmpty()); + assertTrue(decoder.isComplete()); + } + + @Test + public void multipleChunksDecode() { + byte[] originalData = getRandomData(256); + byte[] encodedBytes = encodeToBytes(originalData, 128); + int encodedLength = encodedBytes.length; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + + int chunkSize = 32; + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + for (int offset = 0; offset < encodedLength; offset += chunkSize) { + int len = Math.min(chunkSize, encodedLength - offset); + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, offset, len); + + List decodedPayload = decoder.decodeChunk(chunk); + writeDecodedPayload(output, decodedPayload); + if (decoder.isComplete()) { + break; + } + } + + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, output.toByteArray()); + } + + @Test + public void decodeWithNoCrc() { + byte[] originalData = getRandomData(256); + ByteBuffer encodedData = encodeToByteBuffer(originalData, 128, StructuredMessageFlags.NONE); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List decodedPayload = decoder.decodeChunk(encodedData); + + assertTrue(decoder.isComplete()); + assertFalse(decodedPayload.isEmpty()); + assertArrayEquals(originalData, collectDecodedBytes(decodedPayload)); + } + + @Test + public void handlesZeroLengthBuffer() { + byte[] originalData = getRandomData(256); + byte[] encodedBytes = encodeToBytes(originalData, 128); + int encodedLength = encodedBytes.length; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + ByteBuffer emptyBuffer = ByteBuffer.allocate(0); + List firstDecodedPayload = decoder.decodeChunk(emptyBuffer); + assertTrue(firstDecodedPayload.isEmpty()); + ByteBuffer dataBuffer = ByteBuffer.wrap(encodedBytes); + List decodedPayload = decoder.decodeChunk(dataBuffer); + assertFalse(decodedPayload.isEmpty()); + assertTrue(decoder.isComplete()); + } + + /** + * Payload bytes for a segment must not be emitted until the segment's CRC footer has been read and + * validated. While the footer is incomplete, decodeChunk must return null. + */ + @Test + public void withholdsPayloadUntilSegmentFooterValidated() { + byte[] originalData = getRandomData(1024); + byte[] encodedBytes = encodeToBytes(originalData, 1024); + int encodedLength = encodedBytes.length; + + // The segment payload cannot be emitted until the final segment CRC byte arrives in chunk2. + int segCrcAllButLast + = MESSAGE_HEADER_LENGTH + SEGMENT_HEADER_LENGTH + 1024 + StructuredMessageConstants.CRC64_LENGTH - 1; + ByteBuffer chunk1 = ByteBuffer.wrap(encodedBytes, 0, segCrcAllButLast); + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + List firstDecodedPayload = decoder.decodeChunk(chunk1); + assertTrue(firstDecodedPayload.isEmpty(), "Decoder must not emit payload before segment CRC is validated"); + assertFalse(decoder.isComplete()); + + ByteBuffer chunk2 = ByteBuffer.wrap(encodedBytes, segCrcAllButLast, encodedLength - segCrcAllButLast); + List emittedPayload = decoder.decodeChunk(chunk2); + assertFalse(emittedPayload.isEmpty()); + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, collectDecodedBytes(emittedPayload)); + } + + @Test + public void throwsOnUnsupportedStructuredMessageVersion() { + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + // Byte 0 of the message header is reserved for the version. Changing to an incompatible version (2) + encodedBytes[0] = (byte) (StructuredMessageConstants.DEFAULT_MESSAGE_VERSION + 1); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Unsupported structured message version")); + } + + @Test + public void throwsOnMessageLengthMismatch() { + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Construct decoder with wrong expected encoded length. + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length + 1); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("did not match content length")); + } + + @Test + public void throwsOnUnexpectedSegmentNumber() { + byte[] data = getRandomData(300); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Corrupt first segment number from 1 to 2 (offset 13). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(MESSAGE_HEADER_LENGTH, (short) 2); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Unexpected segment number")); + } + + @Test + public void throwsOnInvalidSegmentSize() { + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Corrupt first segment size to an impossible value (8 bytes at offset 15). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(MESSAGE_HEADER_LENGTH + 2, Long.MAX_VALUE); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Invalid segment size detected")); + } + + @Test + public void throwsOnSegmentCrcMismatch() { + byte[] data = getRandomData(512); + byte[] encodedBytes = encodeToBytes(data, 512); + + // msgHdr(13) + segHdr(10) + payload(512) + segCrc(8) + msgCrc(8). Flip one bit of the segment CRC. + int segmentCrcOffset = MESSAGE_HEADER_LENGTH + SEGMENT_HEADER_LENGTH + data.length; + encodedBytes[segmentCrcOffset] ^= 0x01; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("CRC64 mismatch in segment")); + } + + @Test + public void throwsOnSegmentCrcMismatchInLaterSegment() { + // Multi-segment message where segment 1 is intact but segment 2's CRC is corrupted; verifies + // CRC validation runs on every segment, not just the first. + byte[] data = getRandomData(300); + byte[] encodedBytes = encodeToBytes(data, 100); + + // Per-segment block: segHdr(10) + payload(100) + segCrc(8) = 118. + // Segment 2's CRC starts at: msgHdr(13) + segBlock(118) + segHdr(10) + payload(100). + int seg2CrcOffset + = MESSAGE_HEADER_LENGTH + (SEGMENT_HEADER_LENGTH + 100 + CRC64_LENGTH) + SEGMENT_HEADER_LENGTH + 100; + encodedBytes[seg2CrcOffset] ^= 0x01; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("CRC64 mismatch in segment 2")); + } + + @Test + public void throwsOnMessageCrcMismatch() { + byte[] data = getRandomData(512); + byte[] encodedBytes = encodeToBytes(data, 512); + + int messageCrcOffset = encodedBytes.length - CRC64_LENGTH; + byte[] corrupted = Arrays.copyOf(encodedBytes, encodedBytes.length); + corrupted[messageCrcOffset] ^= 0x01; + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(corrupted.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(corrupted))); + assertTrue(exception.getMessage().contains("CRC64 mismatch in message footer")); + } + + @Test + public void throwsOnUnsupportedFlags() { + // Flags value 2 is not in the StructuredMessageFlags enum (NONE=0, STORAGE_CRC64=1) and must be rejected. + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + // Flags live at offset 9 (2 bytes, little-endian). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(9, (short) 2); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Invalid value for StructuredMessageFlags")); + } + + @Test + public void throwsOnZeroSegments() { + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + // numSegments lives at offset 11 (2 bytes, little-endian). Force it to zero. + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(11, (short) 0); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("at least one segment")); + } + + @Test + public void throwsOnSkippedSegmentNumber() { + // 3 segments of 100 bytes each. Layout per segment: segHdr(10) + payload(100) + segCrc(8) = 118. + // Rewrite segment 2's number to 3 to simulate a stream that skips segment 2. + byte[] data = getRandomData(300); + byte[] encodedBytes = encodeToBytes(data, 100); + + int seg2NumberOffset = MESSAGE_HEADER_LENGTH + (SEGMENT_HEADER_LENGTH + 100 + CRC64_LENGTH); + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putShort(seg2NumberOffset, (short) 3); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Unexpected segment number")); + } + + @Test + public void truncatedMessageHeaderLeavesDecoderIncomplete() { + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + // Feed only the first 5 bytes of the 13-byte message header. + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, 0, 5); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(chunk); + assertTrue(decodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + } + + @Test + public void truncatedSegmentHeaderLeavesDecoderIncomplete() { + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 256); + + // Feed full message header + 5 of the 10 segment-header bytes. + int truncated = MESSAGE_HEADER_LENGTH + 5; + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, 0, truncated); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(chunk); + assertTrue(decodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + } + + @Test + public void truncatedSegmentFooterLeavesDecoderIncomplete() { + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Layout: msgHdr(13) + segHdr(10) + payload(128) + segCrc(8) + msgCrc(8). Truncate mid-segCrc. + int truncated = MESSAGE_HEADER_LENGTH + SEGMENT_HEADER_LENGTH + 128 + 4; + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, 0, truncated); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(chunk); + assertTrue(decodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + } + + @Test + public void truncatedMessageFooterLeavesDecoderIncomplete() { + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Feed everything except the last 4 bytes of the 8-byte message CRC footer. + int truncated = encodedBytes.length - 4; + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, 0, truncated); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(chunk); + // Segment payload has been released, but the message footer is still incomplete. + assertFalse(decodedPayload.isEmpty()); + assertFalse(decoder.isComplete()); + } + + @Test + public void extraBytesAfterMessageFooterAreNotConsumed() { + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + // Append garbage bytes after the message footer; the decoder must stop at the declared message length. + int extras = 16; + byte[] padded = new byte[encodedBytes.length + extras]; + System.arraycopy(encodedBytes, 0, padded, 0, encodedBytes.length); + byte[] noise = getRandomData(extras); + System.arraycopy(noise, 0, padded, encodedBytes.length, extras); + ByteBuffer buffer = ByteBuffer.wrap(padded); + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + List decodedPayload = decoder.decodeChunk(buffer); + + assertTrue(decoder.isComplete()); + assertFalse(decodedPayload.isEmpty()); + assertArrayEquals(data, collectDecodedBytes(decodedPayload)); + // Trailing bytes must not be consumed; buffer position stops at the declared message length. + assertEquals(extras, buffer.remaining()); + } + + @Test + public void throwsOnEncodedPayloadLargerThanExpectedSize() { + // Wire payload is larger than the expectedEncodedMessageLength supplied to the decoder. Must be rejected by the msgLen check. + byte[] data = getRandomData(128); + byte[] encodedBytes = encodeToBytes(data, 128); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length - 8); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("did not match content length")); + } + + @Test + public void throwsOnNegativeMessageLength() { + // msgLen lives at offset 1 (8 bytes, little-endian). A negative value must be rejected before + // any further bounds math runs. + byte[] data = getRandomData(64); + byte[] encodedBytes = encodeToBytes(data, 64); + + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(1, Long.MIN_VALUE); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Message length too small")); + } + + @Test + public void throwsOnNegativeSegmentSize() { + // Companion to throwsOnInvalidSegmentSize covering the negative-value branch of the segment-size check. + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 256); + + // Segment size lives at offset MESSAGE_HEADER_LENGTH + 2 (after the 2-byte segment number). + ByteBuffer.wrap(encodedBytes).order(ByteOrder.LITTLE_ENDIAN).putLong(MESSAGE_HEADER_LENGTH + 2, Long.MIN_VALUE); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + IllegalArgumentException exception + = assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(encodedBytes))); + assertTrue(exception.getMessage().contains("Invalid segment size detected")); + } + + @Test + public void throwsOnInjectedRandomByte() { + // Insert a single random byte at a random offset in the encoded wire bytes. The msgLen field still + // declares the original size, so any insertion must be rejected by validation. + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 128); + + int insertAt = ThreadLocalRandom.current().nextInt(encodedBytes.length + 1); + byte injected = (byte) ThreadLocalRandom.current().nextInt(256); + + byte[] tampered = new byte[encodedBytes.length + 1]; + System.arraycopy(encodedBytes, 0, tampered, 0, insertAt); + tampered[insertAt] = injected; + System.arraycopy(encodedBytes, insertAt, tampered, insertAt + 1, encodedBytes.length - insertAt); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(tampered.length); + assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(tampered))); + } + + @Test + public void throwsOnRemovedRandomBytes() { + // Remove a random run of bytes from a random offset in the encoded wire. The msgLen field still + // declares the original size, so any deletion must be rejected by validation. + byte[] data = getRandomData(256); + byte[] encodedBytes = encodeToBytes(data, 128); + + int removeCount = 1 + ThreadLocalRandom.current().nextInt(8); + int removeAt = ThreadLocalRandom.current().nextInt(encodedBytes.length - removeCount); + + byte[] tampered = new byte[encodedBytes.length - removeCount]; + System.arraycopy(encodedBytes, 0, tampered, 0, removeAt); + System.arraycopy(encodedBytes, removeAt + removeCount, tampered, removeAt, + encodedBytes.length - removeAt - removeCount); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(tampered.length); + assertThrows(IllegalArgumentException.class, () -> decoder.decodeChunk(ByteBuffer.wrap(tampered))); + } + + /** + * Multi-segment round-trip with CRC enabled. Exercises the per-segment CRC validation followed by the + * message-wide CRC concat fold: if the concat math is wrong the trailing message footer check would fail + * even though every individual segment CRC matched, so this test directly guards the concat optimization. + */ + @Test + public void multipleSegmentsRoundTripWithCrc() { + // 16 segments of 1 KiB each. Small enough to feed in one chunk; large enough that there is meaningful + // CRC accumulation across segments. + int segmentSize = 1024; + int numSegments = 16; + byte[] originalData = getRandomData(segmentSize * numSegments); + ByteBuffer encoded = encodeToByteBuffer(originalData, segmentSize, StructuredMessageFlags.STORAGE_CRC64); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encoded.remaining()); + List decodedPayload = decoder.decodeChunk(encoded); + + assertTrue(decoder.isComplete()); + assertFalse(decodedPayload.isEmpty()); + assertArrayEquals(originalData, collectDecodedBytes(decodedPayload)); + } + + /** + * Multi-segment round-trip with CRC where the decoder is fed many small chunks rather than the whole encoded + * blob at once. This ensures the per-segment CRC computation is correct when payload bytes for a single + * segment arrive split across many decodeChunk calls (the typical production wire pattern) and that the + * O(1) concat fold at each segment boundary still produces a matching message CRC at the end. + */ + @Test + public void multipleSegmentsRoundTripWithCrcAcrossManyChunks() { + int segmentSize = 4 * 1024; + int numSegments = 8; + byte[] originalData = getRandomData(segmentSize * numSegments); + byte[] encodedBytes = encodeToBytes(originalData, segmentSize); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedBytes.length); + ByteArrayOutputStream collected = new ByteArrayOutputStream(); + int chunkSize = 137; // deliberately non-power-of-two and smaller than the segment so footers split + + for (int offset = 0; offset < encodedBytes.length; offset += chunkSize) { + int len = Math.min(chunkSize, encodedBytes.length - offset); + ByteBuffer chunk = ByteBuffer.wrap(encodedBytes, offset, len); + writeDecodedPayload(collected, decoder.decodeChunk(chunk)); + } + + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, collected.toByteArray()); + } + + /** + * Verifies that emitted payload buffers are exact-sized for the copied payload they contain and do not expose + * an oversized backing array. In this single-segment scenario the decoder emits one buffer whose backing + * array length matches the payload length, guarding against accidental reintroduction of intermediate + * buffering (for example, a growing {@code ByteArrayOutputStream} that hands off a larger-than-needed + * buffer). + */ + @Test + public void singleSegmentEmissionIsExactSized() { + int segmentSize = 2 * 1024; + byte[] originalData = getRandomData(segmentSize); + ByteBuffer encoded = encodeToByteBuffer(originalData, segmentSize, StructuredMessageFlags.STORAGE_CRC64); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encoded.remaining()); + List decoded = decoder.decodeChunk(encoded); + + assertEquals(1, decoded.size()); + assertTrue(decoder.isComplete()); + ByteBuffer payload = decoded.get(0); + assertTrue(payload.hasArray()); + assertEquals(segmentSize, payload.remaining()); + assertEquals(segmentSize, payload.array().length); + assertArrayEquals(originalData, collectDecodedBytes(decoded)); + } + + /** + * Multi-megabyte segment size exercises per-segment length from the wire header, + * not a fixed 4 MiB assumption. + */ + @ParameterizedTest + @MethodSource("segmentPayloadSizeAndTotalPayloadSizeSupplier") + public void decodesMultiMegabyteSegments(int segmentPayloadSize, int totalPayloadSize) { + byte[] originalData = getRandomData(totalPayloadSize); + ByteBuffer encodedData = encodeToByteBuffer(originalData, segmentPayloadSize, StructuredMessageFlags.NONE); + int encodedLength = encodedData.remaining(); + + StructuredMessageDecoder decoder = new StructuredMessageDecoder(encodedLength); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + writeDecodedPayload(output, decoder.decodeChunk(encodedData)); + + assertTrue(decoder.isComplete()); + assertArrayEquals(originalData, output.toByteArray()); + } + + private static Stream segmentPayloadSizeAndTotalPayloadSizeSupplier() { + return Stream.of(Arguments.of(10 * 1024 * 1024, 10 * 1024 * 1024 + 1), + Arguments.of(3 * 1024 * 1024, 3 * 1024 * 1024 + 1), Arguments.of(5 * 1024 * 1024 + 1, 15 * 1024 * 1024)); + } + + // For tests that pass the whole encoded message to decodeChunk. + private static ByteBuffer encodeToByteBuffer(byte[] originalData, int segmentLength, StructuredMessageFlags flags) { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, segmentLength, flags); + Flux flux = encoder.encode(ByteBuffer.wrap(originalData)); + + return ByteBuffer.wrap(Objects.requireNonNull(FluxUtil.collectBytesInByteBufferStream(flux).block())); + } + + // For tests that need random access/mutation/splitting of encoded bytes. + private static byte[] encodeToBytes(byte[] originalData, int segmentLength) { + ByteBuffer encoded = encodeToByteBuffer(originalData, segmentLength, StructuredMessageFlags.STORAGE_CRC64); + byte[] encodedBytes = new byte[encoded.remaining()]; + encoded.get(encodedBytes); + return encodedBytes; + } + + private static byte[] getRandomData(int size) { + byte[] result = new byte[size]; + ThreadLocalRandom.current().nextBytes(result); + return result; + } + + private static byte[] collectDecodedBytes(List decoded) { + if (decoded.isEmpty()) { + return null; + } + ByteArrayOutputStream output = new ByteArrayOutputStream(); + for (ByteBuffer buffer : decoded) { + if (buffer != null && buffer.hasRemaining()) { + byte[] decodedBytes = new byte[buffer.remaining()]; + buffer.get(decodedBytes); + output.write(decodedBytes, 0, decodedBytes.length); + } + } + return output.toByteArray(); + } + + private static void writeDecodedPayload(ByteArrayOutputStream output, List decoded) { + byte[] bytes = collectDecodedBytes(decoded); + if (bytes != null) { + output.write(bytes, 0, bytes.length); + } + } + +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlagsTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlagsTests.java new file mode 100644 index 000000000000..61da6d3b2b4c --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/implementation/contentvalidation/StructuredMessageFlagsTests.java @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.implementation.contentvalidation; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class StructuredMessageFlagsTests { + @Test + public void testNoneFlag() { + StructuredMessageFlags flag = StructuredMessageFlags.NONE; + assertEquals(0, flag.getValue()); + } + + @Test + public void testStorageCrc64Flag() { + StructuredMessageFlags flag = StructuredMessageFlags.STORAGE_CRC64; + assertEquals(1, flag.getValue()); + } + + @Test + public void testFromStringValid() { + assertEquals(StructuredMessageFlags.NONE, StructuredMessageFlags.fromString("0")); + assertEquals(StructuredMessageFlags.STORAGE_CRC64, StructuredMessageFlags.fromString("1")); + } + + @Test + public void testFromStringInvalid() { + assertNull(StructuredMessageFlags.fromString("2")); + assertNull(StructuredMessageFlags.fromString(null)); + assertThrows(NumberFormatException.class, () -> { + StructuredMessageFlags.fromString("invalid"); + }); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/DecodedResponseTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/DecodedResponseTests.java new file mode 100644 index 000000000000..e99c169a1763 --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/DecodedResponseTests.java @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.util.BinaryData; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +public class DecodedResponseTests { + + private static final HttpHeaderName CUSTOM_HEADER = HttpHeaderName.fromString("x-ms-custom"); + + private static HttpRequest newRequest() { + try { + return new HttpRequest(HttpMethod.GET, new URL("http://example.com/")); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + private static HttpHeaders headers(HttpHeaderName name, String value) { + return new HttpHeaders().set(name, value); + } + + private static MockHttpResponse mockResponse(int status, HttpHeaders headers, byte[] body) { + return new MockHttpResponse(newRequest(), status, headers, body); + } + + private static byte[] bytes(String s) { + return s.getBytes(StandardCharsets.UTF_8); + } + + private static Flux fluxOf(byte[] data) { + return Flux.just(ByteBuffer.wrap(data)); + } + + @Test + public void preservesRequestStatusCodeAndHeaders() { + HttpHeaders h = new HttpHeaders().set(HttpHeaderName.CONTENT_LENGTH, "100").set(CUSTOM_HEADER, "value"); + MockHttpResponse original = mockResponse(206, h, bytes("encoded")); + + DecodedResponse wrapper = new DecodedResponse(original, fluxOf(bytes("decoded")), 80L); + + assertSame(original.getRequest(), wrapper.getRequest()); + assertEquals(206, wrapper.getStatusCode()); + // Content-Length is overridden to decoded size; other headers are preserved. + assertEquals("80", wrapper.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + assertEquals("value", wrapper.getHeaders().getValue(CUSTOM_HEADER)); + } + + @Test + public void getHeaderValueByStringReturnsHeaderValue() { + HttpHeaders h = headers(CUSTOM_HEADER, "value"); + DecodedResponse wrapper = new DecodedResponse(mockResponse(200, h, new byte[0]), fluxOf(new byte[0]), 0L); + + assertEquals("value", wrapper.getHeaderValue(CUSTOM_HEADER.getCaseInsensitiveName())); + assertNull(wrapper.getHeaderValue("nonexistent")); + } + + @Test + public void getBodyReturnsDecodedFlux() { + byte[] decoded = bytes("decoded body"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + StepVerifier.create(wrapper.getBody().reduce(new ByteArrayOutputStream(), (sink, buf) -> { + byte[] copy = new byte[buf.remaining()]; + buf.get(copy); + sink.write(copy, 0, copy.length); + return sink; + })).expectNextMatches(sink -> Arrays.equals(decoded, sink.toByteArray())).verifyComplete(); + } + + @Test + public void getBodyAsByteArrayReturnsDecodedBytes() { + byte[] decoded = bytes("decoded body"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + StepVerifier.create(wrapper.getBodyAsByteArray()) + .expectNextMatches(b -> Arrays.equals(decoded, b)) + .verifyComplete(); + } + + @Test + public void getBodyAsStringDefaultsToUtf8WhenNoCharsetSpecified() { + // The no-arg overload routes through CoreUtils.bomAwareToString, which falls back to UTF-8 when neither a + // BOM nor a Content-Type charset parameter is present. This test pins the "no headers, no BOM" path. + String text = "héllo wörld – ✓"; + DecodedResponse wrapper = new DecodedResponse(mockResponse(200, new HttpHeaders(), new byte[0]), + fluxOf(text.getBytes(StandardCharsets.UTF_8)), 0L); + + StepVerifier.create(wrapper.getBodyAsString()).expectNext(text).verifyComplete(); + } + + @Test + public void getBodyAsStringHonorsCharsetFromContentTypeHeader() { + // Per the base HttpResponse contract, the no-arg getBodyAsString() must honor a charset declared in the + // response's Content-Type header. Without the bom-aware decoding the bytes would be (mis)interpreted as + // UTF-8 and the assertion below would fail. + String text = "ümlaut"; + byte[] iso = text.getBytes(StandardCharsets.ISO_8859_1); + HttpHeaders h = new HttpHeaders().set(HttpHeaderName.CONTENT_TYPE, "text/plain; charset=ISO-8859-1"); + DecodedResponse wrapper = new DecodedResponse(mockResponse(200, h, new byte[0]), fluxOf(iso), 0L); + + StepVerifier.create(wrapper.getBodyAsString()).expectNext(text).verifyComplete(); + } + + @Test + public void getBodyAsStringDetectsUtf8BomAndStripsIt() { + // A leading UTF-8 BOM (EF BB BF) must be detected and stripped from the decoded string, matching the base + // HttpResponse contract that CoreUtils.bomAwareToString implements. + String text = "with bom"; + byte[] bom = new byte[] { (byte) 0xEF, (byte) 0xBB, (byte) 0xBF }; + byte[] payload = text.getBytes(StandardCharsets.UTF_8); + byte[] withBom = new byte[bom.length + payload.length]; + System.arraycopy(bom, 0, withBom, 0, bom.length); + System.arraycopy(payload, 0, withBom, bom.length, payload.length); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), new byte[0]), fluxOf(withBom), 0L); + + StepVerifier.create(wrapper.getBodyAsString()).expectNext(text).verifyComplete(); + } + + @Test + public void getBodyAsStringDecodesUsingProvidedCharset() { + String text = "ümlaut"; + byte[] latin1 = text.getBytes(StandardCharsets.ISO_8859_1); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), new byte[0]), fluxOf(latin1), 0L); + + StepVerifier.create(wrapper.getBodyAsString(StandardCharsets.ISO_8859_1)).expectNext(text).verifyComplete(); + } + + @Test + public void inheritedGetBodyAsInputStreamUsesDecodedBytes() throws IOException { + // Base getBodyAsInputStream() routes through getBodyAsByteArray(), so the override is exercised end-to-end. + byte[] decoded = bytes("decoded stream"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + try (InputStream stream = wrapper.getBodyAsInputStream().block()) { + assertNotNull(stream); + assertArrayEquals(decoded, readAll(stream)); + } + } + + @Test + public void inheritedWriteBodyToWritesDecodedBytes() throws IOException { + byte[] decoded = bytes("write me"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + ByteArrayOutputStream sink = new ByteArrayOutputStream(); + try (WritableByteChannel channel = Channels.newChannel(sink)) { + wrapper.writeBodyTo(channel); + } + + assertArrayEquals(decoded, sink.toByteArray()); + } + + @Test + public void inheritedBufferReturnsResponseBackedByDecodedBytes() { + byte[] decoded = bytes("buffered"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, new HttpHeaders(), bytes("encoded")), fluxOf(decoded), 0L); + + HttpResponse buffered = wrapper.buffer(); + assertNotNull(buffered); + StepVerifier.create(buffered.getBodyAsByteArray()) + .expectNextMatches(b -> Arrays.equals(decoded, b)) + .verifyComplete(); + } + + @Test + public void inheritedGetBodyAsBinaryDataReturnsDecodedBytes() { + // Base HttpResponse#getBodyAsBinaryData() pulls from getBody() (our override), so the resulting BinaryData + // must contain the decoded payload, not the original wire body. A divergent Content-Length header is set + // to make the wire vs decoded distinction explicit and guard against regressions in header forwarding. + byte[] decoded = bytes("decoded payload"); + long decodedSize = decoded.length; + HttpHeaders h = new HttpHeaders().set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(decodedSize + 32)); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, h, bytes("encoded wire body")), fluxOf(decoded), decodedSize); + + BinaryData data = wrapper.getBodyAsBinaryData(); + assertNotNull(data); + assertArrayEquals(decoded, data.toBytes()); + } + + @Test + public void contentLengthIsOverriddenToDecodedSize() { + long wireSize = 500L; + long decodedSize = 300L; + HttpHeaders h = new HttpHeaders().set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(wireSize)) + .set(CUSTOM_HEADER, "preserve-me"); + DecodedResponse wrapper + = new DecodedResponse(mockResponse(200, h, new byte[0]), fluxOf(new byte[0]), decodedSize); + + assertEquals(String.valueOf(decodedSize), wrapper.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + assertEquals("preserve-me", wrapper.getHeaders().getValue(CUSTOM_HEADER)); + // Deprecated getHeaderValue must reflect the same override. + assertEquals(String.valueOf(decodedSize), + wrapper.getHeaderValue(HttpHeaderName.CONTENT_LENGTH.getCaseInsensitiveName())); + } + + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[1024]; + int n; + while ((n = stream.read(buf)) != -1) { + out.write(buf, 0, n); + } + return out.toByteArray(); + } +} diff --git a/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTests.java b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTests.java new file mode 100644 index 000000000000..6b0b8474ee5c --- /dev/null +++ b/sdk/storage/azure-storage-common/src/test/java/com/azure/storage/common/policy/StorageContentValidationDecoderPolicyTests.java @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.common.policy; + +import com.azure.core.http.HttpClient; +import com.azure.core.http.HttpHeaderName; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpMethod; +import com.azure.core.http.HttpPipeline; +import com.azure.core.http.HttpPipelineBuilder; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.HttpResponse; +import com.azure.core.test.http.MockHttpResponse; +import com.azure.core.util.Context; +import com.azure.core.util.FluxUtil; +import com.azure.storage.common.implementation.Constants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageConstants; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageEncoder; +import com.azure.storage.common.implementation.contentvalidation.StructuredMessageFlags; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests {@link StorageContentValidationDecoderPolicy} together with {@link StructuredMessageEncoder} / + * wire-format payloads so the reactive decode path matches what the blob download pipeline uses. + */ +public class StorageContentValidationDecoderPolicyTests { + + /** + * End-to-end through the policy: encoded body uses multi-megabyte segment payload lengths (not the default + * 4 MiB framing only); decoded flux must match the original bytes. + */ + @ParameterizedTest + @MethodSource("segmentPayloadSizeAndTotalPayloadSizeSupplier") + public void decodesDynamicallySizedSegmentStructuredMessageThroughPipeline(int segmentPayloadSize, + int totalPayloadSize) throws IOException { + byte[] originalData = new byte[totalPayloadSize]; + ThreadLocalRandom.current().nextBytes(originalData); + + byte[] encodedBytes + = encodeStructuredMessageWireBytes(originalData, segmentPayloadSize, StructuredMessageFlags.STORAGE_CRC64); + + AtomicReference requestAfterPolicies = new AtomicReference<>(); + HttpClient httpClient = request -> { + requestAfterPolicies.set(request); + HttpHeaders headers = structuredDownloadResponseHeaders(encodedBytes.length, totalPayloadSize); + return Mono.just(new MockHttpResponse(request, 200, headers, encodedBytes)); + }; + + HttpPipeline pipeline = new HttpPipelineBuilder().policies((context, next) -> { + context.setData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + return next.process(); + }, new StorageContentValidationDecoderPolicy()).httpClient(httpClient).build(); + + HttpRequest request = new HttpRequest(HttpMethod.GET, "https://example.blob.core.windows.net/c/b"); + try (HttpResponse response = pipeline.send(request, Context.NONE).block()) { + assertNotNull(response); + assertTrue(response instanceof DecodedResponse); + byte[] decoded = Objects.requireNonNull(response.getBodyAsByteArray().block()); + assertArrayEquals(originalData, decoded); + } + + HttpRequest sent = requestAfterPolicies.get(); + assertNotNull(sent); + assertEquals(StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE, + sent.getHeaders().getValue(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME)); + } + + @Test + public void contentLengthIsOverriddenToDecodedSizeWhenDecodingApplied() throws IOException { + byte[] payload = new byte[64]; + ThreadLocalRandom.current().nextBytes(payload); + + byte[] encoded = encodeStructuredMessageWireBytes(payload, 64, StructuredMessageFlags.STORAGE_CRC64); + long encodedLen = encoded.length; + long decodedLen = payload.length; + + HttpClient httpClient = request -> Mono.just( + new MockHttpResponse(request, 200, structuredDownloadResponseHeaders(encodedLen, decodedLen), encoded)); + + HttpPipeline pipeline = new HttpPipelineBuilder().policies((context, next) -> { + context.setData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + return next.process(); + }, new StorageContentValidationDecoderPolicy()).httpClient(httpClient).build(); + + HttpRequest request = new HttpRequest(HttpMethod.GET, "https://example.blob.core.windows.net/c/b"); + try (HttpResponse response = pipeline.send(request, Context.NONE).block()) { + assertNotNull(response); + assertEquals(String.valueOf(decodedLen), response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + } + } + + @Test + public void contentLengthMatchesActualDecodedBodySize() throws IOException { + byte[] payload = new byte[128]; + ThreadLocalRandom.current().nextBytes(payload); + + byte[] encoded = encodeStructuredMessageWireBytes(payload, 64, StructuredMessageFlags.STORAGE_CRC64); + long encodedLen = encoded.length; + long decodedLen = payload.length; + + HttpClient httpClient = request -> Mono.just( + new MockHttpResponse(request, 200, structuredDownloadResponseHeaders(encodedLen, decodedLen), encoded)); + + HttpPipeline pipeline = new HttpPipelineBuilder().policies((context, next) -> { + context.setData(StructuredMessageConstants.STRUCTURED_MESSAGE_DECODING_CONTEXT_KEY, true); + return next.process(); + }, new StorageContentValidationDecoderPolicy()).httpClient(httpClient).build(); + + HttpRequest request = new HttpRequest(HttpMethod.GET, "https://example.blob.core.windows.net/c/b"); + try (HttpResponse response = pipeline.send(request, Context.NONE).block()) { + assertNotNull(response); + assertEquals(String.valueOf(decodedLen), response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + + byte[] body = Objects.requireNonNull(FluxUtil.collectBytesInByteBufferStream(response.getBody()).block()); + assertEquals(decodedLen, body.length); + assertArrayEquals(payload, body); + } + } + + @Test + public void contentLengthIsUnchangedWhenDecodingNotApplied() throws IOException { + byte[] payload = new byte[64]; + ThreadLocalRandom.current().nextBytes(payload); + + byte[] encoded = encodeStructuredMessageWireBytes(payload, 64, StructuredMessageFlags.STORAGE_CRC64); + long encodedLen = encoded.length; + + HttpHeaders responseHeaders = new HttpHeaders().set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(encodedLen)); + HttpClient httpClient = request -> Mono.just(new MockHttpResponse(request, 200, responseHeaders, encoded)); + + HttpPipeline pipeline = new HttpPipelineBuilder().policies(new StorageContentValidationDecoderPolicy()) + .httpClient(httpClient) + .build(); + + HttpRequest request = new HttpRequest(HttpMethod.GET, "https://example.blob.core.windows.net/c/b"); + HttpResponse response = pipeline.send(request, Context.NONE).block(); + + assertNotNull(response); + assertEquals(String.valueOf(encodedLen), response.getHeaders().getValue(HttpHeaderName.CONTENT_LENGTH)); + } + + private static Stream segmentPayloadSizeAndTotalPayloadSizeSupplier() { + return Stream.of(Arguments.of(10 * 1024 * 1024, 10 * 1024 * 1024 + 1), // larger than 4 MiB + Arguments.of(3 * 1024 * 1024, 3 * 1024 * 1024 + 1), // smaller than 4 MiB, but not KB + Arguments.of(5 * 1024 * 1024 + 1, 15 * 1024 * 1024)); + } + + private static HttpHeaders structuredDownloadResponseHeaders(long contentLength, long structuredContentLength) { + HttpHeaders headers = new HttpHeaders(); + headers.set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(contentLength)); + headers.set(Constants.HeaderConstants.STRUCTURED_BODY_TYPE_HEADER_NAME, + StructuredMessageConstants.STRUCTURED_BODY_TYPE_VALUE); + headers.set(Constants.HeaderConstants.STRUCTURED_CONTENT_LENGTH_HEADER_NAME, + String.valueOf(structuredContentLength)); + return headers; + } + + private static byte[] encodeStructuredMessageWireBytes(byte[] originalData, int segmentLength, + StructuredMessageFlags flags) throws IOException { + StructuredMessageEncoder encoder = new StructuredMessageEncoder(originalData.length, segmentLength, flags); + Flux flux = encoder.encode(ByteBuffer.wrap(originalData)); + return Objects.requireNonNull(FluxUtil.collectBytesInByteBufferStream(flux).block()); + } +} diff --git a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/DataLakeServiceVersion.java b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/DataLakeServiceVersion.java index 3e3ec7595cee..70b574b062aa 100644 --- a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/DataLakeServiceVersion.java +++ b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/DataLakeServiceVersion.java @@ -162,7 +162,17 @@ public enum DataLakeServiceVersion implements ServiceVersion { /** * Service version {@code 2026-06-06}. */ - V2026_06_06("2026-06-06"); + V2026_06_06("2026-06-06"), + + /** + * Service version {@code 2026-10-06}. + */ + V2026_10_06("2026-10-06"), + + /** + * Service version {@code 2026-12-06}. + */ + V2026_12_06("2026-12-06"); private final String version; @@ -184,6 +194,6 @@ public String getVersion() { * @return the latest {@link DataLakeServiceVersion} */ public static DataLakeServiceVersion getLatest() { - return V2026_06_06; + return V2026_12_06; } } diff --git a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/implementation/util/TransformUtils.java b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/implementation/util/TransformUtils.java index fe07d636f95c..c53079d81b0d 100644 --- a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/implementation/util/TransformUtils.java +++ b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/implementation/util/TransformUtils.java @@ -107,6 +107,12 @@ public static BlobServiceVersion toBlobServiceVersion(DataLakeServiceVersion ver case V2026_06_06: return BlobServiceVersion.V2026_06_06; + case V2026_10_06: + return BlobServiceVersion.V2026_10_06; + + case V2026_12_06: + return BlobServiceVersion.V2026_12_06; + default: return null; } diff --git a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareServiceVersion.java b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareServiceVersion.java index 20d1e6de7cea..5c5aa77fd24c 100644 --- a/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareServiceVersion.java +++ b/sdk/storage/azure-storage-file-share/src/main/java/com/azure/storage/file/share/ShareServiceVersion.java @@ -162,7 +162,17 @@ public enum ShareServiceVersion implements ServiceVersion { /** * Service version {@code 2026-06-06}. */ - V2026_06_06("2026-06-06"); + V2026_06_06("2026-06-06"), + + /** + * Service version {@code 2026-10-06}. + */ + V2026_10_06("2026-10-06"), + + /** + * Service version {@code 2026-12-06}. + */ + V2026_12_06("2026-12-06"); private final String version; @@ -184,6 +194,6 @@ public String getVersion() { * @return the latest {@link ShareServiceVersion} */ public static ShareServiceVersion getLatest() { - return V2026_06_06; + return V2026_12_06; } } diff --git a/sdk/storage/azure-storage-queue/src/main/java/com/azure/storage/queue/QueueServiceVersion.java b/sdk/storage/azure-storage-queue/src/main/java/com/azure/storage/queue/QueueServiceVersion.java index efd0cded7d73..246f3f4fd797 100644 --- a/sdk/storage/azure-storage-queue/src/main/java/com/azure/storage/queue/QueueServiceVersion.java +++ b/sdk/storage/azure-storage-queue/src/main/java/com/azure/storage/queue/QueueServiceVersion.java @@ -162,7 +162,17 @@ public enum QueueServiceVersion implements ServiceVersion { /** * Service version {@code 2026-06-06}. */ - V2026_06_06("2026-06-06"); + V2026_06_06("2026-06-06"), + + /** + * Service version {@code 2026-10-06}. + */ + V2026_10_06("2026-10-06"), + + /** + * Service version {@code 2026-12-06}. + */ + V2026_12_06("2026-12-06"); private final String version; @@ -184,6 +194,6 @@ public String getVersion() { * @return the latest {@link QueueServiceVersion} */ public static QueueServiceVersion getLatest() { - return V2026_06_06; + return V2026_12_06; } }