diff --git a/core/src/main/java/org/apache/iceberg/ContentFileParser.java b/core/src/main/java/org/apache/iceberg/ContentFileParser.java index 0499fef6d90a..832ec713ab31 100644 --- a/core/src/main/java/org/apache/iceberg/ContentFileParser.java +++ b/core/src/main/java/org/apache/iceberg/ContentFileParser.java @@ -48,6 +48,8 @@ public class ContentFileParser { private static final String EQUALITY_IDS = "equality-ids"; private static final String SORT_ORDER_ID = "sort-order-id"; private static final String FIRST_ROW_ID = "first-row-id"; + private static final String DATA_SEQUENCE_NUMBER = "data-sequence-number"; + private static final String FILE_SEQUENCE_NUMBER = "file-sequence-number"; private static final String REFERENCED_DATA_FILE = "referenced-data-file"; private static final String CONTENT_OFFSET = "content-offset"; private static final String CONTENT_SIZE = "content-size-in-bytes"; @@ -122,6 +124,8 @@ public static void toJson(ContentFile contentFile, PartitionSpec spec, JsonGe } JsonUtil.writeLongFieldIfPresent(FIRST_ROW_ID, contentFile.firstRowId(), generator); + JsonUtil.writeLongFieldIfPresent(DATA_SEQUENCE_NUMBER, contentFile.dataSequenceNumber(), generator); + JsonUtil.writeLongFieldIfPresent(FILE_SEQUENCE_NUMBER, contentFile.fileSequenceNumber(), generator); if (contentFile instanceof DeleteFile) { DeleteFile deleteFile = (DeleteFile) contentFile; @@ -170,38 +174,48 @@ public static ContentFile fromJson(JsonNode jsonNode, Map deserialized = + ContentFileParser.fromJson(jsonNode, Map.of(0, spec)); + assertThat(deserialized).isInstanceOf(DataFile.class); + assertThat(deserialized.dataSequenceNumber()).isEqualTo(5L); + assertThat(deserialized.fileSequenceNumber()).isEqualTo(7L); + + String reSerialized = ContentFileParser.toJson(deserialized, spec); + assertThat(reSerialized).contains("\"data-sequence-number\":5"); + assertThat(reSerialized).contains("\"file-sequence-number\":7"); + + // Verify round-trip + ContentFile reParsed = + ContentFileParser.fromJson( + JsonUtil.mapper().readTree(reSerialized), Map.of(0, spec)); + assertThat(reParsed.dataSequenceNumber()).isEqualTo(5L); + assertThat(reParsed.fileSequenceNumber()).isEqualTo(7L); + } + + @Test + public void testSequenceNumberDefaultsToNull() throws Exception { + // JSON without sequence numbers should deserialize to null (not 0) + PartitionSpec spec = PartitionSpec.unpartitioned(); + String jsonStr = + "{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[],\"file-size-in-bytes\":10," + + "\"record-count\":1}"; + + JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr); + ContentFile deserialized = + ContentFileParser.fromJson(jsonNode, Map.of(0, spec)); + assertThat(deserialized.dataSequenceNumber()).isNull(); + assertThat(deserialized.fileSequenceNumber()).isNull(); + } + + @Test + public void testSequenceNumberForDeleteFile() throws Exception { + PartitionSpec spec = PartitionSpec.unpartitioned(); + String jsonStr = + "{\"spec-id\":0,\"content\":\"position-deletes\",\"file-path\":\"/path/to/delete.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[],\"file-size-in-bytes\":1234," + + "\"record-count\":10,\"data-sequence-number\":3,\"file-sequence-number\":4}"; + + JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr); + ContentFile deserialized = + ContentFileParser.fromJson(jsonNode, Map.of(0, spec)); + assertThat(deserialized).isInstanceOf(DeleteFile.class); + assertThat(deserialized.dataSequenceNumber()).isEqualTo(3L); + assertThat(deserialized.fileSequenceNumber()).isEqualTo(4L); + + String reSerialized = ContentFileParser.toJson(deserialized, spec); + assertThat(reSerialized).contains("\"data-sequence-number\":3"); + assertThat(reSerialized).contains("\"file-sequence-number\":4"); + } + private static Stream enumContentTypeCases() { return Stream.of( Arguments.of(FileContent.DATA, "data"), diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e9a530a12804..1ecbe7266296 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -5222,6 +5222,17 @@ components: description: "List of splittable offsets" sort-order-id: type: integer + data-sequence-number: + type: integer + format: int64 + description: + "The data sequence number of the file, used to determine which delete + files apply to it during reads (see the Iceberg spec)" + file-sequence-number: + type: integer + format: int64 + description: + "The sequence number of the snapshot in which the underlying file was added" DataFile: allOf: