From 14dfcc21ad899f865678fbd3e9b164b8cf5d553d Mon Sep 17 00:00:00 2001 From: waterWang Date: Thu, 27 Aug 2026 06:00:00 +0000 Subject: [PATCH] Core: Fix REST scan tasks dropping content file sequence numbers ContentFileParser serialized neither the data sequence number nor the file sequence number of a content file, and the REST ContentFile schema did not define fields for them. Any file scan task that crosses the REST scan planning API therefore arrived with null sequence numbers on its data file and delete files. Since delete files apply only to data files with an equal or smaller data sequence number, engines consuming those tasks could not determine delete applicability at read time, breaking merge-on-read for tables with delete files. This change serializes and deserializes data-sequence-number and file-sequence-number in ContentFileParser (following the existing first-row-id pattern), and adds both fields to the REST ContentFile schema. Fixes #17833 --- .../org/apache/iceberg/ContentFileParser.java | 66 +++++++++++-------- .../apache/iceberg/TestContentFileParser.java | 63 ++++++++++++++++++ open-api/rest-catalog-open-api.yaml | 11 ++++ 3 files changed, 114 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/ContentFileParser.java b/core/src/main/java/org/apache/iceberg/ContentFileParser.java index 0499fef6d90a..832ec713ab31 100644 --- a/core/src/main/java/org/apache/iceberg/ContentFileParser.java +++ b/core/src/main/java/org/apache/iceberg/ContentFileParser.java @@ -48,6 +48,8 @@ public class ContentFileParser { private static final String EQUALITY_IDS = "equality-ids"; private static final String SORT_ORDER_ID = "sort-order-id"; private static final String FIRST_ROW_ID = "first-row-id"; + private static final String DATA_SEQUENCE_NUMBER = "data-sequence-number"; + private static final String FILE_SEQUENCE_NUMBER = "file-sequence-number"; private static final String REFERENCED_DATA_FILE = "referenced-data-file"; private static final String CONTENT_OFFSET = "content-offset"; private static final String CONTENT_SIZE = "content-size-in-bytes"; @@ -122,6 +124,8 @@ public static void toJson(ContentFile contentFile, PartitionSpec spec, JsonGe } JsonUtil.writeLongFieldIfPresent(FIRST_ROW_ID, contentFile.firstRowId(), generator); + JsonUtil.writeLongFieldIfPresent(DATA_SEQUENCE_NUMBER, contentFile.dataSequenceNumber(), generator); + JsonUtil.writeLongFieldIfPresent(FILE_SEQUENCE_NUMBER, contentFile.fileSequenceNumber(), generator); if (contentFile instanceof DeleteFile) { DeleteFile deleteFile = (DeleteFile) contentFile; @@ -170,38 +174,48 @@ public static ContentFile fromJson(JsonNode jsonNode, Map deserialized = + ContentFileParser.fromJson(jsonNode, Map.of(0, spec)); + assertThat(deserialized).isInstanceOf(DataFile.class); + assertThat(deserialized.dataSequenceNumber()).isEqualTo(5L); + assertThat(deserialized.fileSequenceNumber()).isEqualTo(7L); + + String reSerialized = ContentFileParser.toJson(deserialized, spec); + assertThat(reSerialized).contains("\"data-sequence-number\":5"); + assertThat(reSerialized).contains("\"file-sequence-number\":7"); + + // Verify round-trip + ContentFile reParsed = + ContentFileParser.fromJson( + JsonUtil.mapper().readTree(reSerialized), Map.of(0, spec)); + assertThat(reParsed.dataSequenceNumber()).isEqualTo(5L); + assertThat(reParsed.fileSequenceNumber()).isEqualTo(7L); + } + + @Test + public void testSequenceNumberDefaultsToNull() throws Exception { + // JSON without sequence numbers should deserialize to null (not 0) + PartitionSpec spec = PartitionSpec.unpartitioned(); + String jsonStr = + "{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[],\"file-size-in-bytes\":10," + + "\"record-count\":1}"; + + JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr); + ContentFile deserialized = + ContentFileParser.fromJson(jsonNode, Map.of(0, spec)); + assertThat(deserialized.dataSequenceNumber()).isNull(); + assertThat(deserialized.fileSequenceNumber()).isNull(); + } + + @Test + public void testSequenceNumberForDeleteFile() throws Exception { + PartitionSpec spec = PartitionSpec.unpartitioned(); + String jsonStr = + "{\"spec-id\":0,\"content\":\"position-deletes\",\"file-path\":\"/path/to/delete.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[],\"file-size-in-bytes\":1234," + + "\"record-count\":10,\"data-sequence-number\":3,\"file-sequence-number\":4}"; + + JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr); + ContentFile deserialized = + ContentFileParser.fromJson(jsonNode, Map.of(0, spec)); + assertThat(deserialized).isInstanceOf(DeleteFile.class); + assertThat(deserialized.dataSequenceNumber()).isEqualTo(3L); + assertThat(deserialized.fileSequenceNumber()).isEqualTo(4L); + + String reSerialized = ContentFileParser.toJson(deserialized, spec); + assertThat(reSerialized).contains("\"data-sequence-number\":3"); + assertThat(reSerialized).contains("\"file-sequence-number\":4"); + } + private static Stream enumContentTypeCases() { return Stream.of( Arguments.of(FileContent.DATA, "data"), diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e9a530a12804..1ecbe7266296 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -5222,6 +5222,17 @@ components: description: "List of splittable offsets" sort-order-id: type: integer + data-sequence-number: + type: integer + format: int64 + description: + "The data sequence number of the file, used to determine which delete + files apply to it during reads (see the Iceberg spec)" + file-sequence-number: + type: integer + format: int64 + description: + "The sequence number of the snapshot in which the underlying file was added" DataFile: allOf: