[refine](column) enforce nullable nested types for array (#63088)

be/src/core/data_type/data_type_array.cpp

@@ -48,7 +48,13 @@ namespace ErrorCodes {
 extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
 }
 
-DataTypeArray::DataTypeArray(const DataTypePtr& nested_) : nested {nested_} {}
+DataTypeArray::DataTypeArray(const DataTypePtr& nested_) {
+    DataTypePtr nullable_nested = make_nullable(nested_);
+    auto nested_type = std::dynamic_pointer_cast<const DataTypeNullable>(nullable_nested);
+    DORIS_CHECK(nested_type != nullptr);
+    nested = std::move(nested_type);
+    nested_as_base = nested;
+}
 
 MutableColumnPtr DataTypeArray::create_column() const {
     return ColumnArray::create(nested->create_column(), ColumnArray::ColumnOffsets::create());
@@ -66,9 +72,10 @@ bool DataTypeArray::equals(const IDataType& rhs) const {
 
 // here we should remove nullable, otherwise here always be 1
 size_t DataTypeArray::get_number_of_dimensions() const {
-    const DataTypeArray* nested_array =
-            typeid_cast<const DataTypeArray*>(remove_nullable(nested).get());
-    if (!nested_array) return 1;
+    auto* nested_array = typeid_cast<const DataTypeArray*>(remove_nullable(nested).get());
+    if (!nested_array) {
+        return 1;
+    }
     return 1 +
            nested_array
                    ->get_number_of_dimensions(); /// Every modern C++ compiler optimizes tail recursion.
@@ -166,7 +173,7 @@ const char* DataTypeArray::deserialize(const char* buf, MutableColumnPtr* column
 
 void DataTypeArray::to_pb_column_meta(PColumnMeta* col_meta) const {
     IDataType::to_pb_column_meta(col_meta);
-    auto children = col_meta->add_children();
+    auto* children = col_meta->add_children();
     get_nested_type()->to_pb_column_meta(children);
 }
 

be/src/core/data_type/data_type_array.h

@@ -29,6 +29,7 @@
 
 #include "common/status.h"
 #include "core/data_type/data_type.h"
+#include "core/data_type/data_type_nullable.h"
 #include "core/data_type/define_primitive_type.h"
 #include "core/data_type_serde/data_type_array_serde.h"
 #include "core/data_type_serde/data_type_serde.h"
@@ -47,7 +48,8 @@ namespace doris {
 class DataTypeArray final : public IDataType {
 private:
     /// The type of array elements.
-    DataTypePtr nested;
+    DataTypeNullablePtr nested;
+    DataTypePtr nested_as_base;
 
 public:
     static constexpr PrimitiveType PType = TYPE_ARRAY;
@@ -74,7 +76,8 @@ class DataTypeArray final : public IDataType {
 
     bool equals(const IDataType& rhs) const override;
 
-    const DataTypePtr& get_nested_type() const { return nested; }
+    const DataTypePtr& get_nested_type() const { return nested_as_base; }
+    const DataTypeNullablePtr& get_nullable_nested_type() const { return nested; }
 
     /// 1 for plain array, 2 for array of arrays and so on.
     size_t get_number_of_dimensions() const;
@@ -94,15 +97,15 @@ class DataTypeArray final : public IDataType {
     void to_protobuf(PTypeDesc* ptype, PTypeNode* node, PScalarType* scalar_type) const override {
         node->set_type(TTypeNodeType::ARRAY);
         node->set_contains_null(nested->is_nullable());
-        nested->to_protobuf(ptype);
+        get_nested_type()->to_protobuf(ptype);
     }
 
 #ifdef BE_TEST
     void to_thrift(TTypeDesc& thrift_type, TTypeNode& node) const override {
         node.type = TTypeNodeType::ARRAY;
         node.__isset.contains_nulls = true;
         node.contains_nulls.push_back(nested->is_nullable());
-        nested->to_thrift(thrift_type);
+        get_nested_type()->to_thrift(thrift_type);
     }
 #endif
 };

be/src/core/data_type/primitive_type.h

@@ -93,10 +93,12 @@ class DataTypeHLL;
 class DataTypeJsonb;
 class DataTypeArray;
 class DataTypeMap;
+class DataTypeNullable;
 class DataTypeVariant;
 class DataTypeStruct;
 class DataTypeBitMap;
 class DataTypeQuantileState;
+using DataTypeNullablePtr = std::shared_ptr<const DataTypeNullable>;
 template <PrimitiveType T>
 class ColumnVector;
 using ColumnUInt8 = ColumnVector<TYPE_BOOLEAN>;

be/src/core/data_type_serde/data_type_array_serde.cpp

@@ -86,7 +86,7 @@ Status DataTypeArraySerDe::deserialize_one_cell_from_json(IColumn& column, Slice
     auto& array_column = assert_cast<ColumnArray&>(column);
     auto& offsets = array_column.get_offsets();
     IColumn& nested_column = array_column.get_data();
-    DCHECK(nested_column.is_nullable());
+    DORIS_CHECK(nested_column.is_nullable());
     if (slice[0] != '[') {
         return Status::InvalidArgument("Array does not start with '[' character, found '{}'",
                                        slice[0]);
@@ -164,7 +164,7 @@ Status DataTypeArraySerDe::deserialize_one_cell_from_hive_text(
     auto& array_column = assert_cast<ColumnArray&>(column);
     auto& offsets = array_column.get_offsets();
     IColumn& nested_column = array_column.get_data();
-    DCHECK(nested_column.is_nullable());
+    DORIS_CHECK(nested_column.is_nullable());
 
     char collection_delimiter =
             options.get_collection_delimiter(hive_text_complex_type_delimiter_level);

be/src/core/data_type_serde/data_type_array_serde.h

@@ -25,6 +25,7 @@
 #include <utility>
 
 #include "common/status.h"
+#include "core/data_type_serde/data_type_nullable_serde.h"
 #include "core/data_type_serde/data_type_serde.h"
 
 namespace doris {
@@ -38,7 +39,12 @@ class IDataType;
 class DataTypeArraySerDe : public DataTypeSerDe {
 public:
     DataTypeArraySerDe(DataTypeSerDeSPtr _nested_serde, int nesting_level = 1)
-            : DataTypeSerDe(nesting_level), nested_serde(std::move(_nested_serde)) {}
+            : DataTypeSerDe(nesting_level) {
+        auto nullable_serde =
+                std::dynamic_pointer_cast<DataTypeNullableSerDe>(std::move(_nested_serde));
+        DORIS_CHECK(nullable_serde != nullptr);
+        nested_serde = std::move(nullable_serde);
+    }
 
     std::string get_name() const override { return "Array(" + nested_serde->get_name() + ")"; }
 
@@ -132,6 +138,6 @@ class DataTypeArraySerDe : public DataTypeSerDe {
     template <bool is_strict_mode>
     Status _from_string(StringRef& str, IColumn& column, const FormatOptions& options) const;
 
-    DataTypeSerDeSPtr nested_serde;
+    DataTypeNullableSerDeSPtr nested_serde;
 };
 } // namespace doris

be/src/core/data_type_serde/data_type_nullable_serde.h

@@ -130,5 +130,8 @@ class DataTypeNullableSerDe : public DataTypeSerDe {
 private:
     DataTypeSerDeSPtr nested_serde;
 };
+
+using DataTypeNullableSerDeSPtr = std::shared_ptr<DataTypeNullableSerDe>;
+
 #include "common/compile_check_end.h"
 } // namespace doris

be/src/exprs/function/cast/cast_to_array.h

@@ -52,7 +52,7 @@ WrapperType create_array_wrapper(FunctionContext* context, const DataTypePtr& fr
                 "CAST AS Array can only be performed between same-dimensional array types");
     }
 
-    const DataTypePtr& to_nested_type = to_type.get_nested_type();
+    DataTypePtr to_nested_type = to_type.get_nested_type();
 
     /// Prepare nested type conversion
     const auto nested_function =

be/test/core/block/block_test.cpp

@@ -99,7 +99,8 @@ static void fill_block_with_array_int(Block& block) {
         data_column->insert_data((const char*)(&v), 0);
     }
 
-    auto column_array_ptr = ColumnArray::create(std::move(data_column), std::move(off_column));
+    auto column_array_ptr =
+            ColumnArray::create(make_nullable(std::move(data_column)), std::move(off_column));
     DataTypePtr nested_type(std::make_shared<DataTypeInt32>());
     DataTypePtr array_type(std::make_shared<DataTypeArray>(nested_type));
     ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type, "test_array_int");
@@ -119,7 +120,8 @@ static void fill_block_with_array_string(Block& block) {
         data_column->insert_data(v.data(), v.size());
     }
 
-    auto column_array_ptr = ColumnArray::create(std::move(data_column), std::move(off_column));
+    auto column_array_ptr =
+            ColumnArray::create(make_nullable(std::move(data_column)), std::move(off_column));
     DataTypePtr nested_type(std::make_shared<DataTypeString>());
     DataTypePtr array_type(std::make_shared<DataTypeArray>(nested_type));
     ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,

be/test/core/data_type_serde/data_type_serde_get_name_test.cpp

@@ -45,7 +45,7 @@ TEST(DataTypeSerDeGetNameTest, test) {
     {
         auto type = std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt64>());
         auto serde = type->get_serde();
-        EXPECT_EQ(serde->get_name(), "Array(BIGINT)");
+        EXPECT_EQ(serde->get_name(), "Array(Nullable(BIGINT))");
     }
 
     {
@@ -114,7 +114,7 @@ TEST(DataTypeSerDeGetNameTest, test) {
         auto serde = type->get_serde();
         EXPECT_EQ(
                 serde->get_name(),
-                R"(Struct(field1:String, field2:BIGINT, field3:DOUBLE, field4:Array(INT), field5:Map(String, BIGINT), field6:Nullable(String), field7:Nullable(BIGINT)))");
+                R"(Struct(field1:String, field2:BIGINT, field3:DOUBLE, field4:Array(Nullable(INT)), field5:Map(String, BIGINT), field6:Nullable(String), field7:Nullable(BIGINT)))");
     }
 }
 

be/test/core/data_type_serde/data_type_serde_string_test.cpp

@@ -32,6 +32,7 @@
 #include "core/assert_cast.h"
 #include "core/column/column.h"
 #include "core/column/column_array.h"
+#include "core/column/column_nullable.h"
 #include "core/data_type/common_data_type_serder_test.h"
 #include "core/data_type/common_data_type_test.h"
 #include "core/data_type/data_type.h"
@@ -308,9 +309,12 @@ TEST_F(DataTypeStringSerDeTest, ArrowMemNotAlignedNestedArr) {
     EXPECT_EQ(values_address % 4, 1);
 
     // 5.Test read_column_from_arrow
-    auto ser_col = ColumnArray::create(ColumnString::create(), ColumnOffset64::create());
+    auto ser_col = ColumnArray::create(
+            ColumnNullable::create(ColumnString::create(), ColumnUInt8::create()),
+            ColumnOffset64::create());
     cctz::time_zone tz;
-    auto serde_list = std::make_shared<DataTypeArraySerDe>(serde_str);
+    auto serde_nullable_str = std::make_shared<DataTypeNullableSerDe>(serde_str);
+    auto serde_list = std::make_shared<DataTypeArraySerDe>(serde_nullable_str);
     auto st = serde_list->read_column_from_arrow(*ser_col, arr.get(), 0, 1, tz);
     EXPECT_TRUE(st.ok());
 }

be/test/core/jsonb/serialize_test.cpp

@@ -591,7 +591,8 @@ static void fill_block_with_array_int(Block& block) {
         data_column->insert_data((const char*)(&v), 0);
     }
 
-    auto column_array_ptr = ColumnArray::create(std::move(data_column), std::move(off_column));
+    auto column_array_ptr =
+            ColumnArray::create(make_nullable(std::move(data_column)), std::move(off_column));
     DataTypePtr nested_type(std::make_shared<DataTypeInt32>());
     DataTypePtr array_type(std::make_shared<DataTypeArray>(nested_type));
     ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type, "test_array_int");
@@ -611,7 +612,8 @@ static void fill_block_with_array_string(Block& block) {
         data_column->insert_data(v.data(), v.size());
     }
 
-    auto column_array_ptr = ColumnArray::create(std::move(data_column), std::move(off_column));
+    auto column_array_ptr =
+            ColumnArray::create(make_nullable(std::move(data_column)), std::move(off_column));
     DataTypePtr nested_type(std::make_shared<DataTypeString>());
     DataTypePtr array_type(std::make_shared<DataTypeArray>(nested_type));
     ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,

be/test/exec/common/schema_util_test.cpp

@@ -889,8 +889,8 @@ TEST_F(SchemaUtilTest, TestCastColumnWithExecuteFailure) {
     auto simple_type = std::make_shared<DataTypeJsonb>();
 
     // Insert some test dataset
-    auto nested_array =
-            ColumnArray::create(ColumnIPv4::create(), ColumnArray::ColumnOffsets::create());
+    auto nested_array = ColumnArray::create(make_nullable(ColumnIPv4::create()),
+                                            ColumnArray::ColumnOffsets::create());
     nested_array->insert(Field::create_field<PrimitiveType::TYPE_ARRAY>(Array(IPv4(1))));
     nested_array->insert(Field::create_field<PrimitiveType::TYPE_ARRAY>(Array(IPv4(2))));