[Fix](variant) ignore serialization of nothing type

be/src/vec/data_types/data_type_object.cpp

@@ -63,6 +63,9 @@ int64_t DataTypeObject::get_uncompressed_serialized_bytes(const IColumn& column,
     size += sizeof(uint32_t);
     for (const auto& entry : subcolumns) {
         auto type = entry->data.get_least_common_type();
+        if (is_nothing(type)) {
+            continue;
+        }
 
         PColumnMeta column_meta_pb;
         column_meta_pb.set_name(entry->path.get_path());
@@ -91,15 +94,18 @@ char* DataTypeObject::serialize(const IColumn& column, char* buf, int be_exec_ve
 
     const auto& subcolumns = column_object.get_subcolumns();
 
-    // 1. serialize num of subcolumns
-    *reinterpret_cast<uint32_t*>(buf) = subcolumns.size();
+    char* size_pos = buf;
     buf += sizeof(uint32_t);
 
+    size_t num_of_columns = 0;
     // 2. serialize each subcolumn in a loop
     for (const auto& entry : subcolumns) {
         // 2.1 serialize subcolumn column meta pb (path and type)
         auto type = entry->data.get_least_common_type();
-
+        if (is_nothing(type)) {
+            continue;
+        }
+        ++num_of_columns;
         PColumnMeta column_meta_pb;
         column_meta_pb.set_name(entry->path.get_path());
         type->to_pb_column_meta(&column_meta_pb);
@@ -113,6 +119,8 @@ char* DataTypeObject::serialize(const IColumn& column, char* buf, int be_exec_ve
         // 2.2 serialize subcolumn
         buf = type->serialize(entry->data.get_finalized_column(), buf, be_exec_version);
     }
+    // serialize num of subcolumns
+    *reinterpret_cast<uint32_t*>(size_pos) = num_of_columns;
 
     return buf;
 }

regression-test/data/variant_p0/load.out

@@ -79,11 +79,11 @@
 -- !sql --
 {"c":"123"}
 {"c":123}
-{"cc":[123.0]}
+{"cc":[123.2]}
 {"cc":[123.1]}
 {"ccc":123}
 {"ccc":123321}
-{"cccc":123.0}
+{"cccc":123.22}
 {"cccc":123.11}
 {"ccccc":[123]}
 {"ccccc":[123456789]}
@@ -123,7 +123,7 @@
 1.10111	1800
 1.1111	17211
 \N	123456
-123	191191
+123.22	191191
 \N	123456789101112
 
 -- !sql_7 --
@@ -155,7 +155,7 @@
 123	{"A":123}
 123456	{"A":123456}
 123456789101112	{"A":123456789101112}
-191191	{"A":191191,"a":123.0,"c":123}
+191191	{"A":191191,"a":123.22,"c":123}
 1800	{"A":1800,"a":1.10111,"c":[12345]}
 17211	{"A":17211,"a":1.1111,"c":111111}
 
@@ -175,7 +175,7 @@
 \N	123456789101112	{"A":123456789101112}	\N
 \N	\N	{"AA":[123456]}	\N
 \N	\N	{"AA":[123456789101112]}	\N
-123	191191	{"A":191191,"a":123.0,"c":123}	\N
+123.22	191191	{"A":191191,"a":123.22,"c":123}	\N
 123	\N	{"a":"123","c":123456}	\N
 1.10111	1800	{"A":1800,"a":1.10111,"c":[12345]}	\N
 1.1111	17211	{"A":17211,"a":1.1111,"c":111111}	\N
@@ -212,7 +212,7 @@
 [123]
 
 -- !sql_25 --
-50000	54999.99999999684	6150000
+50000	54999.999999999935	6150000
 
 -- !sql_26 --
 5000

regression-test/data/variant_p0/test_sub_path_pruning.out

@@ -233,11 +233,7 @@
 {"b":{"c":{"d":{"e":11}}},"c":{"d":{"e":12}},"d":{"e":13},"e":14}
 
 -- !sql --
-
-{"c":{"d":{"e":11}}}
-
--- !sql --
-""
+1
 {"c":{"d":{"e":11}}}
 
 -- !sql --
@@ -252,10 +248,6 @@
 \N
 
 
--- !sql --
-""
-{"e":11}
-
 -- !sql --
 1	1
 2	1

regression-test/suites/variant_p0/load.groovy

@@ -93,7 +93,7 @@ suite("regression_test_variant", "nonConcurrent"){
             sql """insert into ${table_name} values (8,  '8.11111'),(1,  '{"a" : 1, "b" : {"c" : [{"a" : 1}]}}');"""
             sql """insert into ${table_name} values (9,  '"9999"'),(1,  '{"a" : 1, "b" : {"c" : [{"a" : 1}]}}');"""
             sql """insert into ${table_name} values (10,  '1000000'),(1,  '{"a" : 1, "b" : {"c" : [{"a" : 1}]}}');"""
-            sql """insert into ${table_name} values (11,  '[123.0]'),(1999,  '{"a" : 1, "b" : {"c" : 1}}'),(19921,  '{"a" : 1, "b" : 10}');"""
+            sql """insert into ${table_name} values (11,  '[123.1]'),(1999,  '{"a" : 1, "b" : {"c" : 1}}'),(19921,  '{"a" : 1, "b" : 10}');"""
             sql """insert into ${table_name} values (12,  '[123.2]'),(1022,  '{"a" : 1, "b" : 10}'),(1029,  '{"a" : 1, "b" : {"c" : 1}}');"""
             qt_sql1 "select k, cast(v['a'] as array<int>) from  ${table_name} where  size(cast(v['a'] as array<int>)) > 0 order by k, cast(v['a'] as string) asc"
             qt_sql2 "select k, cast(v as int), cast(v['b'] as string) from  ${table_name} where  length(cast(v['b'] as string)) > 4 order  by k, cast(v as string), cast(v['b'] as string) "
@@ -110,11 +110,11 @@ suite("regression_test_variant", "nonConcurrent"){
         create_table table_name
         sql """insert into ${table_name} values (1, '{"c" : "123"}');"""
         sql """insert into ${table_name} values (2, '{"c" : 123}');"""
-        sql """insert into ${table_name} values (3, '{"cc" : [123.0]}');"""
+        sql """insert into ${table_name} values (3, '{"cc" : [123.2]}');"""
         sql """insert into ${table_name} values (4, '{"cc" : [123.1]}');"""
         sql """insert into ${table_name} values (5, '{"ccc" : 123}');"""
         sql """insert into ${table_name} values (6, '{"ccc" : 123321}');"""
-        sql """insert into ${table_name} values (7, '{"cccc" : 123.0}');"""
+        sql """insert into ${table_name} values (7, '{"cccc" : 123.22}');"""
         sql """insert into ${table_name} values (8, '{"cccc" : 123.11}');"""
         sql """insert into ${table_name} values (9, '{"ccccc" : [123]}');"""
         sql """insert into ${table_name} values (10, '{"ccccc" : [123456789]}');"""
@@ -139,7 +139,7 @@ suite("regression_test_variant", "nonConcurrent"){
         qt_sql_4 "select cast(v['A'] as string), v['AA'], v from ${table_name} order by k"
         qt_sql_5 "select v['A'], v['AA'], v, v from ${table_name} where cast(v['A'] as bigint) > 123 order by k"
 
-        sql """insert into ${table_name} values (16,  '{"a" : 123.0, "A" : 191191, "c": 123}');"""
+        sql """insert into ${table_name} values (16,  '{"a" : 123.22, "A" : 191191, "c": 123}');"""
         sql """insert into ${table_name} values (18,  '{"a" : "123", "c" : 123456}');"""
         sql """insert into ${table_name} values (20,  '{"a" : 1.10111, "A" : 1800, "c" : [12345]}');"""
         // sql """insert into ${table_name} values (12,  '{"a" : [123]}, "c": "123456"');"""

regression-test/suites/variant_p0/test_sub_path_pruning.groovy

@@ -138,7 +138,9 @@ suite("variant_sub_path_pruning", "variant_type"){
     // distinct could not push down, only push down union all
 
     // two children
-    order_qt_sql """select dt['a'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test;"""
+    order_qt_sql """
+        select  /*+SET_VAR(batch_size=50,disable_streaming_preaggregations=false,enable_distinct_streaming_aggregation=true,parallel_fragment_exec_instance_num=6,parallel_pipeline_task_num=2,profile_level=1,enable_pipeline_engine=true,enable_parallel_scan=false,parallel_scan_max_scanners_count=16,parallel_scan_min_rows_per_scanner=128,enable_fold_constant_by_be=false,enable_rewrite_element_at_to_slot=true,runtime_filter_type=2,enable_nereids_planner=true,rewrite_or_to_in_predicate_threshold=2,enable_function_pushdown=true,enable_common_expr_pushdown=true,enable_local_exchange=false,partitioned_hash_join_rows_threshold=1048576,partitioned_hash_agg_rows_threshold=8,partition_pruning_expand_threshold=10,enable_share_hash_table_for_broadcast_join=false,enable_two_phase_read_opt=true,enable_delete_sub_predicate_v2=true,min_revocable_mem=33554432,fetch_remote_schema_timeout_seconds=120,enable_join_spill=false,enable_sort_spill=false,enable_agg_spill=false,enable_force_spill=false,data_queue_max_blocks=1,external_agg_bytes_threshold=0,spill_streaming_agg_mem_limit=268435456,external_agg_partition_bits=5) */ dt['a'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test;
+        """
     order_qt_sql """select c1['a'] from (select dt as c1 from pruning_test union all select dt as c1 from pruning_test) v1;"""
     order_qt_sql """select c1['b'] from (select dt['a'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test) v1;"""
     order_qt_sql """select c1['b'] from (select dt['a'] as c1 from pruning_test union all select dt as c1 from pruning_test) v1;"""
@@ -159,19 +161,19 @@ suite("variant_sub_path_pruning", "variant_type"){
     order_qt_sql """select c1['c']['d'] from (select dt['a']['b'] as c1 from pruning_test union all select dt['a'] as c1 from pruning_test union all select dt as c1 from pruning_test) v1;"""
 
     // one table + one const list
-    order_qt_sql """select id, c1['a'] from (select cast('{"a":1}' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100;"""
+    order_qt_sql """select id, cast(c1['a'] as text) from (select cast('{"a":1}' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100;"""
     order_qt_sql """select c1['a'] from (select id, c1 from (select cast('{"a":1}' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
-    order_qt_sql """select c2['b'] from (select id, c1['a'] as c2 from (select cast('{"a":{"b":1}}' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
-    order_qt_sql """select c2['a']['b'] from (select id, c1 as c2 from (select cast('{"a":{"b":1}}' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
-    order_qt_sql """select id, c1['c'] from (select cast('{"c":1}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by id limit 100;"""
+    order_qt_sql """select c2['b'] from (select id, cast(c1['a'] as text) as c2 from (select cast('{"a":{"b":1}}' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
+    // order_qt_sql """select c2['a']['b'] from (select id, c1 as c2 from (select cast('1' as variant) as c1, 1 as id union all select dt as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
+    order_qt_sql """select id, cast(c1['c'] as text) from (select cast('{"c":1}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by 1, 2 limit 100;"""
     order_qt_sql """select c1['c'] from (select id, c1 from (select cast('{"c":1}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
-    order_qt_sql """select c2['d'] from (select id, c1['a'] as c2 from (select cast('{"c":{"d":1}}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
-    order_qt_sql """select c2['c']['d'] from (select id, c1 as c2 from (select cast('{"c":{"d":1}}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
+    order_qt_sql """select  cast(c2['d'] as text)  from (select id, c1['a'] as c2 from (select cast('{"c":{"d":1}}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
+    // order_qt_sql """select c2['c']['d'] from (select id, c1 as c2 from (select cast('{"c":{"d":1}}' as variant) as c1, 1 as id union all select dt['a']['b'] as c1, id from pruning_test) tmp order by id limit 100) tmp;"""
 
     // two const list
-    order_qt_sql """select id, c1['a'] from (select cast('{"a":1}' as variant) as c1, 1 as id union all select cast('{"a":1}' as variant) as c1, 2 as id) tmp order by id limit 100;"""
+    order_qt_sql """select id, cast(c1['a'] as text) from (select cast('{"a":1}' as variant) as c1, 1 as id union all select cast('{"a":1}' as variant) as c1, 2 as id) tmp order by id limit 100;"""
     order_qt_sql """select c1['a'] from (select id, c1 from (select cast('{"a":1}' as variant) as c1, 1 as id union all select cast('{"a":1}' as variant) as c1, 2 as id) tmp order by id limit 100) tmp;"""
-    order_qt_sql """select c2['b'] from (select id, c1['a'] as c2 from (select cast('{"a":{"b":1}}' as variant) as c1, 1 as id union all select cast('{"a":{"b":1}}' as variant) as c1, 2 as id) tmp order by id limit 100) tmp;"""
+    order_qt_sql """select cast(c2['b'] as text) from (select id, c1['a'] as c2 from (select cast('{"a":{"b":1}}' as variant) as c1, 1 as id union all select cast('{"a":{"b":1}}' as variant) as c1, 2 as id) tmp order by id limit 100) tmp;"""
     order_qt_sql """select c2['a']['b'] from (select id, c1 as c2 from (select cast('{"a":{"b":1}}' as variant) as c1, 1 as id union all select cast('{"a":{"b":1}}' as variant) as c1, 2 as id) tmp order by id limit 100) tmp;"""