[Fix](multi-catalog) Fix string dictionary filtering when using null related functions in parquet and orc reader by disabling dictionary filtering when predicates contain functions.

be/src/vec/exec/format/orc/vorc_reader.cpp

@@ -1987,21 +1987,20 @@ bool OrcReader::_can_filter_by_dict(int slot_id) {
         return false;
     }
 
-    // TODO：check expr like 'a > 10 is null', 'a > 10' should can be filter by dict.
     std::function<bool(const VExpr* expr)> visit_function_call = [&](const VExpr* expr) {
+        // TODO: The current implementation of dictionary filtering does not take into account
+        //  the implementation of NULL values because the dictionary itself does not contain
+        //  NULL value encoding. As a result, many NULL-related functions or expressions
+        //  cannot work properly, such as is null, is not null, coalesce, etc.
+        //  Here we first disable dictionary filtering when predicate contains functions.
+        //  Implementation of NULL value dictionary filtering will be carried out later.
         if (expr->node_type() == TExprNodeType::FUNCTION_CALL) {
-            std::string is_null_str;
-            std::string function_name = expr->fn().name.function_name;
-            if (function_name.compare("is_null_pred") == 0 ||
-                function_name.compare("is_not_null_pred") == 0) {
+            return false;
+        }
+        for (auto& child : expr->children()) {
+            if (!visit_function_call(child.get())) {
                 return false;
             }
-        } else {
-            for (auto& child : expr->children()) {
-                if (!visit_function_call(child.get())) {
-                    return false;
-                }
-            }
         }
         return true;
     };

be/src/vec/exec/format/parquet/vparquet_group_reader.cpp

@@ -203,21 +203,20 @@ bool RowGroupReader::_can_filter_by_dict(int slot_id,
         return false;
     }
 
-    // TODO：check expr like 'a > 10 is null', 'a > 10' should can be filter by dict.
     std::function<bool(const VExpr* expr)> visit_function_call = [&](const VExpr* expr) {
+        // TODO: The current implementation of dictionary filtering does not take into account
+        //  the implementation of NULL values because the dictionary itself does not contain
+        //  NULL value encoding. As a result, many NULL-related functions or expressions
+        //  cannot work properly, such as is null, is not null, coalesce, etc.
+        //  Here we first disable dictionary filtering when predicate contains functions.
+        //  Implementation of NULL value dictionary filtering will be carried out later.
         if (expr->node_type() == TExprNodeType::FUNCTION_CALL) {
-            std::string is_null_str;
-            std::string function_name = expr->fn().name.function_name;
-            if (function_name.compare("is_null_pred") == 0 ||
-                function_name.compare("is_not_null_pred") == 0) {
+            return false;
+        }
+        for (auto& child : expr->children()) {
+            if (!visit_function_call(child.get())) {
                 return false;
             }
-        } else {
-            for (auto& child : expr->children()) {
-                if (!visit_function_call(child.get())) {
-                    return false;
-                }
-            }
         }
         return true;
     };

docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql

@@ -1942,6 +1942,48 @@ TBLPROPERTIES (
 
 msck repair table string_col_dict_plain_mixed_orc;
 
+CREATE TABLE `test_string_dict_filter_parquet`(
+  `o_orderkey` int,
+  `o_custkey` int,
+  `o_orderstatus` string,
+  `o_totalprice` decimal(15,2),
+  `o_orderdate` date,
+  `o_orderpriority` string,
+  `o_clerk` string,
+  `o_shippriority` int,
+  `o_comment` string)
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+LOCATION
+  '/user/doris/preinstalled_data/parquet_table/test_string_dict_filter_parquet';
+
+msck repair table test_string_dict_filter_parquet;
+
+CREATE TABLE `test_string_dict_filter_orc`(
+  `o_orderkey` int,
+  `o_custkey` int,
+  `o_orderstatus` string,
+  `o_totalprice` decimal(15,2),
+  `o_orderdate` date,
+  `o_orderpriority` string,
+  `o_clerk` string,
+  `o_shippriority` int,
+  `o_comment` string)
+ROW FORMAT SERDE
+  'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
+STORED AS INPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
+OUTPUTFORMAT
+  'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
+LOCATION
+  '/user/doris/preinstalled_data/orc_table/test_string_dict_filter_orc';
+
+msck repair table test_string_dict_filter_orc;
+
 show tables;
 
 

regression-test/data/external_table_p0/hive/test_string_dict_filter.out

@@ -0,0 +1,229 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q01 --
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q02 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+4	136777	O	32151.78	1995-10-11	\N	Clerk#000000124	0	sits. slyly regular warthogs cajole. regular, regular theodolites acro
+
+-- !q03 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+4	136777	O	32151.78	1995-10-11	\N	Clerk#000000124	0	sits. slyly regular warthogs cajole. regular, regular theodolites acro
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q04 --
+4	136777	O	32151.78	1995-10-11	\N	Clerk#000000124	0	sits. slyly regular warthogs cajole. regular, regular theodolites acro
+
+-- !q05 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q06 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q07 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q08 --
+
+-- !q09 --
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q10 --
+null
+
+-- !q11 --
+null
+
+-- !q12 --
+null
+
+-- !q13 --
+null
+
+-- !q14 --
+null
+
+-- !q01 --
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q02 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+4	136777	O	32151.78	1995-10-11	\N	Clerk#000000124	0	sits. slyly regular warthogs cajole. regular, regular theodolites acro
+
+-- !q03 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+4	136777	O	32151.78	1995-10-11	\N	Clerk#000000124	0	sits. slyly regular warthogs cajole. regular, regular theodolites acro
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q04 --
+4	136777	O	32151.78	1995-10-11	\N	Clerk#000000124	0	sits. slyly regular warthogs cajole. regular, regular theodolites acro
+
+-- !q05 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q06 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q07 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q08 --
+
+-- !q09 --
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q10 --
+null
+
+-- !q11 --
+null
+
+-- !q12 --
+null
+
+-- !q13 --
+null
+
+-- !q14 --
+null
+
+-- !q01 --
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q02 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+4	136777	O	32151.78	1995-10-11	\N	Clerk#000000124	0	sits. slyly regular warthogs cajole. regular, regular theodolites acro
+
+-- !q03 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+4	136777	O	32151.78	1995-10-11	\N	Clerk#000000124	0	sits. slyly regular warthogs cajole. regular, regular theodolites acro
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q04 --
+4	136777	O	32151.78	1995-10-11	\N	Clerk#000000124	0	sits. slyly regular warthogs cajole. regular, regular theodolites acro
+
+-- !q05 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q06 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q07 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q08 --
+
+-- !q09 --
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q10 --
+null
+
+-- !q11 --
+null
+
+-- !q12 --
+null
+
+-- !q13 --
+null
+
+-- !q14 --
+null
+
+-- !q01 --
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q02 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+4	136777	O	32151.78	1995-10-11	\N	Clerk#000000124	0	sits. slyly regular warthogs cajole. regular, regular theodolites acro
+
+-- !q03 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+4	136777	O	32151.78	1995-10-11	\N	Clerk#000000124	0	sits. slyly regular warthogs cajole. regular, regular theodolites acro
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q04 --
+4	136777	O	32151.78	1995-10-11	\N	Clerk#000000124	0	sits. slyly regular warthogs cajole. regular, regular theodolites acro
+
+-- !q05 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q06 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q07 --
+1	36901	O	173665.47	1996-01-02	5-LOW	Clerk#000000951	0	nstructions sleep furiously among
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q08 --
+
+-- !q09 --
+2	78002	O	46929.18	1996-12-01	1-URGENT	Clerk#000000880	0	foxes. pending accounts at the pending, silent asymptot
+3	123314	F	193846.25	1993-10-14	5-LOW	Clerk#000000955	0	sly final accounts boost. carefully regular ideas cajole carefully. depos
+5	44485	F	144659.20	1994-07-30	5-LOW	Clerk#000000925	0	quickly. bold deposits sleep slyly. packages use slyly
+
+-- !q10 --
+null
+
+-- !q11 --
+null
+
+-- !q12 --
+null
+
+-- !q13 --
+null
+
+-- !q14 --
+null
+