Skip to content

[FIX](type)fix matchExactType for complex type #28233

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morrySnow merged 8 commits into from
Dec 21, 2023
Merged

[FIX](type)fix matchExactType for complex type #28233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d548282
fix type match type
Dec 11, 2023
e3caf16
add cases and fix type match
Dec 12, 2023
054be24
Merge branch 'master' into fix-struct-nested-empty-literal
Dec 12, 2023
331968e
fix tests
Dec 14, 2023
79886fd
update functions
Dec 14, 2023
1fcd09b
fix bugs
Dec 15, 2023
15dee69
format fe code
Dec 15, 2023
d8c9bce
move zip log to function call
Dec 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,13 +88,11 @@ public boolean matchesType(Type t) {
return false;
}

// Array(Null) is a virtual Array type, can match any Array(...) type
if (itemType.isNull() || ((ArrayType) t).getItemType().isNull()) {
return true;
if (((ArrayType) t).getContainsNull() != getContainsNull()) {
Comment thread
amorynan marked this conversation as resolved.
return false;
}

return itemType.matchesType(((ArrayType) t).itemType)
&& (((ArrayType) t).containsNull || !containsNull);
return itemType.matchesType(((ArrayType) t).itemType);
}

@Override
Expand Down
8 changes: 5 additions & 3 deletions fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,11 @@ public boolean matchesType(Type t) {
return false;
}

if ((keyType.isNull() || ((MapType) t).getKeyType().isNull())
&& (valueType.isNull() || ((MapType) t).getKeyType().isNull())) {
return true;
if (((MapType) t).getIsKeyContainsNull() != getIsKeyContainsNull()) {
return false;
}
if (((MapType) t).getIsValueContainsNull() != getIsValueContainsNull()) {
return false;
}

return keyType.matchesType(((MapType) t).keyType)
Expand Down
3 changes: 2 additions & 1 deletion fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ public Type specializeTemplateType(Type specificType, Map<String, Type> speciali
}

if (specializedType != null
&& !specializedType.isNull()
&& !specificType.equals(specializedType)
&& !specificType.matchesType(specializedType)
&& !Type.isImplicitlyCastable(specificType, specializedType, true, enableDecimal256)
Expand All @@ -104,7 +105,7 @@ public Type specializeTemplateType(Type specificType, Map<String, Type> speciali
name, specificType, specializedType));
}

if (specializedType == null) {
if (specializedType == null || specializedType.isNull()) {
specializedTypeMap.put(name, specificType);
}
return specializedTypeMap.get(name);
Expand Down
17 changes: 1 addition & 16 deletions fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2240,6 +2240,7 @@ public static boolean matchExactType(Type type1, Type type2) {
}

public static boolean matchExactType(Type type1, Type type2, boolean ignorePrecision) {
// we should make type decide to match other for itself to impl matchesType instead of switch case types
if (type1.matchesType(type2)) {
if (PrimitiveType.typeWithPrecision.contains(type2.getPrimitiveType())) {
// For types which has precision and scale, we also need to check quality between precisions and scales
Expand All @@ -2252,22 +2253,6 @@ public static boolean matchExactType(Type type1, Type type2, boolean ignorePreci
return isSameDecimalTypeWithDifferentPrecision(((ScalarType) type2).decimalPrecision(),
((ScalarType) type1).decimalPrecision());
}
} else if (type2.isArrayType()) {
Comment thread
xiaokang marked this conversation as resolved.
// For types array, we also need to check contains null for case like
// cast(array<not_null(int)> as array<int>)
if (((ArrayType) type2).getContainsNull() != ((ArrayType) type1).getContainsNull()) {
return false;
}
return matchExactType(((ArrayType) type2).getItemType(), ((ArrayType) type1).getItemType());
} else if (type2.isMapType()) {
if (((MapType) type2).getIsKeyContainsNull() != ((MapType) type1).getIsKeyContainsNull()) {
return false;
}
if (((MapType) type2).getIsValueContainsNull() != ((MapType) type1).getIsValueContainsNull()) {
return false;
}
return matchExactType(((MapType) type2).getKeyType(), ((MapType) type1).getKeyType())
&& matchExactType(((MapType) type2).getValueType(), ((MapType) type1).getValueType());
} else {
return true;
}
Expand Down
8 changes: 8 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1858,6 +1858,14 @@ && collectChildReturnTypes()[0].isDecimalV3()) {
&& fnName.getFunction().equalsIgnoreCase("map")) {
ix = i % 2 == 0 ? 0 : 1;
}
// array_zip varargs special case array_zip(array1, array2, ...)
// we only specialize array_zip with first array type, next type we same with custom type
if (i >= args.length && (fnName.getFunction().equalsIgnoreCase("array_zip"))) {
if (argTypes[i].isNull()) {
uncheckedCastChild(args[i - 1], i);
}
continue;
}

if (i == 0 && (fnName.getFunction().equalsIgnoreCase("char"))) {
continue;
Expand Down
17 changes: 7 additions & 10 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,17 +47,14 @@ public static void initBuiltins(FunctionSet functionSet) {

functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NOT_NULL,
null, Lists.newArrayList(t), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));
}
// for array type
for (Type complexType : Lists.newArrayList(Type.ARRAY, Type.MAP, Type.GENERIC_STRUCT)) {
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NULL, null,
Lists.newArrayList(complexType), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));

// for array type
for (Type complexType : Lists.newArrayList(Type.ARRAY, Type.MAP, Type.GENERIC_STRUCT)) {
Comment thread
amorynan marked this conversation as resolved.
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NULL, null,
Lists.newArrayList(complexType), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));

functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NOT_NULL,
null, Lists.newArrayList(complexType), Type.BOOLEAN,
NullableMode.ALWAYS_NOT_NULLABLE));
}

functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NOT_NULL, null,
Lists.newArrayList(complexType), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));
}
}

Expand Down
7 changes: 7 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import org.apache.doris.common.AnalysisException;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;

import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -95,6 +97,11 @@ public String getStringValueInFe() {
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.STRUCT_LITERAL;
((StructType) type).getFields().forEach(v -> msg.setChildType(v.getType().getPrimitiveType().toThrift()));
Comment thread
amorynan marked this conversation as resolved.
TTypeDesc container = new TTypeDesc();
container.setTypes(new ArrayList<TTypeNode>());
type.toThrift(container);
msg.setType(container);
}

@Override
Expand Down
42 changes: 35 additions & 7 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,16 +386,31 @@ public Function specializeTemplateFunction(Function templateFunction, Function r
throw new TypeException(templateFunction
+ " is not support for template since it's not a ScalarFunction");
}
Type[] args = specializedFunction.getArgs();
ArrayList<Type> args = new ArrayList<>();
Collections.addAll(args, specializedFunction.getArgs());
Map<String, Type> specializedTypeMap = Maps.newHashMap();
boolean enableDecimal256 = SessionVariable.getEnableDecimal256();
for (int i = 0; i < args.length; i++) {
if (args[i].hasTemplateType()) {
int i = 0;
for (; i < args.size(); i++) {
if (args.get(i).hasTemplateType()) {
hasTemplateType = true;
args[i] = args[i].specializeTemplateType(requestFunction.getArgs()[i], specializedTypeMap, false,
enableDecimal256);
// if args[i] is template type, and requestFunction.getArgs()[i] NULL_TYPE, we need call function
// deduce to get the specific type
Type deduceType = requestFunction.getArgs()[i];
if (requestFunction.getArgs()[i].isNull()
|| (requestFunction.getArgs()[i] instanceof ArrayType
Comment thread
xiaokang marked this conversation as resolved.
&& ((ArrayType) requestFunction.getArgs()[i]).getItemType().isNull())
&& FunctionTypeDeducers.DEDUCERS.containsKey(specializedFunction.functionName())) {
deduceType = FunctionTypeDeducers.deduce(specializedFunction.functionName(), i, requestFunction.getArgs());
args.set(i, args.get(i).specializeTemplateType(deduceType == null ? requestFunction.getArgs()[i]
: deduceType, specializedTypeMap, false, enableDecimal256));
} else {
args.set(i, args.get(i).specializeTemplateType(requestFunction.getArgs()[i],
specializedTypeMap, false, enableDecimal256));
}
}
}
specializedFunction.setArgs(args);
if (specializedFunction.getReturnType().hasTemplateType()) {
hasTemplateType = true;
specializedFunction.setReturnType(
Expand Down Expand Up @@ -426,7 +441,7 @@ public Function resolveInferenceFunction(Function inferenceFunction, Function re
newTypes[i] = inputType;
}
}
Type newRetType = FunctionTypeDeducers.deduce(inferenceFunction.functionName(), newTypes);
Type newRetType = FunctionTypeDeducers.deduce(inferenceFunction.functionName(), 0, newTypes);
if (newRetType != null && inferenceFunction instanceof ScalarFunction) {
ScalarFunction f = (ScalarFunction) inferenceFunction;
return new ScalarFunction(f.getFunctionName(), Lists.newArrayList(newTypes), newRetType, f.hasVarArgs(),
Expand All @@ -448,7 +463,20 @@ public static boolean isCastMatchAllowed(Function desc, Function candicate) {
final Type[] candicateArgTypes = candicate.getArgs();
if (!(descArgTypes[0] instanceof ScalarType)
|| !(candicateArgTypes[0] instanceof ScalarType)) {
if (candicateArgTypes[0] instanceof ArrayType || candicateArgTypes[0] instanceof MapType) {
if (candicateArgTypes[0] instanceof ArrayType) {
// match is exactly type. but for null type , with in array|map elem can not return true, because for
// be will make null_type to uint8
Comment thread
amorynan marked this conversation as resolved.
// so here meet null_type just make true as allowed, descArgTypes[0]).getItemType().isNull() is for
// empty literal like: []|{}
if (descArgTypes[0] instanceof ArrayType && ((ArrayType) descArgTypes[0]).getItemType().isNull()) {
return true;
}
return descArgTypes[0].matchesType(candicateArgTypes[0]);
} else if (candicateArgTypes[0] instanceof MapType) {
if (descArgTypes[0] instanceof MapType && ((MapType) descArgTypes[0]).getKeyType().isNull()
&& ((MapType) descArgTypes[0]).getValueType().isNull()) {
return true;
}
return descArgTypes[0].matchesType(candicateArgTypes[0]);
}
return false;
Expand Down
34 changes: 28 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,26 +23,48 @@
import java.util.List;

public class FunctionTypeDeducers {

public interface TypeDeducer {
public Type deduce(Type[] args);
public Type deduce(int argIdx, Type[] args);
}

public static final ImmutableMap<String, TypeDeducer> DEDUCERS = ImmutableMap.<String, TypeDeducer>builder()
.put("named_struct", new NamedStructDeducer())
.put("struct_element", new StructElementDeducer())
.put("array_contains", new ArrayElemFuncDeducer())
.put("array_pushback", new ArrayElemFuncDeducer())
.put("element_at", new ArrayElemFuncDeducer())
.build();

public static Type deduce(String fnName, Type[] args) {
public static Type deduce(String fnName, int argIdx, Type[] args) {
if (DEDUCERS.containsKey(fnName)) {
return DEDUCERS.get(fnName).deduce(args);
return DEDUCERS.get(fnName).deduce(argIdx, args);
}
return null;
}

public static class ArrayElemFuncDeducer implements TypeDeducer {
Comment thread
amorynan marked this conversation as resolved.
@Override
public Type deduce(int argIdx, Type[] args) {
if (args.length >= 2) {
if (argIdx == 0) {
// first args should only to be array or null
return args[0] instanceof ArrayType || args[0].isNull() ? new ArrayType(args[1]) : args[0];
} else if (args[0].isNull()) {
// first arg is null, later element is not contains
return args[argIdx];
} else if (Type.isImplicitlyCastable(args[argIdx], ((ArrayType) args[0]).getItemType(), false, true)) {
return args[argIdx];
} else {
return null;
}
}
return null;
}
}

public static class NamedStructDeducer implements TypeDeducer {
@Override
public Type deduce(Type[] args) {
public Type deduce(int argIdx, Type[] args) {
List<Type> evenArgs = Lists.newArrayList();
for (int i = 0; i < args.length; i++) {
if ((i & 1) == 1) {
Expand All @@ -55,7 +77,7 @@ public Type deduce(Type[] args) {

public static class StructElementDeducer implements TypeDeducer {
@Override
public Type deduce(Type[] args) {
public Type deduce(int argIdx, Type[] args) {
if (args[0] instanceof StructType) {
return Type.ANY_ELEMENT_TYPE;
}
Expand Down
27 changes: 7 additions & 20 deletions gensrc/script/doris_builtins_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,24 +165,7 @@
[['arrays_overlap'], 'BOOLEAN', ['ARRAY_VARCHAR', 'ARRAY_VARCHAR'], 'ALWAYS_NULLABLE'],
[['arrays_overlap'], 'BOOLEAN', ['ARRAY_STRING', 'ARRAY_STRING'], 'ALWAYS_NULLABLE'],

[['array_contains'], 'BOOLEAN', ['ARRAY_BOOLEAN', 'BOOLEAN'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_TINYINT', 'TINYINT'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_SMALLINT', 'SMALLINT'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_INT', 'INT'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_BIGINT', 'BIGINT'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_LARGEINT', 'LARGEINT'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DATETIME', 'DATETIME'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DATE', 'DATE'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DATETIMEV2', 'DATETIMEV2'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DATEV2', 'DATEV2'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_FLOAT', 'FLOAT'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DOUBLE', 'DOUBLE'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DECIMALV2', 'DECIMALV2'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DECIMAL32', 'DECIMAL32'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DECIMAL64', 'DECIMAL64'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DECIMAL128', 'DECIMAL128'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_VARCHAR', 'VARCHAR'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_STRING', 'STRING'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY<T>', 'T'], 'CUSTOM', ['T']],

[['array_cum_sum'], 'ARRAY_BIGINT', ['ARRAY_TINYINT'], ''],
[['array_cum_sum'], 'ARRAY_BIGINT', ['ARRAY_SMALLINT'], ''],
Expand Down Expand Up @@ -272,7 +255,7 @@
[['array_position'], 'BIGINT', ['ARRAY_VARCHAR', 'VARCHAR'], 'CUSTOM'],
[['array_position'], 'BIGINT', ['ARRAY_STRING', 'STRING'], 'CUSTOM'],

[['cardinality', 'size', 'array_size'], 'BIGINT', ['ARRAY'], ''],
[['cardinality', 'size', 'array_size'], 'BIGINT', ['ARRAY<T>'], '', ['T']],
[['array_distinct'], 'ARRAY_BOOLEAN', ['ARRAY_BOOLEAN'], ''],
[['array_distinct'], 'ARRAY_TINYINT', ['ARRAY_TINYINT'], ''],
[['array_distinct'], 'ARRAY_SMALLINT', ['ARRAY_SMALLINT'], ''],
Expand Down Expand Up @@ -772,6 +755,7 @@
[['array_pushfront'], 'ARRAY_VARCHAR', ['ARRAY_VARCHAR', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['array_pushfront'], 'ARRAY_STRING', ['ARRAY_STRING', 'STRING'], 'ALWAYS_NULLABLE'],

[['array_pushback'], 'ARRAY<T>', ['ARRAY<T>', 'T'], 'ALWAYS_NULLABLE', ['T']],
[['array_pushback'], 'ARRAY_BOOLEAN', ['ARRAY_BOOLEAN', 'BOOLEAN'], 'ALWAYS_NULLABLE'],
[['array_pushback'], 'ARRAY_TINYINT', ['ARRAY_TINYINT', 'TINYINT'], 'ALWAYS_NULLABLE'],
[['array_pushback'], 'ARRAY_SMALLINT', ['ARRAY_SMALLINT', 'SMALLINT'], 'ALWAYS_NULLABLE'],
Expand Down Expand Up @@ -833,7 +817,7 @@
[['array_range'], 'ARRAY_INT', ['INT', 'INT'], 'ALWAYS_NULLABLE'],
[['array_range'], 'ARRAY_INT', ['INT', 'INT', 'INT'], 'ALWAYS_NULLABLE'],

[['array_zip'], 'ARRAY', ['ARRAY', '...'], ''],
[['array_zip'], 'ARRAY', ['ARRAY<T>', '...'], '', ['T']],


# reverse function for string builtin
Expand Down Expand Up @@ -1469,6 +1453,9 @@
[['nullif'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['nullif'], 'STRING', ['STRING', 'STRING'], 'ALWAYS_NULLABLE'],

[['is_null_pred'], "BOOLEAN", ['T', '...'], 'ALWAYS_NULLABLE', ['T']],
[['is_not_null_pred'], "BOOLEAN", ['T', '...'], 'ALWAYS_NULLABLE', ['T']],

[['ifnull', 'nvl'], 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], 'CUSTOM'],
[['ifnull', 'nvl'], 'TINYINT', ['TINYINT', 'TINYINT'], 'CUSTOM'],
[['ifnull', 'nvl'], 'SMALLINT', ['SMALLINT', 'SMALLINT'], 'CUSTOM'],
Expand Down
Loading