Skip to content

[branch-2.1](partition) Support use Auto and Dynamic partition at the same time (#39580) #40649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 1 commit into from
Sep 11, 2024
Merged

[branch-2.1](partition) Support use Auto and Dynamic partition at the same time (#39580) #40649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 33 additions & 8 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@

package org.apache.doris.common.util;

import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.TimestampArithmeticExpr.TimeUnit;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
Expand Down Expand Up @@ -127,13 +130,14 @@ private static int checkStart(String start) throws DdlException {
return DynamicPartitionProperty.MIN_START_OFFSET;
}

private static int checkEnd(String end) throws DdlException {
private static int checkEnd(String end, boolean enableAutoPartition) throws DdlException {
if (Strings.isNullOrEmpty(end)) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_EMPTY);
}
try {
int endInt = Integer.parseInt(end);
if (endInt <= 0) {
// with auto partition sometime we dont like to create future partition by dynamic partition.
if (endInt < 0 || endInt == 0 && !enableAutoPartition) {
ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_END_ZERO, end);
}
return endInt;
Expand Down Expand Up @@ -503,6 +507,25 @@ public static void registerOrRemoveDynamicPartitionTable(long dbId, OlapTable ol
}
}

public static void partitionIntervalCompatible(String dynamicUnit, ArrayList<Expr> autoExprs)
throws AnalysisException {
if (autoExprs == null) {
return;
}
for (Expr autoExpr : autoExprs) {
Expr func = (FunctionCallExpr) autoExpr;
for (Expr child : func.getChildren()) {
if (child instanceof LiteralExpr) {
String autoUnit = ((LiteralExpr) child).getStringValue();
if (!dynamicUnit.equalsIgnoreCase(autoUnit)) {
throw new AnalysisException("If support auto partition and dynamic partition at same time, "
+ "they must have the same interval unit.");
}
}
}
}
}

// Analyze all properties to check their validation
public static Map<String, String> analyzeDynamicPartition(Map<String, String> properties,
OlapTable olapTable, Database db) throws UserException {
Expand All @@ -511,6 +534,12 @@ public static Map<String, String> analyzeDynamicPartition(Map<String, String> pr
if (properties.containsKey(DynamicPartitionProperty.TIME_UNIT)) {
String timeUnitValue = properties.get(DynamicPartitionProperty.TIME_UNIT);
checkTimeUnit(timeUnitValue, olapTable.getPartitionInfo());

// if both enabled, must use same interval.
if (olapTable.getPartitionInfo().enableAutomaticPartition()) {
partitionIntervalCompatible(timeUnitValue, olapTable.getPartitionInfo().getPartitionExprs());
}

properties.remove(DynamicPartitionProperty.TIME_UNIT);
analyzedProperties.put(DynamicPartitionProperty.TIME_UNIT, timeUnitValue);
}
Expand All @@ -535,11 +564,7 @@ public static Map<String, String> analyzeDynamicPartition(Map<String, String> pr
analyzedProperties.put(DynamicPartitionProperty.ENABLE, enableValue);
}

if (Boolean.parseBoolean(analyzedProperties.getOrDefault(DynamicPartitionProperty.ENABLE, "true"))
&& olapTable.getPartitionInfo().enableAutomaticPartition()) {
throw new AnalysisException(
"Can't use Dynamic Partition and Auto Partition at the same time");
}
boolean enableAutoPartition = olapTable.getPartitionInfo().enableAutomaticPartition();

// If dynamic property "start" is not specified, use Integer.MIN_VALUE as default
int start = DynamicPartitionProperty.MIN_START_OFFSET;
Expand All @@ -554,7 +579,7 @@ public static Map<String, String> analyzeDynamicPartition(Map<String, String> pr
boolean hasEnd = false;
if (properties.containsKey(DynamicPartitionProperty.END)) {
String endValue = properties.get(DynamicPartitionProperty.END);
end = checkEnd(endValue);
end = checkEnd(endValue, enableAutoPartition);
properties.remove(DynamicPartitionProperty.END);
analyzedProperties.put(DynamicPartitionProperty.END, endValue);
hasEnd = true;
Expand Down
6 changes: 4 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2859,8 +2859,10 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx
.getDynamicPartitionProperty();
if (dynamicProperty.isExist() && dynamicProperty.getEnable()
&& partitionDesc.isAutoCreatePartitions()) {
throw new AnalysisException(
"Can't use Dynamic Partition and Auto Partition at the same time");
String dynamicUnit = dynamicProperty.getTimeUnit();
ArrayList<Expr> autoExprs = partitionDesc.getPartitionExprs();
// check same interval. fail will leading to AnalysisException
DynamicPartitionUtil.partitionIntervalCompatible(dynamicUnit, autoExprs);
}
} catch (AnalysisException e) {
throw new DdlException(e.getMessage());
Expand Down
10 changes: 6 additions & 4 deletions regression-test/data/partition_p0/auto_partition/test_auto_partition_behavior.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,10 +102,7 @@ xxX 3
2013-12-12T00:00 2013-12-12T00:00
2020-12-12T00:00 2020-12-12T12:12:12.123456

-- !sql_overwrite1 --
Yyy

-- !sql_overwrite2 --
-- !sql_overwrite --
Xxx

-- !sql_non_order1 --
Expand All @@ -117,3 +114,8 @@ Xxx
-- !sql_non_order3 --
3 2013-12-12T00:00

-- !sql_dynamic_auto --
2024-01-01T00:00
2900-01-01T00:00
3000-01-01T00:00

Loading