Skip to content

Commit 1496b4c

Browse files
committed
fix: modify use case
1 parent f093eba commit 1496b4c

File tree

2 files changed

+19
-21
lines changed

2 files changed

+19
-21
lines changed

amoro-format-iceberg/src/main/java/org/apache/amoro/utils/ExpressionUtil.java

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -164,15 +164,13 @@ private static Expression convertSparkExpressionToIceberg(
164164
* only support `max` function, like `max(pt,1)` for partition, `max(day(ts),5)` for hidden
165165
* partition
166166
*
167-
* @param expressLeft
168167
* @param func
169168
* @param partitionSpec
170169
* @param maxPartitionFunc
171170
* @return
172171
*/
173172
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"})
174173
private static Expression applyFunc(
175-
String expressLeft,
176174
net.sf.jsqlparser.expression.Function func,
177175
PartitionSpec partitionSpec,
178176
BiFunction<String, Integer, List<String>> maxPartitionFunc) {
@@ -186,13 +184,17 @@ private static Expression applyFunc(
186184
return Expressions.alwaysTrue();
187185
}
188186
int topN = Integer.parseInt(func.getParameters().get(1) + "");
189-
List<String> values = maxPartitionFunc.apply(expressLeft, topN);
190-
LOG.info("use max partition value: {} for column: {}", String.join(",", values), expressLeft);
187+
191188
// like max(transformer(pt),1)
192189
if (func.getParameters().get(0) instanceof net.sf.jsqlparser.expression.Function) {
193190
net.sf.jsqlparser.expression.Function transformerFunc =
194191
(net.sf.jsqlparser.expression.Function) func.getParameters().get(0);
195192
String ptName = transformerFunc.getParameters().get(0).toString();
193+
List<String> values = maxPartitionFunc.apply(ptName, topN);
194+
LOG.info(
195+
"use max partition value: {} for column: {}",
196+
String.join(",", values),
197+
func.getParameters().get(0).toString());
196198
PartitionField field =
197199
partitionSpec
198200
.getFieldsBySourceId(partitionSpec.schema().findField(ptName).fieldId())
@@ -203,6 +205,8 @@ private static Expression applyFunc(
203205
} else {
204206
// like max(pt,1), and ptName should equal to 'left'
205207
String ptName = func.getParameters().get(0).toString();
208+
List<String> values = maxPartitionFunc.apply(ptName, topN);
209+
LOG.info("use partition value: {} for column: {}", String.join(",", values), ptName);
206210
return Expressions.in(ptName, values.toArray(new String[0]));
207211
}
208212
} else {
@@ -225,18 +229,7 @@ private static Expression convertSparkExpressionToIceberg(
225229
} else if (whereExpr instanceof EqualsTo) {
226230
EqualsTo eq = (EqualsTo) whereExpr;
227231
Types.NestedField column = getColumn(eq.getLeftExpression(), tableColumns);
228-
// support function expression, like `max(pt,1)` for partition, `max(day(ts),5)` for hidden
229-
// partition
230-
if (eq.getRightExpression() instanceof net.sf.jsqlparser.expression.Function) {
231-
return applyFunc(
232-
column.name(),
233-
(net.sf.jsqlparser.expression.Function) eq.getRightExpression(),
234-
partitionSpec,
235-
maxPartitionFunc);
236-
} else {
237-
return Expressions.equal(column.name(), getValue(eq.getRightExpression(), column));
238-
}
239-
232+
return Expressions.equal(column.name(), getValue(eq.getRightExpression(), column));
240233
} else if (whereExpr instanceof NotEqualsTo) {
241234
NotEqualsTo ne = (NotEqualsTo) whereExpr;
242235
Types.NestedField column = getColumn(ne.getLeftExpression(), tableColumns);
@@ -287,6 +280,11 @@ private static Expression convertSparkExpressionToIceberg(
287280
return Expressions.or(
288281
convertSparkExpressionToIceberg(or.getLeftExpression(), tableColumns, partitionSpec),
289282
convertSparkExpressionToIceberg(or.getRightExpression(), tableColumns, partitionSpec));
283+
} else if (whereExpr instanceof net.sf.jsqlparser.expression.Function) {
284+
// support function expression, like `max(pt,1)` for partition, `max(day(ts),5)` for hidden
285+
// partition
286+
return applyFunc(
287+
(net.sf.jsqlparser.expression.Function) whereExpr, partitionSpec, maxPartitionFunc);
290288
}
291289
throw new UnsupportedOperationException("Unsupported expression: " + whereExpr);
292290
}

amoro-format-iceberg/src/test/java/org/apache/amoro/utils/TestExpressionUtil.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,15 @@ public void testConvertSqlToIcebergExpression() {
5454

5555
assertEqualExpressions(
5656
Expressions.in(Expressions.day("ts"), "2023-01-01", "2023-02-02"),
57-
convertSqlFilterToIcebergExpression("pt=max(day(ts),5)", fields, partitionSpec, func));
57+
convertSqlFilterToIcebergExpression("max(day(ts),5)", fields, partitionSpec, func));
5858

5959
assertEqualExpressions(
60-
Expressions.alwaysTrue(),
61-
convertSqlFilterToIcebergExpression("pt=max(day(ts),5, 10)", fields, partitionSpec, func));
60+
Expressions.in("pt", "2023-01-01", "2023-02-02"),
61+
convertSqlFilterToIcebergExpression("max(pt,5)", fields, partitionSpec, func));
6262

6363
assertEqualExpressions(
64-
Expressions.in("pt", "2023-01-01", "2023-02-02"),
65-
convertSqlFilterToIcebergExpression("pt=max(pt,5)", fields, partitionSpec, func));
64+
Expressions.alwaysTrue(),
65+
convertSqlFilterToIcebergExpression("max(day(ts),5, 10)", fields, partitionSpec, func));
6666

6767
assertEqualExpressions(
6868
Expressions.isNull("column_a"),

0 commit comments

Comments
 (0)