Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HIVE-28302: Let SUM UDF return NULL when all rows have non-numeric texts #5283

Merged
merged 2 commits into from
Jul 22, 2024
Merged

HIVE-28302: Let SUM UDF return NULL when all rows have non-numeric texts #5283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5337685
HIVE-28302: Let SUM UDF return NULL when all rows have non-numeric texts
okumin Jun 6, 2024
8b8136e
Unround expressions in cbo_aggregate_reduce_functions_rule.q
okumin Jul 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,9 +288,9 @@ public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveExcep
assert (parameters.length == 1);
try {
if (isEligibleValue((SumHiveDecimalWritableAgg) agg, parameters[0])) {
((SumHiveDecimalWritableAgg)agg).nonNullCount++;
((SumHiveDecimalWritableAgg)agg).sum.mutateAdd(
PrimitiveObjectInspectorUtils.getHiveDecimal(parameters[0], inputOI));
((SumHiveDecimalWritableAgg)agg).nonNullCount++;
}
} catch (NumberFormatException e) {
if (!warned) {
Expand Down Expand Up @@ -440,8 +440,8 @@ public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveExcep
assert (parameters.length == 1);
try {
if (isEligibleValue((SumDoubleAgg) agg, parameters[0])) {
((SumDoubleAgg)agg).nonNullCount++;
((SumDoubleAgg)agg).sum += PrimitiveObjectInspectorUtils.getDouble(parameters[0], inputOI);
((SumDoubleAgg)agg).nonNullCount++;
}
} catch (NumberFormatException e) {
if (!warned) {
Expand Down Expand Up @@ -578,8 +578,8 @@ public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveExcep
assert (parameters.length == 1);
try {
if (isEligibleValue((SumLongAgg) agg, parameters[0])) {
((SumLongAgg)agg).nonNullCount++;
((SumLongAgg)agg).sum += PrimitiveObjectInspectorUtils.getLong(parameters[0], inputOI);
((SumLongAgg)agg).nonNullCount++;
}
} catch (NumberFormatException e) {
if (!warned) {
Expand Down
282 changes: 141 additions & 141 deletions ql/src/test/queries/clientpositive/cbo_aggregate_reduce_functions_rule.q
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,55 +11,55 @@ INSERT INTO test VALUES

EXPLAIN CBO
SELECT
ROUND(`$SUM0`(c_numeric), 3),
ROUND(`$SUM0`(CAST(c_numeric AS DOUBLE)), 3),
ROUND(`$SUM0`(c_non_numeric), 3),
ROUND(`$SUM0`(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(`$SUM0`(c_mix), 3),
ROUND(`$SUM0`(CAST(c_mix AS DOUBLE)), 3),

ROUND(AVG(c_numeric), 3),
ROUND(AVG(CAST(c_numeric AS DOUBLE)), 3),
ROUND(AVG(c_non_numeric), 3),
ROUND(AVG(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(AVG(c_mix), 3),
ROUND(AVG(CAST(c_mix AS DOUBLE)), 3),

ROUND(STDDEV_POP(c_numeric), 3),
ROUND(STDDEV_POP(CAST(c_numeric AS DOUBLE)), 3),
ROUND(STDDEV_POP(c_non_numeric), 3),
ROUND(STDDEV_POP(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(STDDEV_POP(c_mix), 3),
ROUND(STDDEV_POP(CAST(c_mix AS DOUBLE)), 3),

ROUND(STDDEV_SAMP(c_numeric), 3),
ROUND(STDDEV_SAMP(CAST(c_numeric AS DOUBLE)), 3),
ROUND(STDDEV_SAMP(c_non_numeric), 3),
ROUND(STDDEV_SAMP(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(STDDEV_SAMP(c_mix), 3),
ROUND(STDDEV_SAMP(CAST(c_mix AS DOUBLE)), 3),

ROUND(VAR_POP(c_numeric), 3),
ROUND(VAR_POP(CAST(c_numeric AS DOUBLE)), 3),
ROUND(VAR_POP(c_non_numeric), 3),
ROUND(VAR_POP(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(VAR_POP(c_mix), 3),
ROUND(VAR_POP(CAST(c_mix AS DOUBLE)), 3),

ROUND(VAR_SAMP(c_numeric), 3),
ROUND(VAR_SAMP(CAST(c_numeric AS DOUBLE)), 3),
ROUND(VAR_SAMP(c_non_numeric), 3),
ROUND(VAR_SAMP(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(VAR_SAMP(c_mix), 3),
ROUND(VAR_SAMP(CAST(c_mix AS DOUBLE)), 3),
`$SUM0`(c_numeric),
`$SUM0`(CAST(c_numeric AS DOUBLE)),
`$SUM0`(c_non_numeric),
`$SUM0`(CAST(c_non_numeric AS DOUBLE)),
`$SUM0`(c_mix),
`$SUM0`(CAST(c_mix AS DOUBLE)),

AVG(c_numeric),
AVG(CAST(c_numeric AS DOUBLE)),
AVG(c_non_numeric),
AVG(CAST(c_non_numeric AS DOUBLE)),
AVG(c_mix),
AVG(CAST(c_mix AS DOUBLE)),

STDDEV_POP(c_numeric),
STDDEV_POP(CAST(c_numeric AS DOUBLE)),
STDDEV_POP(c_non_numeric),
STDDEV_POP(CAST(c_non_numeric AS DOUBLE)),
STDDEV_POP(c_mix),
STDDEV_POP(CAST(c_mix AS DOUBLE)),

STDDEV_SAMP(c_numeric),
STDDEV_SAMP(CAST(c_numeric AS DOUBLE)),
STDDEV_SAMP(c_non_numeric),
STDDEV_SAMP(CAST(c_non_numeric AS DOUBLE)),
STDDEV_SAMP(c_mix),
STDDEV_SAMP(CAST(c_mix AS DOUBLE)),

VAR_POP(c_numeric),
VAR_POP(CAST(c_numeric AS DOUBLE)),
VAR_POP(c_non_numeric),
VAR_POP(CAST(c_non_numeric AS DOUBLE)),
VAR_POP(c_mix),
VAR_POP(CAST(c_mix AS DOUBLE)),

VAR_SAMP(c_numeric),
VAR_SAMP(CAST(c_numeric AS DOUBLE)),
VAR_SAMP(c_non_numeric),
VAR_SAMP(CAST(c_non_numeric AS DOUBLE)),
VAR_SAMP(c_mix),
VAR_SAMP(CAST(c_mix AS DOUBLE)),

-- SUM and COUNT are not converted but used in the transformations
ROUND(SUM(c_numeric), 3),
ROUND(SUM(CAST(c_numeric AS DOUBLE)), 3),
ROUND(SUM(c_non_numeric), 3),
ROUND(SUM(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(SUM(c_mix), 3),
ROUND(SUM(CAST(c_mix AS DOUBLE)), 3),
SUM(c_numeric),
SUM(CAST(c_numeric AS DOUBLE)),
SUM(c_non_numeric),
SUM(CAST(c_non_numeric AS DOUBLE)),
SUM(c_mix),
SUM(CAST(c_mix AS DOUBLE)),

COUNT(c_numeric),
COUNT(CAST(c_numeric AS DOUBLE)),
Expand All @@ -70,55 +70,55 @@ SELECT
FROM test;

SELECT
ROUND(`$SUM0`(c_numeric), 3),
ROUND(`$SUM0`(CAST(c_numeric AS DOUBLE)), 3),
ROUND(`$SUM0`(c_non_numeric), 3),
ROUND(`$SUM0`(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(`$SUM0`(c_mix), 3),
ROUND(`$SUM0`(CAST(c_mix AS DOUBLE)), 3),

ROUND(AVG(c_numeric), 3),
ROUND(AVG(CAST(c_numeric AS DOUBLE)), 3),
ROUND(AVG(c_non_numeric), 3),
ROUND(AVG(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(AVG(c_mix), 3),
ROUND(AVG(CAST(c_mix AS DOUBLE)), 3),

ROUND(STDDEV_POP(c_numeric), 3),
ROUND(STDDEV_POP(CAST(c_numeric AS DOUBLE)), 3),
ROUND(STDDEV_POP(c_non_numeric), 3),
ROUND(STDDEV_POP(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(STDDEV_POP(c_mix), 3),
ROUND(STDDEV_POP(CAST(c_mix AS DOUBLE)), 3),

ROUND(STDDEV_SAMP(c_numeric), 3),
ROUND(STDDEV_SAMP(CAST(c_numeric AS DOUBLE)), 3),
ROUND(STDDEV_SAMP(c_non_numeric), 3),
ROUND(STDDEV_SAMP(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(STDDEV_SAMP(c_mix), 3),
ROUND(STDDEV_SAMP(CAST(c_mix AS DOUBLE)), 3),

ROUND(VAR_POP(c_numeric), 3),
ROUND(VAR_POP(CAST(c_numeric AS DOUBLE)), 3),
ROUND(VAR_POP(c_non_numeric), 3),
ROUND(VAR_POP(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(VAR_POP(c_mix), 3),
ROUND(VAR_POP(CAST(c_mix AS DOUBLE)), 3),

ROUND(VAR_SAMP(c_numeric), 3),
ROUND(VAR_SAMP(CAST(c_numeric AS DOUBLE)), 3),
ROUND(VAR_SAMP(c_non_numeric), 3),
ROUND(VAR_SAMP(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(VAR_SAMP(c_mix), 3),
ROUND(VAR_SAMP(CAST(c_mix AS DOUBLE)), 3),
`$SUM0`(c_numeric),
`$SUM0`(CAST(c_numeric AS DOUBLE)),
`$SUM0`(c_non_numeric),
`$SUM0`(CAST(c_non_numeric AS DOUBLE)),
`$SUM0`(c_mix),
`$SUM0`(CAST(c_mix AS DOUBLE)),

AVG(c_numeric),
AVG(CAST(c_numeric AS DOUBLE)),
AVG(c_non_numeric),
AVG(CAST(c_non_numeric AS DOUBLE)),
AVG(c_mix),
AVG(CAST(c_mix AS DOUBLE)),

STDDEV_POP(c_numeric),
STDDEV_POP(CAST(c_numeric AS DOUBLE)),
STDDEV_POP(c_non_numeric),
STDDEV_POP(CAST(c_non_numeric AS DOUBLE)),
STDDEV_POP(c_mix),
STDDEV_POP(CAST(c_mix AS DOUBLE)),

STDDEV_SAMP(c_numeric),
STDDEV_SAMP(CAST(c_numeric AS DOUBLE)),
STDDEV_SAMP(c_non_numeric),
STDDEV_SAMP(CAST(c_non_numeric AS DOUBLE)),
STDDEV_SAMP(c_mix),
STDDEV_SAMP(CAST(c_mix AS DOUBLE)),

VAR_POP(c_numeric),
VAR_POP(CAST(c_numeric AS DOUBLE)),
VAR_POP(c_non_numeric),
VAR_POP(CAST(c_non_numeric AS DOUBLE)),
VAR_POP(c_mix),
VAR_POP(CAST(c_mix AS DOUBLE)),

VAR_SAMP(c_numeric),
VAR_SAMP(CAST(c_numeric AS DOUBLE)),
VAR_SAMP(c_non_numeric),
VAR_SAMP(CAST(c_non_numeric AS DOUBLE)),
VAR_SAMP(c_mix),
VAR_SAMP(CAST(c_mix AS DOUBLE)),

-- SUM and COUNT are not converted but used in the transformations
ROUND(SUM(c_numeric), 3),
ROUND(SUM(CAST(c_numeric AS DOUBLE)), 3),
ROUND(SUM(c_non_numeric), 3),
ROUND(SUM(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(SUM(c_mix), 3),
ROUND(SUM(CAST(c_mix AS DOUBLE)), 3),
SUM(c_numeric),
SUM(CAST(c_numeric AS DOUBLE)),
SUM(c_non_numeric),
SUM(CAST(c_non_numeric AS DOUBLE)),
SUM(c_mix),
SUM(CAST(c_mix AS DOUBLE)),

COUNT(c_numeric),
COUNT(CAST(c_numeric AS DOUBLE)),
Expand All @@ -132,55 +132,55 @@ FROM test;
SET hive.cbo.enable=false;

SELECT
ROUND(`$SUM0`(c_numeric), 3),
ROUND(`$SUM0`(CAST(c_numeric AS DOUBLE)), 3),
ROUND(`$SUM0`(c_non_numeric), 3),
ROUND(`$SUM0`(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(`$SUM0`(c_mix), 3),
ROUND(`$SUM0`(CAST(c_mix AS DOUBLE)), 3),

ROUND(AVG(c_numeric), 3),
ROUND(AVG(CAST(c_numeric AS DOUBLE)), 3),
ROUND(AVG(c_non_numeric), 3),
ROUND(AVG(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(AVG(c_mix), 3),
ROUND(AVG(CAST(c_mix AS DOUBLE)), 3),

ROUND(STDDEV_POP(c_numeric), 3),
ROUND(STDDEV_POP(CAST(c_numeric AS DOUBLE)), 3),
ROUND(STDDEV_POP(c_non_numeric), 3),
ROUND(STDDEV_POP(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(STDDEV_POP(c_mix), 3),
ROUND(STDDEV_POP(CAST(c_mix AS DOUBLE)), 3),

ROUND(STDDEV_SAMP(c_numeric), 3),
ROUND(STDDEV_SAMP(CAST(c_numeric AS DOUBLE)), 3),
ROUND(STDDEV_SAMP(c_non_numeric), 3),
ROUND(STDDEV_SAMP(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(STDDEV_SAMP(c_mix), 3),
ROUND(STDDEV_SAMP(CAST(c_mix AS DOUBLE)), 3),

ROUND(VAR_POP(c_numeric), 3),
ROUND(VAR_POP(CAST(c_numeric AS DOUBLE)), 3),
ROUND(VAR_POP(c_non_numeric), 3),
ROUND(VAR_POP(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(VAR_POP(c_mix), 3),
ROUND(VAR_POP(CAST(c_mix AS DOUBLE)), 3),

ROUND(VAR_SAMP(c_numeric), 3),
ROUND(VAR_SAMP(CAST(c_numeric AS DOUBLE)), 3),
ROUND(VAR_SAMP(c_non_numeric), 3),
ROUND(VAR_SAMP(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(VAR_SAMP(c_mix), 3),
ROUND(VAR_SAMP(CAST(c_mix AS DOUBLE)), 3),
`$SUM0`(c_numeric),
`$SUM0`(CAST(c_numeric AS DOUBLE)),
`$SUM0`(c_non_numeric),
`$SUM0`(CAST(c_non_numeric AS DOUBLE)),
`$SUM0`(c_mix),
`$SUM0`(CAST(c_mix AS DOUBLE)),

AVG(c_numeric),
AVG(CAST(c_numeric AS DOUBLE)),
AVG(c_non_numeric),
AVG(CAST(c_non_numeric AS DOUBLE)),
AVG(c_mix),
AVG(CAST(c_mix AS DOUBLE)),

STDDEV_POP(c_numeric),
STDDEV_POP(CAST(c_numeric AS DOUBLE)),
STDDEV_POP(c_non_numeric),
STDDEV_POP(CAST(c_non_numeric AS DOUBLE)),
STDDEV_POP(c_mix),
STDDEV_POP(CAST(c_mix AS DOUBLE)),

STDDEV_SAMP(c_numeric),
STDDEV_SAMP(CAST(c_numeric AS DOUBLE)),
STDDEV_SAMP(c_non_numeric),
STDDEV_SAMP(CAST(c_non_numeric AS DOUBLE)),
STDDEV_SAMP(c_mix),
STDDEV_SAMP(CAST(c_mix AS DOUBLE)),

VAR_POP(c_numeric),
VAR_POP(CAST(c_numeric AS DOUBLE)),
VAR_POP(c_non_numeric),
VAR_POP(CAST(c_non_numeric AS DOUBLE)),
VAR_POP(c_mix),
VAR_POP(CAST(c_mix AS DOUBLE)),

VAR_SAMP(c_numeric),
VAR_SAMP(CAST(c_numeric AS DOUBLE)),
VAR_SAMP(c_non_numeric),
VAR_SAMP(CAST(c_non_numeric AS DOUBLE)),
VAR_SAMP(c_mix),
VAR_SAMP(CAST(c_mix AS DOUBLE)),

-- SUM and COUNT are not converted but used in the transformations
ROUND(SUM(c_numeric), 3),
ROUND(SUM(CAST(c_numeric AS DOUBLE)), 3),
ROUND(SUM(c_non_numeric), 3),
ROUND(SUM(CAST(c_non_numeric AS DOUBLE)), 3),
ROUND(SUM(c_mix), 3),
ROUND(SUM(CAST(c_mix AS DOUBLE)), 3),
SUM(c_numeric),
SUM(CAST(c_numeric AS DOUBLE)),
SUM(c_non_numeric),
SUM(CAST(c_non_numeric AS DOUBLE)),
SUM(c_mix),
SUM(CAST(c_mix AS DOUBLE)),

COUNT(c_numeric),
COUNT(CAST(c_numeric AS DOUBLE)),
Expand Down
Loading
Loading