Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

HIVE-28347: Make a UDAF 'collect_set' work with complex types, even when map-side aggregation is disabled. #5323

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+781 −26
Open

HIVE-28347: Make a UDAF 'collect_set' work with complex types, even when map-side aggregation is disabled. #5323

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5b1435c
HIVE-28347
JeongDaeKim Jun 25, 2024
ea79e1a
Remove duplicates
JeongDaeKim Jun 26, 2024
a145a2f
Update q.out file
JeongDaeKim Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 6 additions & 12 deletions ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMkCollectionEvaluator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,22 +62,16 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters)
super.init(m, parameters);
// init output object inspectors
// The output of a partial aggregation is a list
if (m == Mode.PARTIAL1) {
if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
inputOI = parameters[0];
return ObjectInspectorFactory.getStandardListObjectInspector(
ObjectInspectorUtils.getStandardObjectInspector(inputOI));
} else {
if (!(parameters[0] instanceof ListObjectInspector)) {
//no map aggregation.
inputOI = ObjectInspectorUtils.getStandardObjectInspector(parameters[0]);
return ObjectInspectorFactory.getStandardListObjectInspector(inputOI);
} else {
internalMergeOI = (ListObjectInspector) parameters[0];
inputOI = internalMergeOI.getListElementObjectInspector();
loi = (StandardListObjectInspector)
ObjectInspectorUtils.getStandardObjectInspector(internalMergeOI);
return loi;
}
internalMergeOI = (ListObjectInspector) parameters[0];
inputOI = internalMergeOI.getListElementObjectInspector();
loi = (StandardListObjectInspector)
ObjectInspectorUtils.getStandardObjectInspector(internalMergeOI);
return loi;
}
}

Expand Down
191 changes: 191 additions & 0 deletions ql/src/test/queries/clientpositive/udaf_collect_set_2.q
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,43 @@ LOAD DATA LOCAL INPATH "../../data/files/nested_orders.txt" INTO TABLE nested_or

-- 1.1 when field is primitive

set hive.map.aggr = true;

SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.d, "amount", o.amount)))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.d, "amount", o.amount)))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

-- cast decimal

SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.d, "amount", cast(o.amount as decimal(10,1)))))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.d, "amount", cast(o.amount as decimal(10,1)))))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;


SELECT c.id, sort_array(collect_set(struct(c.name, o.d, o.amount)))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(struct(c.name, o.d, o.amount)))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

set hive.map.aggr = false;

SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.d, "amount", o.amount)))
FROM customers c
INNER JOIN orders o
Expand Down Expand Up @@ -67,6 +104,8 @@ ON (c.id = o.cid) GROUP BY c.id;

-- 1.2 when field is map

set hive.map.aggr = true;

SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.d, "sub", o.sub)))
FROM customers c
INNER JOIN nested_orders o
Expand All @@ -87,9 +126,54 @@ FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

set hive.map.aggr = false;

SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.d, "sub", o.sub)))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.d, "sub", o.sub)))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_set(struct(c.name, o.d, o.sub)))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(struct(c.name, o.d, o.sub)))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

-- 1.3 when field is list

set hive.map.aggr = true;

SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.d, "sub", map_values(o.sub))))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(named_struct("name", c.name, "date", o.d, "sub", map_values(o.sub))))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_set(struct(c.name, o.d, map_values(o.sub))))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(struct(c.name, o.d, map_values(o.sub))))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

set hive.map.aggr = false;

SELECT c.id, sort_array(collect_set(named_struct("name", c.name, "date", o.d, "sub", map_values(o.sub))))
FROM customers c
INNER JOIN nested_orders o
Expand All @@ -115,6 +199,32 @@ ON (c.id = o.cid) GROUP BY c.id;

-- 2.1 when field is primitive

set hive.map.aggr = true;

SELECT c.id, sort_array(collect_set(array(o.amount)))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(array(o.amount)))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

-- cast decimal

SELECT c.id, sort_array(collect_set(array(cast(o.amount as decimal(10,1)))))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(array(cast(o.amount as decimal(10,1)))))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

set hive.map.aggr = false;

SELECT c.id, sort_array(collect_set(array(o.amount)))
FROM customers c
INNER JOIN orders o
Expand All @@ -139,6 +249,20 @@ ON (c.id = o.cid) GROUP BY c.id;

-- 2.2 when field is struct

set hive.map.aggr = true;

SELECT c.id, sort_array(collect_set(array(o.sub)))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(array(o.sub)))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

set hive.map.aggr = false;

SELECT c.id, sort_array(collect_set(array(o.sub)))
FROM customers c
INNER JOIN nested_orders o
Expand All @@ -151,6 +275,20 @@ ON (c.id = o.cid) GROUP BY c.id;

-- 2.3 when field is list

set hive.map.aggr = true;

SELECT c.id, sort_array(collect_set(array(map_values(o.sub))))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(array(map_values(o.sub))))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

set hive.map.aggr = false;

SELECT c.id, sort_array(collect_set(array(map_values(o.sub))))
FROM customers c
INNER JOIN nested_orders o
Expand All @@ -166,6 +304,32 @@ ON (c.id = o.cid) GROUP BY c.id;

-- 3.1 when field is primitive

set hive.map.aggr = true;

SELECT c.id, sort_array(collect_set(map("amount", o.amount)))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(map("amount", o.amount)))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

-- cast decimal

SELECT c.id, sort_array(collect_set(map("amount", cast(o.amount as decimal(10,1)))))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(map("amount", cast(o.amount as decimal(10,1)))))
FROM customers c
INNER JOIN orders o
ON (c.id = o.cid) GROUP BY c.id;

set hive.map.aggr = false;

SELECT c.id, sort_array(collect_set(map("amount", o.amount)))
FROM customers c
INNER JOIN orders o
Expand All @@ -190,6 +354,20 @@ ON (c.id = o.cid) GROUP BY c.id;

-- 3.2 when field is struct

set hive.map.aggr = true;

SELECT c.id, sort_array(collect_set(map("sub", o.sub)))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(map("sub", o.sub)))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

set hive.map.aggr = false;

SELECT c.id, sort_array(collect_set(map("sub", o.sub)))
FROM customers c
INNER JOIN nested_orders o
Expand All @@ -202,6 +380,8 @@ ON (c.id = o.cid) GROUP BY c.id;

-- 3.3 when field is list

set hive.map.aggr = true;

SELECT c.id, sort_array(collect_set(map("sub", map_values(o.sub))))
FROM customers c
INNER JOIN nested_orders o
Expand All @@ -212,6 +392,17 @@ FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

set hive.map.aggr = false;

SELECT c.id, sort_array(collect_set(map("sub", map_values(o.sub))))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

SELECT c.id, sort_array(collect_list(map("sub", map_values(o.sub))))
FROM customers c
INNER JOIN nested_orders o
ON (c.id = o.cid) GROUP BY c.id;

-- clean up

Expand Down
Loading
Loading