Skip to content

Commit

Permalink
TEZ-4577: SortSpan could be created real small, resulting in eventual…
Browse files Browse the repository at this point in the history
… job failure (#367) (Chenyu Zheng reviewed by Laszlo Bodor)
  • Loading branch information
zhengchenyu authored Dec 12, 2024
1 parent b5bf8dc commit d84fdca
Showing 1 changed file with 2 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ public void sort() throws IOException {
}
Preconditions.checkArgument(buffers.get(bufferIndex) != null, "block should not be empty");
//TODO: fix per item being passed.
span = new SortSpan((ByteBuffer)buffers.get(bufferIndex).clear(), items,
span = new SortSpan((ByteBuffer)buffers.get(bufferIndex).clear(), (1024*1024),
perItem, ConfigUtils.getIntermediateOutputKeyComparator(this.conf));
} else {
// queue up the sort
Expand Down Expand Up @@ -940,7 +940,7 @@ private final class SortSpan implements IndexedSortable {
public SortSpan(ByteBuffer source, int maxItems, int perItem, RawComparator comparator) {
capacity = source.remaining();
int metasize = METASIZE*maxItems;
int dataSize = maxItems * perItem;
long dataSize = (long) maxItems * (long) perItem;
if(capacity < (metasize+dataSize)) {
// try to allocate less meta space, because we have sample data
metasize = METASIZE*(capacity/(perItem+METASIZE));
Expand Down

0 comments on commit d84fdca

Please sign in to comment.