From 5c7f5abf9f58bebb9af0732bcec09dd7688d3dbf Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 31 May 2024 17:53:39 -0700 Subject: [PATCH 1/3] Changes in ds-java required to move to java 17 as ds-java 7.x and compatible with the new ds-memory branch integrateJava17_v3 -> ds-memory 3.X --- pom.xml | 10 +- .../apache/datasketches/cpc/CpcSketch.java | 2 +- .../datasketches/hll/DirectHllArray.java | 8 +- .../org/apache/datasketches/theta/Sketch.java | 6 +- .../apache/datasketches/theta/Sketches.java | 12 +- .../filters/bloomfilter/BloomFilterTest.java | 13 +- .../bloomfilter/DirectBitArrayRTest.java | 4 +- .../bloomfilter/DirectBitArrayTest.java | 12 +- ...ash3v2Test.java => MurmurHash3v3Test.java} | 75 ++++---- .../hll/DirectAuxHashMapTest.java | 14 +- .../hll/DirectCouponListTest.java | 11 +- .../datasketches/hll/PreambleUtilTest.java | 8 +- .../quantiles/DebugUnionTest.java | 12 +- .../DirectQuantilesMemoryRequestTest.java | 58 +++--- .../quantiles/DoublesSketchTest.java | 39 ++-- .../quantiles/PreambleUtilTest.java | 13 +- .../datasketches/theta/CompactSketchTest.java | 12 +- .../theta/DirectQuickSelectSketchTest.java | 166 +++++++++-------- .../theta/HeapifyWrapSerVer1and2Test.java | 171 +++++++++--------- .../datasketches/theta/SketchesTest.java | 2 +- .../datasketches/theta/UnionImplTest.java | 26 ++- .../datasketches/theta/UpdateSketchTest.java | 4 +- 22 files changed, 364 insertions(+), 314 deletions(-) rename src/test/java/org/apache/datasketches/hash/{MurmurHash3v2Test.java => MurmurHash3v3Test.java} (82%) diff --git a/pom.xml b/pom.xml index 35107f24c..e43a426ba 100644 --- a/pom.xml +++ b/pom.xml @@ -88,7 +88,7 @@ under the License. - 7.5.1 + 7.10.0 generate_java_files check_cpp_files @@ -96,7 +96,7 @@ under the License. 3.6.3 - 1.8 + 17 ${java.version} ${java.version} -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 @@ -134,12 +134,12 @@ under the License. - + @@ -193,7 +193,7 @@ under the License. - [1.8.0,9.0),[11.0,12.0) + [1.8.0,9.0),[11.0,12.0), [17,18) [${maven.version},) diff --git a/src/main/java/org/apache/datasketches/cpc/CpcSketch.java b/src/main/java/org/apache/datasketches/cpc/CpcSketch.java index a374b5745..4ed89dd19 100644 --- a/src/main/java/org/apache/datasketches/cpc/CpcSketch.java +++ b/src/main/java/org/apache/datasketches/cpc/CpcSketch.java @@ -300,7 +300,7 @@ public byte[] toByteArray() { final long cap = state.getRequiredSerializedBytes(); final WritableMemory wmem = WritableMemory.allocate((int) cap); state.exportToMemory(wmem); - return (byte[]) wmem.getArray(); + return wmem.getArray(); } /** diff --git a/src/main/java/org/apache/datasketches/hll/DirectHllArray.java b/src/main/java/org/apache/datasketches/hll/DirectHllArray.java index 5b3e1a4ff..07835da69 100644 --- a/src/main/java/org/apache/datasketches/hll/DirectHllArray.java +++ b/src/main/java/org/apache/datasketches/hll/DirectHllArray.java @@ -56,7 +56,7 @@ abstract class DirectHllArray extends AbstractHllArray { WritableMemory wmem; Memory mem; Object memObj; - long memAdd; + //long memAdd; final boolean compact; private static int checkMemCompactFlag(final WritableMemory wmem, final int lgConfigK) { @@ -70,7 +70,7 @@ private static int checkMemCompactFlag(final WritableMemory wmem, final int lgCo this.wmem = wmem; mem = wmem; memObj = wmem.getArray(); - memAdd = wmem.getCumulativeOffset(0L); + //memAdd = wmem.getCumulativeOffset(0L); compact = extractCompactFlag(mem); insertEmptyFlag(wmem, false); } @@ -81,7 +81,7 @@ private static int checkMemCompactFlag(final WritableMemory wmem, final int lgCo wmem = null; this.mem = mem; memObj = ((WritableMemory) mem).getArray(); - memAdd = mem.getCumulativeOffset(0L); + //memAdd = mem.getCumulativeOffset(0L); compact = extractCompactFlag(mem); } @@ -90,7 +90,7 @@ final void updateMemory(final WritableMemory newWmem) { wmem = newWmem; mem = newWmem; memObj = wmem.getArray(); - memAdd = wmem.getCumulativeOffset(0L); + //memAdd = wmem.getCumulativeOffset(0L); } @Override diff --git a/src/main/java/org/apache/datasketches/theta/Sketch.java b/src/main/java/org/apache/datasketches/theta/Sketch.java index cc1fd4d23..d13fe9b80 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketch.java +++ b/src/main/java/org/apache/datasketches/theta/Sketch.java @@ -311,11 +311,11 @@ public static int getMaxCompactSketchBytes(final int numberOfEntries) { * log_base2 of the number of nominal entries, which is a power of 2. * @param lgNomEntries Nominal Entries * @return the maximum number of storage bytes required for a CompactSketch with the given - * nomEntries. + * lgNomEntries. */ public static int getCompactSketchMaxBytes(final int lgNomEntries) { - return (int)((2 << lgNomEntries) * ThetaUtil.REBUILD_THRESHOLD) - + Family.QUICKSELECT.getMaxPreLongs() * Long.BYTES; + return (int)((2 << lgNomEntries) * ThetaUtil.REBUILD_THRESHOLD + + Family.QUICKSELECT.getMaxPreLongs()) * Long.BYTES; } /** diff --git a/src/main/java/org/apache/datasketches/theta/Sketches.java b/src/main/java/org/apache/datasketches/theta/Sketches.java index 4b1461876..077d9d5ae 100644 --- a/src/main/java/org/apache/datasketches/theta/Sketches.java +++ b/src/main/java/org/apache/datasketches/theta/Sketches.java @@ -86,7 +86,7 @@ public static int getMaxAnotBResultBytes(final int maxNomEntries) { * @return the maximum number of storage bytes required for a CompactSketch with the given number * of entries. * @see Sketch#getMaxCompactSketchBytes(int) - * @deprecated as a public method. Use {@link #getCompactSketchMaxBytes(int) instead} + * @deprecated as a public method. Use {@link #getCompactSketchMaxBytes(int) getCompactSketchMaxBytes(int)} instead. */ @Deprecated public static int getMaxCompactSketchBytes(final int numberOfEntries) { @@ -95,14 +95,14 @@ public static int getMaxCompactSketchBytes(final int numberOfEntries) { /** * Returns the maximum number of storage bytes required for a CompactSketch given the configured - * number of nominal entries (power of 2). - * @param nomEntries Nominal Entries + * log_base2 of the number of nominal entries, which is a power of 2. + * @param lgNomEntries Nominal Entries * @return the maximum number of storage bytes required for a CompactSketch with the given - * nomEntries. + * lgNomEntries. * @see Sketch#getCompactSketchMaxBytes(int) */ - public static int getCompactSketchMaxBytes(final int nomEntries) { - return Sketch.getCompactSketchMaxBytes(nomEntries); + public static int getCompactSketchMaxBytes(final int lgNomEntries) { + return Sketch.getCompactSketchMaxBytes(lgNomEntries); } /** diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java index dedaf9db3..7a0fbb7ff 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java @@ -24,14 +24,19 @@ import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; +import java.nio.ByteOrder; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesReadOnlyException; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; +import jdk.incubator.foreign.ResourceScope; + public class BloomFilterTest { @Test @@ -51,8 +56,10 @@ public void createNewFilterTest() throws Exception { assertFalse(bf1.isDirect()); assertFalse(bf1.isReadOnly()); - try (WritableHandle wh = WritableMemory.allocateDirect(sizeBytes)) { - final WritableMemory wmem = wh.getWritable(); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(sizeBytes, + new DefaultMemoryRequestServer())).scope()) { + final BloomFilter bf2 = new BloomFilter(numBits, numHashes, seed, wmem); assertTrue(bf2.isEmpty()); assertTrue(bf2.hasMemory()); diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java index 521019e62..bbedd2fb7 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayRTest.java @@ -142,8 +142,8 @@ public void bitAddressOutOfBoundsNonEmptyTest() { final Memory mem = bitArrayToMemory(hba); DirectBitArrayR dba = DirectBitArrayR.wrap(mem, hba.isEmpty()); - assertThrows(AssertionError.class, () -> dba.getBit(-10)); - assertThrows(AssertionError.class, () -> dba.getBit(2048)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.getBit(-10)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.getBit(2048)); } @Test diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java index a45bcbb82..1df6cc9d9 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/DirectBitArrayTest.java @@ -172,12 +172,12 @@ public void bitAddresOutOfBoundsNonEmptyTest() { dba.getAndSetBit(i); } - assertThrows(AssertionError.class, () -> dba.getBit(-10)); - assertThrows(AssertionError.class, () -> dba.getBit(2048)); - assertThrows(AssertionError.class, () -> dba.setBit(-20)); - assertThrows(AssertionError.class, () -> dba.setBit(4096)); - assertThrows(AssertionError.class, () -> dba.getAndSetBit(-30)); - assertThrows(AssertionError.class, () -> dba.getAndSetBit(8192)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.getBit(-10)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.getBit(2048)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.setBit(-20)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.setBit(4096)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.getAndSetBit(-30)); + assertThrows(IndexOutOfBoundsException.class, () -> dba.getAndSetBit(8192)); } @Test diff --git a/src/test/java/org/apache/datasketches/hash/MurmurHash3v2Test.java b/src/test/java/org/apache/datasketches/hash/MurmurHash3v3Test.java similarity index 82% rename from src/test/java/org/apache/datasketches/hash/MurmurHash3v2Test.java rename to src/test/java/org/apache/datasketches/hash/MurmurHash3v3Test.java index 23f369e63..8699a091a 100644 --- a/src/test/java/org/apache/datasketches/hash/MurmurHash3v2Test.java +++ b/src/test/java/org/apache/datasketches/hash/MurmurHash3v3Test.java @@ -28,13 +28,13 @@ import org.testng.annotations.Test; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.MurmurHash3v2; +import org.apache.datasketches.memory.internal.MurmurHash3v3; import org.apache.datasketches.memory.WritableMemory; /** * @author Lee Rhodes */ -public class MurmurHash3v2Test { +public class MurmurHash3v3Test { private Random rand = new Random(); private static final int trials = 1 << 20; @@ -154,33 +154,33 @@ private static final long[] hashV1(byte[] key, long seed) { } private static final long[] hashV2(long[] key, long seed) { - return MurmurHash3v2.hash(key, seed); + return MurmurHash3v3.hash(key, seed); } private static final long[] hashV2(int[] key2, long seed) { - return MurmurHash3v2.hash(key2, seed); + return MurmurHash3v3.hash(key2, seed); } private static final long[] hashV2(char[] key, long seed) { - return MurmurHash3v2.hash(key, seed); + return MurmurHash3v3.hash(key, seed); } private static final long[] hashV2(byte[] key, long seed) { - return MurmurHash3v2.hash(key, seed); + return MurmurHash3v3.hash(key, seed); } //V2 single primitives private static final long[] hashV2(long key, long seed, long[] out) { - return MurmurHash3v2.hash(key, seed, out); + return MurmurHash3v3.hash(key, seed, out); } // private static final long[] hashV2(double key, long seed, long[] out) { -// return MurmurHash3v2.hash(key, seed, out); +// return MurmurHash3v3.hash(key, seed, out); // } // private static final long[] hashV2(String key, long seed, long[] out) { -// return MurmurHash3v2.hash(key, seed, out); +// return MurmurHash3v3.hash(key, seed, out); // } @@ -199,7 +199,7 @@ public void offsetChecks() { for (int offset = 0; offset < 16; offset++) { int arrLen = cap - offset; - hash1 = MurmurHash3v2.hash(wmem, offset, arrLen, seed, hash1); + hash1 = MurmurHash3v3.hash(wmem, offset, arrLen, seed, hash1); byte[] byteArr2 = new byte[arrLen]; wmem.getByteArray(offset, byteArr2, 0, arrLen); hash2 = MurmurHash3.hash(byteArr2, seed); @@ -222,8 +222,8 @@ public void byteArrChecks() { for (int i = 0; i < j; i++) { wmem.putByte(i, (byte) (-128 + i)); } long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v2.hash(in, seed); + hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v3.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -246,8 +246,8 @@ public void charArrChecks() { for (int i = 0; i < j; i++) { wmem.putInt(i, i); } long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v2.hash(in, seed); + hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v3.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -270,8 +270,8 @@ public void intArrChecks() { for (int i = 0; i < j; i++) { wmem.putInt(i, i); } long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v2.hash(in, seed); + hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v3.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -294,8 +294,8 @@ public void longArrChecks() { for (int i = 0; i < j; i++) { wmem.putLong(i, i); } long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v2.hash(in, seed); + hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v3.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -313,8 +313,8 @@ public void longCheck() { WritableMemory wmem = WritableMemory.writableWrap(in); long[] hash1 = MurmurHash3.hash(in, 0); - hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v2.hash(in, seed); + hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v3.hash(in, seed); assertEquals(hash1, hash2); assertEquals(hash1, hash3); @@ -325,62 +325,57 @@ public void checkEmptiesNulls() { long seed = 123; long[] hashOut = new long[2]; try { - MurmurHash3v2.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut); //mem empty - fail(); - } catch (final IllegalArgumentException e) { } //OK - try { - Memory mem = null; - MurmurHash3v2.hash(mem, 0, 0, seed, hashOut); //mem null + MurmurHash3v3.hash(Memory.wrap(new long[0]), 0, 0, seed, hashOut); //mem empty fail(); } catch (final IllegalArgumentException e) { } //OK try { String s = ""; - MurmurHash3v2.hash(s, seed, hashOut); //string empty + MurmurHash3v3.hash(s, seed, hashOut); //string empty fail(); } catch (final IllegalArgumentException e) { } //OK try { String s = null; - MurmurHash3v2.hash(s, seed, hashOut); //string null + MurmurHash3v3.hash(s, seed, hashOut); //string null fail(); } catch (final IllegalArgumentException e) { } //OK try { byte[] barr = new byte[0]; - MurmurHash3v2.hash(barr, seed); //byte[] empty + MurmurHash3v3.hash(barr, seed); //byte[] empty fail(); } catch (final IllegalArgumentException e) { } //OK try { byte[] barr = null; - MurmurHash3v2.hash(barr, seed); //byte[] null + MurmurHash3v3.hash(barr, seed); //byte[] null fail(); } catch (final IllegalArgumentException e) { } //OK try { char[] carr = new char[0]; - MurmurHash3v2.hash(carr, seed); //char[] empty + MurmurHash3v3.hash(carr, seed); //char[] empty fail(); } catch (final IllegalArgumentException e) { } //OK try { char[] carr = null; - MurmurHash3v2.hash(carr, seed); //char[] null + MurmurHash3v3.hash(carr, seed); //char[] null fail(); } catch (final IllegalArgumentException e) { } //OK try { int[] iarr = new int[0]; - MurmurHash3v2.hash(iarr, seed); //int[] empty + MurmurHash3v3.hash(iarr, seed); //int[] empty fail(); } catch (final IllegalArgumentException e) { } //OK try { int[] iarr = null; - MurmurHash3v2.hash(iarr, seed); //int[] null + MurmurHash3v3.hash(iarr, seed); //int[] null fail(); } catch (final IllegalArgumentException e) { } //OK try { long[] larr = new long[0]; - MurmurHash3v2.hash(larr, seed); //long[] empty + MurmurHash3v3.hash(larr, seed); //long[] empty fail(); } catch (final IllegalArgumentException e) { } //OK try { long[] larr = null; - MurmurHash3v2.hash(larr, seed); //long[] null + MurmurHash3v3.hash(larr, seed); //long[] null fail(); } catch (final IllegalArgumentException e) { } //OK } @@ -390,9 +385,9 @@ public void checkStringLong() { long seed = 123; long[] hashOut = new long[2]; String s = "123"; - assertTrue(MurmurHash3v2.hash(s, seed, hashOut)[0] != 0); + assertTrue(MurmurHash3v3.hash(s, seed, hashOut)[0] != 0); long v = 123; - assertTrue(MurmurHash3v2.hash(v, seed, hashOut)[0] != 0); + assertTrue(MurmurHash3v3.hash(v, seed, hashOut)[0] != 0); } @Test @@ -420,8 +415,8 @@ private static long[] checkDouble(double dbl) { WritableMemory wmem = WritableMemory.writableWrap(dataArr); long[] hash1 = MurmurHash3.hash(dataArr, 0); - hash2 = MurmurHash3v2.hash(wmem, offset, bytes, seed, hash2); - long[] hash3 = MurmurHash3v2.hash(dbl, seed, hash2); + hash2 = MurmurHash3v3.hash(wmem, offset, bytes, seed, hash2); + long[] hash3 = MurmurHash3v3.hash(dbl, seed, hash2); assertEquals(hash1, hash2); assertEquals(hash1, hash3); diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java index 78b18c1cc..7b3cd5c67 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java @@ -31,12 +31,11 @@ import org.apache.datasketches.common.SketchesStateException; import org.apache.datasketches.memory.DefaultMemoryRequestServer; -import org.testng.annotations.Test; - import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; +import org.testng.annotations.Test; +import jdk.incubator.foreign.ResourceScope; /** * @author Lee Rhodes @@ -48,11 +47,12 @@ public void checkGrow() { int lgConfigK = 4; TgtHllType tgtHllType = TgtHllType.HLL_4; int n = 8; //put lgConfigK == 4 into HLL mode - int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType); + long bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType); HllSketch hllSketch; - try (WritableHandle handle = WritableMemory.allocateDirect(bytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - WritableMemory wmem = handle.getWritable(); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); for (int i = 0; i < n; i++) { hllSketch.update(i); diff --git a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java index 38cbc4977..c6c006f29 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java @@ -27,8 +27,11 @@ import org.testng.annotations.Test; +import jdk.incubator.foreign.ResourceScope; + +import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; /** @@ -69,9 +72,9 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool //println("DIRECT"); byte[] barr1; - WritableMemory wmem = null; - try (WritableHandle hand = WritableMemory.allocateDirect(bytes)) { - wmem = hand.getWritable(); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes, + new DefaultMemoryRequestServer())).scope()) { //byte[] byteArr = new byte[bytes]; //WritableMemory wmem = WritableMemory.wrap(byteArr); hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); diff --git a/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java b/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java index cd1e0cbcf..deb8c5be5 100644 --- a/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java +++ b/src/test/java/org/apache/datasketches/hll/PreambleUtilTest.java @@ -109,7 +109,7 @@ public void checkCorruptMemoryInput() { HllSketch sk = new HllSketch(12); byte[] memObj = sk.toCompactByteArray(); WritableMemory wmem = WritableMemory.writableWrap(memObj); - long memAdd = wmem.getCumulativeOffset(0); + //long memAdd = wmem.getCumulativeOffset(0); HllSketch bad; //checkFamily @@ -148,7 +148,7 @@ public void checkCorruptMemoryInput() { for (int i = 1; i <= 15; i++) { sk.update(i); } memObj = sk.toCompactByteArray(); wmem = WritableMemory.writableWrap(memObj); - memAdd = wmem.getCumulativeOffset(0); + //memAdd = wmem.getCumulativeOffset(0); //check wrong PreInts and SET try { @@ -162,7 +162,7 @@ public void checkCorruptMemoryInput() { for (int i = 15; i <= 1000; i++) { sk.update(i); } memObj = sk.toCompactByteArray(); wmem = WritableMemory.writableWrap(memObj); - memAdd = wmem.getCumulativeOffset(0); + //memAdd = wmem.getCumulativeOffset(0); //check wrong PreInts and HLL try { @@ -179,7 +179,7 @@ public void checkExtractFlags() { int bytes = HllSketch.getMaxUpdatableSerializationBytes(4, TgtHllType.HLL_4); WritableMemory wmem = WritableMemory.allocate(bytes); Object memObj = wmem.getArray(); - long memAdd = wmem.getCumulativeOffset(0L); + //long memAdd = wmem.getCumulativeOffset(0L); HllSketch sk = new HllSketch(4, TgtHllType.HLL_4, wmem); int flags = extractFlags(wmem); assertEquals(flags, EMPTY_FLAG_MASK); diff --git a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java index bbf89f338..d19d3b471 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java @@ -23,10 +23,15 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; +import java.nio.ByteOrder; import java.util.HashSet; import org.testng.annotations.Test; -import org.apache.datasketches.memory.WritableHandle; + +import jdk.incubator.foreign.ResourceScope; + +import org.apache.datasketches.memory.DefaultMemoryRequestServer; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.quantilescommon.QuantilesDoublesSketchIterator; @@ -61,8 +66,9 @@ public void test() { DoublesSketch.setRandom(1); //make deterministic for test DoublesUnion dUnion; DoublesSketch dSketch; - try ( WritableHandle wdh = WritableMemory.allocateDirect(10_000_000) ) { - WritableMemory wmem = wdh.getWritable(); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(10_000_000, + new DefaultMemoryRequestServer())).scope()) { dUnion = DoublesUnion.builder().setMaxK(8).build(wmem); for (int s = 0; s < numSketches; s++) { dUnion.union(sketchArr[s]); } dSketch = dUnion.getResult(); //result is on heap diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java index 5c2882c5e..84145239e 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java @@ -30,9 +30,11 @@ import org.testng.annotations.Test; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; +import jdk.incubator.foreign.ResourceScope; + /** * The concept for these tests is that the "MemoryManager" classes below are proxies for the * implementation that owns the native memory allocations, thus is responsible for @@ -47,10 +49,11 @@ public void checkLimitedMemoryScenarios() { //Requesting application final int initBytes = ((2 * k) + 4) << 3; //just the BB //########## Owning Implementation - // This part would actually be part of the Memory owning implemention so it is faked here - try (WritableHandle wdh = WritableMemory.allocateDirect(initBytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - final WritableMemory wmem = wdh.getWritable(); + // This part would actually be part of the Memory owning implementation so it is faked here + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + println("Initial mem size: " + wmem.getCapacity()); //########## Receiving Application @@ -84,13 +87,12 @@ public void checkGrowBaseBuf() { final int u = 32; // don't need the BB to fill here final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything - try (WritableHandle memHandler = WritableMemory.allocateDirect(initBytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - //final MemoryManager memMgr = new MemoryManager(); - //final WritableMemory mem1 = memMgr.request(initBytes); - final WritableMemory mem1 = memHandler.getWritable(); - println("Initial mem size: " + mem1.getCapacity()); - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + + println("Initial mem size: " + wmem.getCapacity()); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); for (int i = 1; i <= u; i++) { usk1.update(i); } @@ -108,13 +110,12 @@ public void checkGrowCombBuf() { final int u = (2 * k) - 1; //just to fill the BB final int initBytes = ((2 * k) + 4) << 3; //just room for BB - try (WritableHandle memHandler = WritableMemory.allocateDirect(initBytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - //final MemoryManager memMgr = new MemoryManager(); - //final WritableMemory mem1 = memMgr.request(initBytes); - final WritableMemory mem1 = memHandler.getWritable(); - println("Initial mem size: " + mem1.getCapacity()); - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(mem1); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + + println("Initial mem size: " + wmem.getCapacity()); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); for (int i = 1; i <= u; i++) { usk1.update(i); } @@ -138,21 +139,22 @@ public void checkGrowFromWrappedEmptySketch() { final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(); final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); - try (WritableHandle memHandle = WritableMemory.allocateDirect(initBytes, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - WritableMemory mem = memHandle.getWritable(); - origSketchMem.copyTo(0, mem, 0, initBytes); - UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(mem); - assertTrue(mem.isSameResource(usk2.getMemory())); - assertEquals(mem.getCapacity(), initBytes); - assertTrue(mem.isDirect()); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + + origSketchMem.copyTo(0, wmem, 0, initBytes); + UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem); + assertTrue(wmem.isSameResource(usk2.getMemory())); + assertEquals(wmem.getCapacity(), initBytes); + assertTrue(wmem.isDirect()); assertTrue(usk2.isEmpty()); //update the sketch forcing it to grow on-heap for (int i = 1; i <= 5; i++) { usk2.update(i); } assertEquals(usk2.getN(), 5); WritableMemory mem2 = usk2.getMemory(); - assertFalse(mem.isSameResource(mem2)); + assertFalse(wmem.isSameResource(mem2)); assertFalse(mem2.isDirect()); //should now be on-heap final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java index 5bc5b4eaa..fbac2af1e 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java @@ -28,13 +28,15 @@ import java.nio.ByteOrder; import org.apache.datasketches.memory.DefaultMemoryRequestServer; -import org.apache.datasketches.memory.WritableHandle; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.quantilescommon.DoublesSortedView; import org.apache.datasketches.quantilescommon.DoublesSortedViewIterator; import org.testng.Assert; import org.testng.annotations.Test; +import jdk.incubator.foreign.ResourceScope; + public class DoublesSketchTest { @Test @@ -140,15 +142,17 @@ public void checkEmptyExceptions() { @Test public void directSketchShouldMoveOntoHeapEventually() { - try (WritableHandle wdh = WritableMemory.allocateDirect(1000, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())) { - WritableMemory mem = wdh.getWritable(); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); - Assert.assertTrue(sketch.isSameResource(mem)); + + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(1000, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); + Assert.assertTrue(sketch.isSameResource(wmem)); for (int i = 0; i < 1000; i++) { sketch.update(i); } - Assert.assertFalse(sketch.isSameResource(mem)); + Assert.assertFalse(sketch.isSameResource(wmem)); } catch (final Exception e) { throw new RuntimeException(e); } @@ -157,13 +161,14 @@ public void directSketchShouldMoveOntoHeapEventually() { @Test public void directSketchShouldMoveOntoHeapEventually2() { int i = 0; - try (WritableHandle wdh = - WritableMemory.allocateDirect(50, ByteOrder.LITTLE_ENDIAN, new DefaultMemoryRequestServer())) { - WritableMemory mem = wdh.getWritable(); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); - Assert.assertTrue(sketch.isSameResource(mem)); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(50, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); + Assert.assertTrue(sketch.isSameResource(wmem)); for (; i < 1000; i++) { - if (sketch.isSameResource(mem)) { + if (sketch.isSameResource(wmem)) { sketch.update(i); } else { //println("MOVED OUT at i = " + i); @@ -177,9 +182,11 @@ public void directSketchShouldMoveOntoHeapEventually2() { @Test public void checkEmptyDirect() { - try (WritableHandle wdh = WritableMemory.allocateDirect(1000)) { - WritableMemory mem = wdh.getWritable(); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(mem); + WritableMemory wmem; + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(1000, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); sketch.toByteArray(); //exercises a specific path } catch (final Exception e) { throw new RuntimeException(e); diff --git a/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java b/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java index be2f328b2..adf916ef3 100644 --- a/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/PreambleUtilTest.java @@ -37,19 +37,24 @@ import static org.apache.datasketches.quantiles.PreambleUtil.insertSerVer; import static org.testng.Assert.assertEquals; -import org.testng.annotations.Test; +import java.nio.ByteOrder; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; +import org.testng.annotations.Test; + +import jdk.incubator.foreign.ResourceScope; public class PreambleUtilTest { @Test public void checkInsertsAndExtracts() { final int bytes = 32; - try (WritableHandle offHeapMemHandler = WritableMemory.allocateDirect(bytes)) { - final WritableMemory offHeapMem = offHeapMemHandler.getWritable(); + WritableMemory offHeapMem; + try (ResourceScope scope = (offHeapMem = WritableMemory.allocateDirect(bytes, 1, + ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + final WritableMemory onHeapMem = WritableMemory.writableWrap(new byte[bytes]); onHeapMem.clear(); diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java index 42b6069da..1ad55696c 100644 --- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java @@ -26,13 +26,18 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; +import java.nio.ByteOrder; + import org.apache.datasketches.common.Family; import org.apache.datasketches.common.SketchesArgumentException; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.testng.annotations.Test; +import jdk.incubator.foreign.ResourceScope; + /** * @author Lee Rhodes */ @@ -78,8 +83,9 @@ public void checkHeapifyWrap(int k, int u, boolean ordered) { //Prepare Memory for direct int bytes = usk.getCompactBytes(); //for Compact - try (WritableHandle wdh = WritableMemory.allocateDirect(bytes)) { - WritableMemory directMem = wdh.getWritable(); + WritableMemory directMem; + try (ResourceScope scope = (directMem = WritableMemory.allocateDirect(bytes, + new DefaultMemoryRequestServer())).scope()) { /**Via CompactSketch.compact**/ refSk = usk.compact(ordered, directMem); diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java index 5191c7c68..1f1bfa9ec 100644 --- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java @@ -38,19 +38,23 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import java.nio.ByteOrder; import java.util.Arrays; import org.apache.datasketches.common.Family; import org.apache.datasketches.common.ResizeFactor; import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesReadOnlyException; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; import org.apache.datasketches.memory.Memory; -import org.apache.datasketches.memory.WritableHandle; +//import org.apache.datasketches.memory.WritableHandle; import org.apache.datasketches.memory.WritableMemory; import org.apache.datasketches.thetacommon.HashOperations; import org.apache.datasketches.thetacommon.ThetaUtil; import org.testng.annotations.Test; +import jdk.incubator.foreign.ResourceScope; + /** * @author Lee Rhodes */ @@ -59,10 +63,10 @@ public class DirectQuickSelectSketchTest { @Test//(expectedExceptions = SketchesArgumentException.class) public void checkBadSerVer() { int k = 512; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); + WritableMemory wmem; + try (ResourceScope scope = (wmem = makeNativeMemory(k)).scope()) { - UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(mem); + UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(wmem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks assertTrue(usk.isEmpty()); @@ -73,9 +77,9 @@ public void checkBadSerVer() { assertEquals(usk.getEstimate(), k, 0.0); assertEquals(sk1.getRetainedEntries(false), k); - mem.putByte(SER_VER_BYTE, (byte) 0); //corrupt the SerVer byte + wmem.putByte(SER_VER_BYTE, (byte) 0); //corrupt the SerVer byte - Sketch.wrap(mem); + Sketch.wrap(wmem); } catch (final Exception e) { if (e instanceof SketchesArgumentException) {} else { throw new RuntimeException(e); } @@ -85,9 +89,9 @@ public void checkBadSerVer() { @Test//(expectedExceptions = SketchesArgumentException.class) public void checkConstructorKtooSmall() { int k = 8; - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); - UpdateSketch.builder().setNominalEntries(k).build(mem); + WritableMemory wmem; + try (ResourceScope scope = (wmem = makeNativeMemory(k)).scope()) { + UpdateSketch.builder().setNominalEntries(k).build(wmem); } catch (final Exception e) { if (e instanceof SketchesArgumentException) {} else { throw new RuntimeException(e); } @@ -97,9 +101,9 @@ public void checkConstructorKtooSmall() { @Test//(expectedExceptions = SketchesArgumentException.class) public void checkConstructorMemTooSmall() { int k = 16; - try (WritableHandle h = makeNativeMemory(k/2)) { - WritableMemory mem = h.getWritable(); - UpdateSketch.builder().setNominalEntries(k).build(mem); + WritableMemory wmem; + try (ResourceScope scope = (wmem = makeNativeMemory(k/2)).scope()) { + UpdateSketch.builder().setNominalEntries(k).build(wmem); } catch (final Exception e) { if (e instanceof SketchesArgumentException) {} else { throw new RuntimeException(e); } @@ -124,10 +128,10 @@ public void checkHeapifyMemoryEstimating() { int k = 512; int u = 2*k; //thus estimating - try (WritableHandle h = makeNativeMemory(k)) { - WritableMemory mem = h.getWritable(); + WritableMemory wmem; + try (ResourceScope scope = (wmem = makeNativeMemory(k)).scope()) { - UpdateSketch sk1 = UpdateSketch.builder().setNominalEntries(k).build(mem); + UpdateSketch sk1 = UpdateSketch.builder().setNominalEntries(k).build(wmem); for (int i=0; i Date: Fri, 14 Jun 2024 17:12:19 -0700 Subject: [PATCH 2/3] Fixes to align with modified Memory --- .github/workflows/auto-jdk-matrix.yml | 41 ++--- .../hll/DirectAuxHashMapTest.java | 83 +++++----- .../DirectQuantilesMemoryRequestTest.java | 147 ++++++++---------- .../quantiles/DoublesSketchTest.java | 43 ++--- .../theta/DirectQuickSelectSketchTest.java | 64 ++++---- .../datasketches/theta/UnionImplTest.java | 25 ++- 6 files changed, 179 insertions(+), 224 deletions(-) diff --git a/.github/workflows/auto-jdk-matrix.yml b/.github/workflows/auto-jdk-matrix.yml index 83da580ca..ce15bb57e 100644 --- a/.github/workflows/auto-jdk-matrix.yml +++ b/.github/workflows/auto-jdk-matrix.yml @@ -7,49 +7,51 @@ on: workflow_dispatch: env: - MAVEN_OPTS: -Xmx4g -Xms1g + MAVEN_OPTS: -Xmx1g -Xms1g jobs: build: name: Build, Test, Install runs-on: ubuntu-latest + strategy: fail-fast: false - matrix: - jdk: [ 8,11 ] + env: - JDK_VERSION: ${{ matrix.jdk }} + JDK_VERSION: 17 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false + - name: Print Current workflow + run: > + cat .github/workflows/auto-jdk-matrix.yml + + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-package: jdk + architecture: x64 + java-version: 17 + - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.m2/repository key: build-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: build-${{ runner.os }}-maven- - - name: Install Matrix JDK - uses: actions/setup-java@v3 - with: - java-version: ${{ matrix.jdk }} - distribution: 'temurin' - java-package: jdk - architecture: x64 -# Architecture options: x86, x64, armv7, aarch64, ppc64le -# setup-java@v3 has a "with cache" option - - name: Echo Java Version run: > java -version - name: Test run: > - mvn clean test + mvn clean test -B -Dmaven.javadoc.skip=true -Dgpg.skip=true @@ -59,7 +61,10 @@ jobs: -DskipTests=true -Dgpg.skip=true +# Architecture options: x86, x64, armv7, aarch64, ppc64le +# setup-java@v4 has a "with cache" option # Lifecycle: validate, compile, test, package, verify, install, deploy -# -B batch mode +# -B batch mode, never stops for user input # -V show Version without stopping +# -X debug mode # -q quiet, only show errors diff --git a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java index 7b3cd5c67..98039616f 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectAuxHashMapTest.java @@ -49,52 +49,47 @@ public void checkGrow() { int n = 8; //put lgConfigK == 4 into HLL mode long bytes = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType); HllSketch hllSketch; - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + WritableMemory wmem = WritableMemory.allocateDirect(bytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); - for (int i = 0; i < n; i++) { - hllSketch.update(i); - } - hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values - hllSketch.couponUpdate(HllUtil.pair(8, 15)); - hllSketch.couponUpdate(HllUtil.pair(9, 15)); - //println(hllSketch.toString(true, true, true, true)); - DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl; - assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2); - assertTrue(hllSketch.isMemory()); - assertTrue(hllSketch.isOffHeap()); - assertTrue(hllSketch.isSameResource(wmem)); - - //Check heapify - byte[] byteArray = hllSketch.toCompactByteArray(); - HllSketch hllSketch2 = HllSketch.heapify(byteArray); - HllArray ha = (HllArray) hllSketch2.hllSketchImpl; - assertEquals(ha.getAuxHashMap().getLgAuxArrInts(), 2); - assertEquals(ha.getAuxHashMap().getAuxCount(), 3); - - //Check wrap - byteArray = hllSketch.toUpdatableByteArray(); - WritableMemory wmem2 = WritableMemory.writableWrap(byteArray); - hllSketch2 = HllSketch.writableWrap(wmem2); - //println(hllSketch2.toString(true, true, true, true)); - DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl; - assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2); - assertEquals(dha2.getAuxHashMap().getAuxCount(), 3); - - //Check grow to on-heap - hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow - //println(hllSketch.toString(true, true, true, true)); - dha = (DirectHllArray) hllSketch.hllSketchImpl; - assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3); - assertEquals(dha.getAuxHashMap().getAuxCount(), 4); - assertTrue(hllSketch.isMemory()); - assertFalse(hllSketch.isOffHeap()); - assertFalse(hllSketch.isSameResource(wmem)); - } catch (final Exception e) { - throw new RuntimeException(e); + hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); + for (int i = 0; i < n; i++) { + hllSketch.update(i); } + hllSketch.couponUpdate(HllUtil.pair(7, 15)); //mock extreme values + hllSketch.couponUpdate(HllUtil.pair(8, 15)); + hllSketch.couponUpdate(HllUtil.pair(9, 15)); + //println(hllSketch.toString(true, true, true, true)); + DirectHllArray dha = (DirectHllArray) hllSketch.hllSketchImpl; + assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 2); + assertTrue(hllSketch.isMemory()); + assertTrue(hllSketch.isOffHeap()); + assertTrue(hllSketch.isSameResource(wmem)); + + //Check heapify + byte[] byteArray = hllSketch.toCompactByteArray(); + HllSketch hllSketch2 = HllSketch.heapify(byteArray); + HllArray ha = (HllArray) hllSketch2.hllSketchImpl; + assertEquals(ha.getAuxHashMap().getLgAuxArrInts(), 2); + assertEquals(ha.getAuxHashMap().getAuxCount(), 3); + + //Check wrap + byteArray = hllSketch.toUpdatableByteArray(); + WritableMemory wmem2 = WritableMemory.writableWrap(byteArray); + hllSketch2 = HllSketch.writableWrap(wmem2); + //println(hllSketch2.toString(true, true, true, true)); + DirectHllArray dha2 = (DirectHllArray) hllSketch2.hllSketchImpl; + assertEquals(dha2.getAuxHashMap().getLgAuxArrInts(), 2); + assertEquals(dha2.getAuxHashMap().getAuxCount(), 3); + + //Check grow to on-heap + hllSketch.couponUpdate(HllUtil.pair(10, 15)); //puts it over the edge, must grow + //println(hllSketch.toString(true, true, true, true)); + dha = (DirectHllArray) hllSketch.hllSketchImpl; + assertEquals(dha.getAuxHashMap().getLgAuxArrInts(), 3); + assertEquals(dha.getAuxHashMap().getAuxCount(), 4); + assertTrue(hllSketch.isMemory()); + assertFalse(hllSketch.isOffHeap()); + assertFalse(hllSketch.isSameResource(wmem)); } @Test diff --git a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java index 84145239e..6552d0ae3 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java @@ -46,39 +46,34 @@ public class DirectQuantilesMemoryRequestTest { public void checkLimitedMemoryScenarios() { //Requesting application final int k = 128; final int u = 40 * k; - final int initBytes = ((2 * k) + 4) << 3; //just the BB + final int initBytes = ((2 * k) + 4) << 3; //just the BaseBuffer //########## Owning Implementation // This part would actually be part of the Memory owning implementation so it is faked here - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - - println("Initial mem size: " + wmem.getCapacity()); - - //########## Receiving Application - // The receiving application has been given wmem to use for a sketch, - // but alas, it is not ultimately large enough. - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); - assertTrue(usk1.isEmpty()); - - //Load the sketch - for (int i = 0; i < u; i++) { - // The sketch uses The MemoryRequest, acquired from wmem, to acquire more memory as - // needed, and requests via the MemoryRequest to free the old allocations. - usk1.update(i); - } - final double result = usk1.getQuantile(0.5); - println("Result: " + result); - assertEquals(result, u / 2.0, 0.05 * u); //Success - - //########## Owning Implementation - //The actual Memory has been re-allocated several times, - // so the above wmem reference is invalid. - println("\nFinal mem size: " + wmem.getCapacity()); - } catch (Exception e) { - throw new RuntimeException(e); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + + println("Initial mem size: " + wmem.getCapacity()); + + //########## Receiving Application + // The receiving application has been given wmem to use for a sketch, + // but alas, it is not ultimately large enough. + final UpdateDoublesSketch usk = DoublesSketch.builder().setK(k).build(wmem); + assertTrue(usk.isEmpty()); + + //Load the sketch + for (int i = 0; i < u; i++) { + // The sketch uses The MemoryRequest, acquired from wmem, to acquire more memory as + // needed, and requests via the MemoryRequest to free the old allocations. + usk.update(i); } + final double result = usk.getQuantile(0.5); + println("Result: " + result); + assertEquals(result, u / 2.0, 0.05 * u); //Success + + //########## Owning Implementation + //The actual Memory has been re-allocated several times, + // so the above wmem reference is invalid. + println("\nFinal mem size: " + wmem.getCapacity()); } @Test @@ -87,21 +82,16 @@ public void checkGrowBaseBuf() { final int u = 32; // don't need the BB to fill here final int initBytes = (4 + (u / 2)) << 3; // not enough to hold everything - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - - println("Initial mem size: " + wmem.getCapacity()); - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); - for (int i = 1; i <= u; i++) { - usk1.update(i); - } - final int currentSpace = usk1.getCombinedBufferItemCapacity(); - println("curCombBufItemCap: " + currentSpace); - assertEquals(currentSpace, 2 * k); - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + + println("Initial mem size: " + wmem.getCapacity()); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); + for (int i = 1; i <= u; i++) { + usk1.update(i); } + final int currentSpace = usk1.getCombinedBufferItemCapacity(); + println("curCombBufItemCap: " + currentSpace); + assertEquals(currentSpace, 2 * k); } @Test @@ -110,25 +100,19 @@ public void checkGrowCombBuf() { final int u = (2 * k) - 1; //just to fill the BB final int initBytes = ((2 * k) + 4) << 3; //just room for BB - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - - println("Initial mem size: " + wmem.getCapacity()); - final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); - for (int i = 1; i <= u; i++) { - usk1.update(i); - } - final int currentSpace = usk1.getCombinedBufferItemCapacity(); - println("curCombBufItemCap: " + currentSpace); - final double[] newCB = usk1.growCombinedBuffer(currentSpace, 3 * k); - final int newSpace = usk1.getCombinedBufferItemCapacity(); - println("newCombBurItemCap: " + newSpace); - assertEquals(newCB.length, 3 * k); - //memMgr.free(mem1); - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + + println("Initial mem size: " + wmem.getCapacity()); + final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(wmem); + for (int i = 1; i <= u; i++) { + usk1.update(i); } + final int currentSpace = usk1.getCombinedBufferItemCapacity(); + println("curCombBufItemCap: " + currentSpace); + final double[] newCB = usk1.growCombinedBuffer(currentSpace, 3 * k); + final int newSpace = usk1.getCombinedBufferItemCapacity(); + println("newCombBurItemCap: " + newSpace); + assertEquals(newCB.length, 3 * k); } @Test @@ -139,29 +123,24 @@ public void checkGrowFromWrappedEmptySketch() { final UpdateDoublesSketch usk1 = DoublesSketch.builder().setK(k).build(); final Memory origSketchMem = Memory.wrap(usk1.toByteArray()); - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(initBytes, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - - origSketchMem.copyTo(0, wmem, 0, initBytes); - UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem); - assertTrue(wmem.isSameResource(usk2.getMemory())); - assertEquals(wmem.getCapacity(), initBytes); - assertTrue(wmem.isDirect()); - assertTrue(usk2.isEmpty()); - - //update the sketch forcing it to grow on-heap - for (int i = 1; i <= 5; i++) { usk2.update(i); } - assertEquals(usk2.getN(), 5); - WritableMemory mem2 = usk2.getMemory(); - assertFalse(wmem.isSameResource(mem2)); - assertFalse(mem2.isDirect()); //should now be on-heap - - final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); - assertEquals(mem2.getCapacity(), expectedSize); - } catch (final Exception e) { - throw new RuntimeException(e); - } + WritableMemory wmem = WritableMemory.allocateDirect(initBytes, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + + origSketchMem.copyTo(0, wmem, 0, initBytes); + UpdateDoublesSketch usk2 = DirectUpdateDoublesSketch.wrapInstance(wmem); + assertTrue(wmem.isSameResource(usk2.getMemory())); + assertEquals(wmem.getCapacity(), initBytes); + assertTrue(wmem.isDirect()); + assertTrue(usk2.isEmpty()); + + //update the sketch forcing it to grow on-heap + for (int i = 1; i <= 5; i++) { usk2.update(i); } + assertEquals(usk2.getN(), 5); + WritableMemory mem2 = usk2.getMemory(); + assertFalse(wmem.isSameResource(mem2)); + assertFalse(mem2.isDirect()); //should now be on-heap + + final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3); + assertEquals(mem2.getCapacity(), expectedSize); } @Test diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java index fbac2af1e..8f4243bfe 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchTest.java @@ -143,46 +143,35 @@ public void checkEmptyExceptions() { @Test public void directSketchShouldMoveOntoHeapEventually() { - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(1000, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - - UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); - Assert.assertTrue(sketch.isSameResource(wmem)); - for (int i = 0; i < 1000; i++) { - sketch.update(i); - } - Assert.assertFalse(sketch.isSameResource(wmem)); - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory wmem = WritableMemory.allocateDirect(1000, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); + Assert.assertTrue(sketch.isSameResource(wmem)); + for (int i = 0; i < 1000; i++) { + sketch.update(i); } + Assert.assertFalse(sketch.isSameResource(wmem)); } @Test public void directSketchShouldMoveOntoHeapEventually2() { int i = 0; - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(50, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { + WritableMemory wmem = WritableMemory.allocateDirect(50, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); - Assert.assertTrue(sketch.isSameResource(wmem)); - for (; i < 1000; i++) { - if (sketch.isSameResource(wmem)) { - sketch.update(i); - } else { - //println("MOVED OUT at i = " + i); - break; - } + UpdateDoublesSketch sketch = DoublesSketch.builder().build(wmem); + Assert.assertTrue(sketch.isSameResource(wmem)); + for (; i < 1000; i++) { + if (sketch.isSameResource(wmem)) { + sketch.update(i); + } else { + //println("MOVED OUT at i = " + i); + break; } - } catch (final Exception e) { - throw new RuntimeException(e); } } @Test public void checkEmptyDirect() { - WritableMemory wmem; + WritableMemory wmem ; try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(1000, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java index 1f1bfa9ec..32b9bbb44 100644 --- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java @@ -876,16 +876,12 @@ public void checkMoveAndResize() { int u = 2 * k; int bytes = Sketches.getMaxUpdateSketchBytes(k); - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes / 2, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); - assertTrue(sketch.isSameResource(wmem)); - for (int i = 0; i < u; i++) { sketch.update(i); } - assertFalse(sketch.isSameResource(wmem)); - } catch (final Exception e) { - throw new RuntimeException(e); - } + WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + + UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); + assertTrue(sketch.isSameResource(wmem)); + for (int i = 0; i < u; i++) { sketch.update(i); } + assertFalse(sketch.isSameResource(wmem)); } @Test @@ -893,36 +889,30 @@ public void checkReadOnlyRebuildResize() { int k = 1 << 12; int u = 2 * k; int bytes = Sketches.getMaxUpdateSketchBytes(k); - WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes / 2, 1, - ByteOrder.nativeOrder(), new DefaultMemoryRequestServer())).scope()) { - UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); - for (int i = 0; i < u; i++) { sketch.update(i); } - double est1 = sketch.getEstimate(); - byte[] ser = sketch.toByteArray(); - Memory mem = Memory.wrap(ser); - UpdateSketch roSketch = (UpdateSketch) Sketches.wrapSketch(mem); - double est2 = roSketch.getEstimate(); - assertEquals(est2, est1); - try { - roSketch.rebuild(); - fail(); - } catch (SketchesReadOnlyException e) { - //expected - } - try { - roSketch.reset(); - fail(); - } catch (SketchesReadOnlyException e) { - //expected - } - } catch (final Exception e) { - throw new RuntimeException(e); + WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2, 1, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + + UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); + for (int i = 0; i < u; i++) { sketch.update(i); } + double est1 = sketch.getEstimate(); + byte[] ser = sketch.toByteArray(); + Memory mem = Memory.wrap(ser); + UpdateSketch roSketch = (UpdateSketch) Sketches.wrapSketch(mem); + double est2 = roSketch.getEstimate(); + assertEquals(est2, est1); + try { + roSketch.rebuild(); + fail(); + } catch (SketchesReadOnlyException e) { + //expected + } + try { + roSketch.reset(); + fail(); + } catch (SketchesReadOnlyException e) { + //expected } - } - @Test public void printlnTest() { println("PRINTING: "+this.getClass().getName()); diff --git a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java index ee0fed44c..fe76240a7 100644 --- a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java +++ b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java @@ -197,24 +197,21 @@ public void checkMoveAndResize() { final int k = 1 << 12; final int u = 2 * k; final int bytes = Sketches.getMaxUpdateSketchBytes(k); - try (ResourceScope scope = ResourceScope.newConfinedScope()) { + ResourceScope scope = ResourceScope.newConfinedScope(); - final WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2, 1, scope, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); - assertTrue(sketch.isSameResource(wmem)); + final WritableMemory wmem = WritableMemory.allocateDirect(bytes / 2, 1, scope, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + final UpdateSketch sketch = Sketches.updateSketchBuilder().setNominalEntries(k).build(wmem); + assertTrue(sketch.isSameResource(wmem)); - final WritableMemory wmem2 = WritableMemory.allocateDirect(bytes / 2, 1, scope, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); - final Union union = SetOperation.builder().buildUnion(wmem2); - assertTrue(union.isSameResource(wmem2)); + final WritableMemory wmem2 = WritableMemory.allocateDirect(bytes / 2, 1, scope, ByteOrder.nativeOrder(), new DefaultMemoryRequestServer()); + final Union union = SetOperation.builder().buildUnion(wmem2); + assertTrue(union.isSameResource(wmem2)); - for (int i = 0; i < u; i++) { union.update(i); } - assertFalse(union.isSameResource(wmem)); + for (int i = 0; i < u; i++) { union.update(i); } + assertFalse(union.isSameResource(wmem)); - final Union union2 = SetOperation.builder().buildUnion(); //on-heap union - assertFalse(union2.isSameResource(wmem2)); //obviously not - } catch (final Exception e) { - throw new RuntimeException(e); - } + final Union union2 = SetOperation.builder().buildUnion(); //on-heap union + assertFalse(union2.isSameResource(wmem2)); //obviously not } @Test From b984e6523343ad6b5143c6b17b461d91a57c45e5 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 20 Jun 2024 08:33:58 -0700 Subject: [PATCH 3/3] Changes required for ds-java to move to java17 and also depend on ds-Memory 4.0.0, which is built around FFM/Panama. --- pom.xml | 157 +++++------------- .../quantiles/DirectUpdateDoublesSketch.java | 3 +- .../filters/bloomfilter/BloomFilterTest.java | 3 +- .../hll/DirectCouponListTest.java | 3 +- .../quantiles/DebugUnionTest.java | 3 +- .../datasketches/theta/CompactSketchTest.java | 3 +- .../theta/DirectQuickSelectSketchTest.java | 5 +- .../theta/HeapifyWrapSerVer1and2Test.java | 2 +- .../datasketches/theta/UnionImplTest.java | 3 +- 9 files changed, 48 insertions(+), 134 deletions(-) diff --git a/pom.xml b/pom.xml index e43a426ba..949152680 100644 --- a/pom.xml +++ b/pom.xml @@ -18,19 +18,11 @@ specific language governing permissions and limitations under the License. --> - + 4.0.0 - - org.apache - apache - 32 - - org.apache.datasketches datasketches-java 6.1.0-SNAPSHOT @@ -66,8 +58,8 @@ under the License. - + + GitHub https://github.com/apache/${project.artifactId}/issues @@ -83,9 +75,7 @@ under the License. - 2.2.0 - 7.10.0 @@ -95,52 +85,51 @@ under the License. check_cpp_historical_files - 3.6.3 + 3.6.3 17 ${java.version} ${java.version} - -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 + -Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 --add-modules=jdk.incubator.foreign UTF-8 ${charset.encoding} ${charset.encoding} ${charset.encoding} yyyy-MM-dd'T'HH-mm-ss'Z' - - 2.16.2 - - 3.7.1 - 3.13.0 - 3.1.1 - 3.4.1 - 3.2.3 - 3.4.0 - 3.6.3 - 3.0.1 - 3.2.0 - 3.3.1 - 3.2.5 + 3.7.1 + 3.13.0 + 3.1.1 + 3.4.1 + 3.2.3 + 3.4.0 + 3.7.0 + 3.0.1 + 3.2.0 + 3.3.1 + 3.2.5 + 3.1.2 - 0.16.1 + 0.16.1 0.8.12 4.3.0 + + 2.16.2 + - 1.0.0 + 1.0.0 4.9.10 - - + org.testng @@ -170,6 +159,11 @@ under the License. org.apache.maven.plugins maven-compiler-plugin ${maven-compiler-plugin.version} + + + --add-modules=jdk.incubator.foreign + + @@ -193,7 +187,7 @@ under the License. - [1.8.0,9.0),[11.0,12.0), [17,18) + [17,18) [${maven.version},) @@ -212,7 +206,6 @@ under the License. - org.apache.maven.plugins maven-jar-plugin ${maven-jar-plugin.version} @@ -235,12 +228,15 @@ under the License. - org.apache.maven.plugins maven-javadoc-plugin ${maven-javadoc-plugin.version} true + public + + --add-modules=jdk.incubator.foreign + @@ -253,14 +249,12 @@ under the License. - org.apache.maven.plugins maven-release-plugin ${maven-release-plugin.version} - org.apache.maven.plugins maven-source-plugin ${maven-source-plugin.version} @@ -283,11 +277,11 @@ under the License. - org.apache.maven.plugins maven-surefire-plugin ${maven-surefire-failsafe-plugins.version} + --add-modules=jdk.incubator.foreign false false true @@ -297,7 +291,6 @@ under the License. - org.apache.rat apache-rat-plugin ${apache-rat-plugin.version} @@ -423,44 +416,8 @@ under the License. - - - - only-eclipse - - - m2e.version - - - - - - - org.apache.maven.plugins - maven-remote-resources-plugin - ${maven-remote-resources-plugin.version} - - - process-resource-bundles - none - - - - - - - - org.apache.maven.plugins - maven-remote-resources-plugin - - - - + + This is performed from a script outside Maven apache-release @@ -624,44 +582,7 @@ under the License. - - - - java8only - - [1.8,1.9),8 - - - - - java11plus - - [11,14) - - - 8 - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - @{argLine} - --add-exports java.base/jdk.internal.misc=ALL-UNNAMED - --add-exports java.base/jdk.internal.ref=ALL-UNNAMED - --add-opens java.base/java.nio=ALL-UNNAMED - --add-opens java.base/sun.nio.ch=ALL-UNNAMED - - ${testng.generate-java-files},${testng.check-cpp-files} - - - - - - + --> generate-java-files diff --git a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java index 96c01d931..0998223e8 100644 --- a/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/DirectUpdateDoublesSketch.java @@ -268,8 +268,7 @@ private WritableMemory growCombinedMemBuffer(final int itemSpaceNeeded) { final WritableMemory newMem = memReqSvr.request(mem_, needBytes); mem_.copyTo(0, newMem, 0, memBytes); - - memReqSvr.requestClose(mem_, newMem); + if (mem_.isCloseable()) { memReqSvr.requestClose(mem_, newMem); } return newMem; } diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java index 7a0fbb7ff..81f9e0d2e 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterTest.java @@ -57,8 +57,7 @@ public void createNewFilterTest() throws Exception { assertFalse(bf1.isReadOnly()); WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(sizeBytes, - new DefaultMemoryRequestServer())).scope()) { + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(sizeBytes)).scope()) { final BloomFilter bf2 = new BloomFilter(numBits, numHashes, seed, wmem); assertTrue(bf2.isEmpty()); diff --git a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java index c6c006f29..985cdf798 100644 --- a/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java +++ b/src/test/java/org/apache/datasketches/hll/DirectCouponListTest.java @@ -73,8 +73,7 @@ private static void promotions(int lgConfigK, int n, TgtHllType tgtHllType, bool //println("DIRECT"); byte[] barr1; WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes, - new DefaultMemoryRequestServer())).scope()) { + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes)).scope()) { //byte[] byteArr = new byte[bytes]; //WritableMemory wmem = WritableMemory.wrap(byteArr); hllSketch = new HllSketch(lgConfigK, tgtHllType, wmem); diff --git a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java index d19d3b471..b1bd5818d 100644 --- a/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/DebugUnionTest.java @@ -67,8 +67,7 @@ public void test() { DoublesUnion dUnion; DoublesSketch dSketch; WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(10_000_000, - new DefaultMemoryRequestServer())).scope()) { + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(10_000_000)).scope()) { dUnion = DoublesUnion.builder().setMaxK(8).build(wmem); for (int s = 0; s < numSketches; s++) { dUnion.union(sketchArr[s]); } dSketch = dUnion.getResult(); //result is on heap diff --git a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java index 1ad55696c..1e2089b51 100644 --- a/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/CompactSketchTest.java @@ -84,8 +84,7 @@ public void checkHeapifyWrap(int k, int u, boolean ordered) { int bytes = usk.getCompactBytes(); //for Compact WritableMemory directMem; - try (ResourceScope scope = (directMem = WritableMemory.allocateDirect(bytes, - new DefaultMemoryRequestServer())).scope()) { + try (ResourceScope scope = (directMem = WritableMemory.allocateDirect(bytes)).scope()) { /**Via CompactSketch.compact**/ refSk = usk.compact(ordered, directMem); diff --git a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java index 32b9bbb44..63957c3cd 100644 --- a/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java +++ b/src/test/java/org/apache/datasketches/theta/DirectQuickSelectSketchTest.java @@ -683,8 +683,7 @@ public void checkEstModeNativeMemory() { int memCapacity = (k << 4) + (Family.QUICKSELECT.getMinPreLongs() << 3); WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(memCapacity, - new DefaultMemoryRequestServer())).scope()) { + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(memCapacity)).scope()) { UpdateSketch usk = UpdateSketch.builder().setNominalEntries(k).build(wmem); DirectQuickSelectSketch sk1 = (DirectQuickSelectSketch)usk; //for internal checks @@ -930,7 +929,7 @@ private static final int getMaxBytes(int k) { } private static WritableMemory makeNativeMemory(int k) { - return WritableMemory.allocateDirect(getMaxBytes(k), new DefaultMemoryRequestServer()); + return WritableMemory.allocateDirect(getMaxBytes(k)); } } diff --git a/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java b/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java index d3617c67e..6c90eafe4 100644 --- a/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java +++ b/src/test/java/org/apache/datasketches/theta/HeapifyWrapSerVer1and2Test.java @@ -566,7 +566,7 @@ public void checkWrapSketchGivenDifferentSeed() { private static WritableMemory putOffHeap(Memory heapMem) { final long cap = heapMem.getCapacity(); - WritableMemory wmem = WritableMemory.allocateDirect(cap, new DefaultMemoryRequestServer()); + WritableMemory wmem = WritableMemory.allocateDirect(cap); heapMem.copyTo(0, wmem, 0, cap); return wmem; } diff --git a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java index fe76240a7..f28212061 100644 --- a/src/test/java/org/apache/datasketches/theta/UnionImplTest.java +++ b/src/test/java/org/apache/datasketches/theta/UnionImplTest.java @@ -234,8 +234,7 @@ public void checkUnionCompactOrderedSource() { final int bytes = Sketches.getCompactSketchMaxBytes(lgK); WritableMemory wmem; - try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes, - new DefaultMemoryRequestServer())).scope()) { + try (ResourceScope scope = (wmem = WritableMemory.allocateDirect(bytes)).scope()) { final CompactSketch csk = sk.compact(true, wmem); //ordered, direct final Union union = Sketches.setOperationBuilder().buildUnion();