From 3fd70bd4725f0e157a7e9043ca153c252b265f92 Mon Sep 17 00:00:00 2001 From: Mikhail Petrov Date: Sun, 5 May 2024 20:08:58 +0300 Subject: [PATCH 1/3] wip --- .../binary/BinaryArrayIdentityResolver.java | 27 +- .../internal/binary/BinaryObjectImpl.java | 25 +- .../internal/binary/BinaryReaderExImpl.java | 7 +- .../internal/binary/BinaryReaderHandles.java | 5 + .../binary/BinaryReaderHandlesHolder.java | 3 + .../binary/BinaryReaderHandlesHolderImpl.java | 5 + .../ignite/internal/binary/BinaryUtils.java | 25 +- .../internal/binary/ObjectDetachHelper.java | 288 +++++++++ .../internal/binary/RawBytesObjectReader.java | 309 +++++++++ .../streams/BinaryAbstractOutputStream.java | 7 + .../binary/streams/BinaryOutputStream.java | 9 + .../internal/client/thin/ClientUtils.java | 26 +- .../memory/PlatformOutputStreamImpl.java | 5 + .../CrossObjetReferenceSerializationTest.java | 594 ++++++++++++++++++ .../binary/RawBytesObjectReaderTest.java | 262 ++++++++ .../mutabletest/GridBinaryTestClasses.java | 75 +++ .../IgniteBinaryObjectsTestSuite.java | 5 + 17 files changed, 1645 insertions(+), 32 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/binary/RawBytesObjectReader.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/CrossObjetReferenceSerializationTest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/RawBytesObjectReaderTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryArrayIdentityResolver.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryArrayIdentityResolver.java index 14792e5abd58b..13eaab2c23473 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryArrayIdentityResolver.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryArrayIdentityResolver.java @@ -50,30 +50,29 @@ public BinaryArrayIdentityResolver() { /** {@inheritDoc} */ @Override protected int hashCode0(BinaryObject obj) { - int hash = 1; - if (obj instanceof BinaryObjectExImpl) { BinaryObjectExImpl ex = (BinaryObjectExImpl)obj; int start = ex.dataStartOffset(); int end = ex.footerStartOffset(); - if (ex.hasArray()) { - // Handle heap object. - byte[] data = ex.array(); - - for (int i = start; i < end; i++) - hash = 31 * hash + data[i]; - } + if (ex.hasArray()) + return hashCode(ex.array(), start, end); else { // Handle offheap object. + int hash = 1; + long ptr = ex.offheapAddress(); for (int i = start; i < end; i++) hash = 31 * hash + BinaryPrimitives.readByte(ptr, i); + + return hash; } } else if (obj instanceof BinaryEnumObjectImpl) { + int hash = 1; + int ord = obj.enumOrdinal(); // Construct hash as if it was an int serialized in little-endian form. @@ -81,10 +80,20 @@ else if (obj instanceof BinaryEnumObjectImpl) { hash = 31 * hash + (ord & 0x0000FF00); hash = 31 * hash + (ord & 0x00FF0000); hash = 31 * hash + (ord & 0xFF000000); + + return hash; } else throw new BinaryObjectException("Array identity resolver cannot be used with provided BinaryObject " + "implementation: " + obj.getClass().getName()); + } + + /** */ + public int hashCode(byte[] data, int startPos, int endPos) { + int hash = 1; + + for (int i = startPos; i < endPos; i++) + hash = 31 * hash + data[i]; return hash; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java index e10b33168fbc4..31f263d9c24fb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java @@ -35,6 +35,8 @@ import org.apache.ignite.internal.GridDirectTransient; import org.apache.ignite.internal.IgniteCodeGeneratingFail; import org.apache.ignite.internal.binary.streams.BinaryHeapInputStream; +import org.apache.ignite.internal.binary.streams.BinaryHeapOutputStream; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.CacheObjectAdapter; import org.apache.ignite.internal.processors.cache.CacheObjectContext; @@ -282,16 +284,33 @@ private byte[] valueBytesFromArray(CacheObjectValueContext ctx) { * @return Detached binary object. */ public BinaryObjectImpl detach() { + return detach(true); + } + + /** */ + public BinaryObjectImpl detach(boolean resolveCrossObjectReferences) { if (!detachAllowed || detached()) return this; int len = length(); - byte[] arr0 = new byte[len]; + if (resolveCrossObjectReferences) { + ObjectDetachHelper detachHelper = ObjectDetachHelper.create(arr, start); + + if (detachHelper.isCrossObjectReferencesDetected()) { + try (BinaryOutputStream out = new BinaryHeapOutputStream(2 * len)) { + detachHelper.detach(out); + + return new BinaryObjectImpl(ctx, out.arrayCopy(), 0); + } + } + } + + byte[] detachedData = new byte[len]; - U.arrayCopy(arr, start, arr0, 0, len); + U.arrayCopy(arr, start, detachedData, 0, len); - return new BinaryObjectImpl(ctx, arr0, 0); + return new BinaryObjectImpl(ctx, detachedData, 0); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java index a88ed7902a4d4..57e6d88198a12 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java @@ -1338,7 +1338,7 @@ float readFloat(int fieldId) throws BinaryObjectException { /** {@inheritDoc} */ @Nullable @Override public Object readObjectDetached(boolean deserialize) throws BinaryObjectException { - return BinaryUtils.unmarshal(in, ctx, ldr, this, true, deserialize); + return BinaryUtils.unmarshal(in, ctx, ldr, new BinaryReaderHandlesHolderImpl(), true, deserialize); } /** {@inheritDoc} */ @@ -2369,6 +2369,11 @@ private void streamPositionRandom(int pos) { return in.remaining(); } + /** {@inheritDoc} */ + @Override public boolean isEmpty() { + return hnds == null || hnds.isEmpty(); + } + /** {@inheritDoc} */ @Override public void close() throws IOException { // No-op. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandles.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandles.java index 72d054502f418..9582177dcf3cc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandles.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandles.java @@ -103,4 +103,9 @@ public void put(int pos, Object obj) { data0.put(pos, obj); } } + + /** */ + public boolean isEmpty() { + return mode == MODE_EMPTY; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolder.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolder.java index 48c9e8ebdd9cb..a60276ced82a1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolder.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolder.java @@ -43,4 +43,7 @@ public interface BinaryReaderHandlesHolder { * @return Handles. */ public BinaryReaderHandles handles(); + + /** */ + public boolean isEmpty(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolderImpl.java index 95e5ff52d798f..a1ee6a91a1600 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolderImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolderImpl.java @@ -41,4 +41,9 @@ public class BinaryReaderHandlesHolderImpl implements BinaryReaderHandlesHolder return hnds; } + + /** {@inheritDoc} */ + @Override public boolean isEmpty() { + return hnds == null || hnds.isEmpty(); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index f513d3ddd2f9a..e8e898648e574 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -836,6 +836,20 @@ public static int length(BinaryPositionReadable in, int start) { return in.readIntPositioned(start + GridBinaryMarshaller.TOTAL_LEN_POS); } + /** */ + public static int dataStartRelative(BinaryPositionReadable in, int start) { + int typeId = in.readIntPositioned(start + GridBinaryMarshaller.TYPE_ID_POS); + + if (typeId == GridBinaryMarshaller.UNREGISTERED_TYPE_ID) { + // Gets the length of the type name which is stored as string. + int len = in.readIntPositioned(start + GridBinaryMarshaller.DFLT_HDR_LEN + /** object type */1); + + return GridBinaryMarshaller.DFLT_HDR_LEN + /** object type */1 + /** string length */ 4 + len; + } + else + return GridBinaryMarshaller.DFLT_HDR_LEN; + } + /** * Get footer start of the object. * @@ -1929,10 +1943,11 @@ public static Object doReadOptimized(BinaryInputStream in, BinaryContext ctx, @N BinaryObjectExImpl po; if (detach) { - // In detach mode we simply copy object's content. - in.position(start); + BinaryObjectImpl binObj = new BinaryObjectImpl(ctx, in.array(), start); + + binObj.detachAllowed(true); - po = new BinaryObjectImpl(ctx, in.readByteArray(len), 0); + po = binObj.detach(!handles.isEmpty()); } else { if (in.offheapPointer() == 0) @@ -1940,10 +1955,10 @@ public static Object doReadOptimized(BinaryInputStream in, BinaryContext ctx, @N else po = new BinaryObjectOffheapImpl(ctx, in.offheapPointer(), start, in.remaining() + in.position()); - - in.position(start + po.length()); } + in.position(start + len); + handles.setHandle(po, start); return po; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java new file mode 100644 index 0000000000000..6d19f020bcea6 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeSet; +import org.apache.ignite.internal.binary.streams.BinaryHeapInputStream; +import org.apache.ignite.internal.binary.streams.BinaryInputStream; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.jetbrains.annotations.NotNull; + +import static org.apache.ignite.internal.binary.BinaryArrayIdentityResolver.instance; +import static org.apache.ignite.internal.binary.BinaryUtils.dataStartRelative; +import static org.apache.ignite.internal.binary.BinaryUtils.footerStartAbsolute; +import static org.apache.ignite.internal.binary.BinaryUtils.length; +import static org.apache.ignite.internal.binary.BinaryUtils.rawOffsetAbsolute; +import static org.apache.ignite.internal.binary.GridBinaryMarshaller.HASH_CODE_POS; + +/** */ +public class ObjectDetachHelper { + /** */ + private final RawBytesObjectReader reader; + + /** */ + private final int rootObjStartPos; + + /** */ + private final Deque objStack = new ArrayDeque<>(); + + /** */ + private final Map> crossObjReferences = new HashMap<>(); + + /** */ + private final TreeSet rehashRequireObjects = new TreeSet<>(); + + /** */ + private ObjectDetachHelper(BinaryInputStream in) { + reader = new RawBytesObjectReader(in); + + rootObjStartPos = in.position(); + } + + /** */ + static ObjectDetachHelper create(byte[] data, int offset) { + ObjectDetachHelper res = new ObjectDetachHelper(BinaryHeapInputStream.create(data, offset)); + + res.readNextObject(); + + return res; + } + + /** */ + public boolean isCrossObjectReferencesDetected() { + return !crossObjReferences.isEmpty(); + } + + /** */ + public int detach(BinaryOutputStream writer) { + reader.position(rootObjStartPos); + + int writerRootObjStartPos = writer.position(); + + reader.copyObject(writer); + + crossObjReferences.forEach((readerObjStartPos, handleOffsets) -> { + int writerObjStartPos = copyObject(readerObjStartPos, writer); + + for (int handleOffset : handleOffsets) { + int writerHandleStartPos = writerRootObjStartPos + handleOffset; + + writer.unsafeWriteInt(writerHandleStartPos + /* skip handle type */ 1, writerHandleStartPos - writerObjStartPos); + } + }); + + for (ObjectDescriptor objDesc : rehashRequireObjects.descendingSet()) + overrideHash(writer, writerRootObjStartPos, objDesc); + + return writerRootObjStartPos; + } + + /** */ + private void readNextObject() { + int objStartPos = reader.position(); + + byte objType = reader.readBytePositioned(objStartPos); + + switch (objType) { + case GridBinaryMarshaller.OBJ: { + int objDataStartPos = objStartPos + dataStartRelative(reader, objStartPos); + int objDataEndPos = rawOffsetAbsolute(reader, objStartPos); + int objFooterStartPos = footerStartAbsolute(reader, objStartPos); + int objEndPos = objStartPos + length(reader, objStartPos); + + reader.position(objDataStartPos); + + objStack.push(new ObjectDescriptor( + rootStartOffset(objStartPos), + rootStartOffset(objDataStartPos), + rootStartOffset(objFooterStartPos)) + ); + + while (reader.position() < objDataEndPos) + readNextObject(); + + objStack.pop(); + + reader.position(objEndPos); + + break; + } + + case GridBinaryMarshaller.HANDLE: { + reader.skipBytes(1); // Object type. + + int offset = reader.readInt(); + + int handleObjPos = objStartPos - offset; + + if (handleObjPos < rootObjStartPos) + saveCrossObjectReferenceData(handleObjPos, objStartPos); + + break; + } + + case GridBinaryMarshaller.OBJ_ARR: { + reader.skipBytes(1); // Object type. + + reader.skipTypeId(); + + int size = reader.readInt(); + + readCortege(size); + + break; + } + + case GridBinaryMarshaller.COL: { + reader.skipBytes(1); // Object type. + + int size = reader.readInt(); + + reader.skipBytes(1); // Collection type. + + readCortege(size); + + break; + } + + case GridBinaryMarshaller.MAP: { + reader.skipBytes(1); // Object type. + + int size = reader.readInt() * 2; + + reader.skipBytes(1); // Map type. + + readCortege(size); + + break; + } + + default: { + reader.skipObject(); + } + } + } + + /** */ + private void readCortege(int cortegeSize) { + for (int elemIdx = 0; elemIdx < cortegeSize; elemIdx++) + readNextObject(); + } + + /** */ + private void saveCrossObjectReferenceData(int objPos, int handlePos) { + crossObjReferences.computeIfAbsent(objPos, k -> new ArrayList<>()).add(rootStartOffset(handlePos)); + + rehashRequireObjects.addAll(objStack); + } + + /** */ + private int rootStartOffset(int pos) { + assert pos - rootObjStartPos >= 0; + + return pos - rootObjStartPos; + } + + /** */ + private void overrideHash(BinaryOutputStream writer, int writerRootObjStartPos, ObjectDescriptor objDesc) { + int hashCode = instance().hashCode( + writer.array(), + writerRootObjStartPos + objDesc.objectDataStartOffset(), + writerRootObjStartPos + objDesc.objectFooterStartOffset() + ); + + writer.unsafeWriteInt(writerRootObjStartPos + objDesc.objectStartOffset() + HASH_CODE_POS, hashCode); + } + + /** */ + private int copyObject(int readerObjStartPos, BinaryOutputStream writer) { + int readerRetPos = reader.position(); + int writerObjStartPos = writer.position(); + + reader.position(readerObjStartPos); + + ObjectDetachHelper detachHelper = create(reader.array(), readerObjStartPos); + + if (detachHelper.isCrossObjectReferencesDetected()) + detachHelper.detach(writer); + else + reader.copyObject(writer); + + reader.position(readerRetPos); + + return writerObjStartPos; + } + + /** */ + private static class ObjectDescriptor implements Comparable { + /** */ + private final int objStartOffset; + + /** */ + private final int objDataStartOffset; + + /** */ + private final int objFooterStartOffset; + + /** */ + public ObjectDescriptor(int objStartOffset, int objDataStartOffset, int objFooterStartOffset) { + this.objStartOffset = objStartOffset; + this.objDataStartOffset = objDataStartOffset; + this.objFooterStartOffset = objFooterStartOffset; + } + + /** */ + public int objectFooterStartOffset() { + return objFooterStartOffset; + } + + /** */ + public int objectDataStartOffset() { + return objDataStartOffset; + } + + /** */ + public int objectStartOffset() { + return objStartOffset; + } + + /** {@inheritDoc} */ + @Override public int compareTo(@NotNull ObjectDetachHelper.ObjectDescriptor other) { + return Integer.compare(objStartOffset, other.objDataStartOffset); + } + + /** */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (!(o instanceof ObjectDescriptor)) + return false; + + ObjectDescriptor that = (ObjectDescriptor)o; + + return objStartOffset == that.objStartOffset; + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/RawBytesObjectReader.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/RawBytesObjectReader.java new file mode 100644 index 0000000000000..d3771c75d92e9 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/RawBytesObjectReader.java @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.internal.binary.streams.BinaryInputStream; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; + +/** */ +public class RawBytesObjectReader implements BinaryPositionReadable { + /** */ + private final BinaryInputStream in; + + /** */ + public RawBytesObjectReader(BinaryInputStream in) { + this.in = in; + } + + /** */ + public byte[] readObject() { + int startPos = in.position(); + + skipObject(); + + int endPos = in.position(); + + in.position(startPos); + + return in.readByteArray(endPos - startPos); + } + + /** */ + public void copyObject(BinaryOutputStream out) { + int startPos = in.position(); + + skipObject(); + + out.writeByteArray(in.array(), startPos, in.position() - startPos); + } + + /** */ + public int readInt() { + return in.readInt(); + } + + /** */ + public void skipObject() { + int objStartPos = in.position(); + + byte type = in.readByte(); + + switch (type) { + case GridBinaryMarshaller.NULL: + break; + + case GridBinaryMarshaller.OBJ: + skipBytes(BinaryUtils.length(in, objStartPos) - /** Object type. */ Byte.BYTES); + + break; + + case GridBinaryMarshaller.BINARY_OBJ: + skipBytes(in.readInt()); + + skipBytes(Integer.BYTES); // Offset. + + break; + + case GridBinaryMarshaller.BYTE: + case GridBinaryMarshaller.BOOLEAN: + skipBytes(Byte.BYTES); + + break; + + case GridBinaryMarshaller.CHAR: + skipBytes(Character.BYTES); + + break; + + case GridBinaryMarshaller.SHORT: + skipBytes(Short.BYTES); + + break; + + case GridBinaryMarshaller.FLOAT: + skipBytes(Float.BYTES); + + break; + + case GridBinaryMarshaller.HANDLE: + case GridBinaryMarshaller.INT: + skipBytes(Integer.BYTES); + + break; + + case GridBinaryMarshaller.ENUM: + case GridBinaryMarshaller.BINARY_ENUM: { + skipTypeId(); + + skipBytes(Integer.BYTES); // Ordinal. + + break; + } + + case GridBinaryMarshaller.LONG: + case GridBinaryMarshaller.DATE: + case GridBinaryMarshaller.TIME: + skipBytes(Long.BYTES); + + break; + + case GridBinaryMarshaller.DOUBLE: + skipBytes(Double.BYTES); + + break; + + case GridBinaryMarshaller.OPTM_MARSH: + case GridBinaryMarshaller.STRING: + skipBytes(in.readInt()); + + break; + + case GridBinaryMarshaller.DECIMAL: + skipBytes(Integer.BYTES); // Scale. + skipBytes(in.readInt()); + + break; + + case GridBinaryMarshaller.UUID: + skipBytes(Long.BYTES + Long.BYTES); + + break; + + case GridBinaryMarshaller.TIMESTAMP: + skipBytes(Long.BYTES + Integer.BYTES); + + break; + + case GridBinaryMarshaller.BYTE_ARR: + case GridBinaryMarshaller.BOOLEAN_ARR: + skipBytes(in.readInt() * Byte.BYTES); + + break; + + case GridBinaryMarshaller.CHAR_ARR: + skipBytes(in.readInt() * Character.BYTES); + + break; + + case GridBinaryMarshaller.SHORT_ARR: + skipBytes(in.readInt() * Short.BYTES); + + break; + + case GridBinaryMarshaller.INT_ARR: + skipBytes(in.readInt() * Integer.BYTES); + + break; + + case GridBinaryMarshaller.FLOAT_ARR: + skipBytes(in.readInt() * Float.BYTES); + + break; + + case GridBinaryMarshaller.LONG_ARR: + skipBytes(in.readInt() * Long.BYTES); + + break; + + case GridBinaryMarshaller.DOUBLE_ARR: + skipBytes(in.readInt() * Double.BYTES); + + break; + + case GridBinaryMarshaller.DECIMAL_ARR: + case GridBinaryMarshaller.DATE_ARR: + case GridBinaryMarshaller.TIMESTAMP_ARR: + case GridBinaryMarshaller.TIME_ARR: + case GridBinaryMarshaller.UUID_ARR: + case GridBinaryMarshaller.STRING_ARR: { + skipCortege(); + + break; + } + + case GridBinaryMarshaller.ENUM_ARR: + case GridBinaryMarshaller.OBJ_ARR: { + skipTypeId(); + + skipCortege(); + + break; + } + + case GridBinaryMarshaller.COL: { + int size = in.readInt(); + + skipBytes(Byte.BYTES); // Collection type. + + skipCortege(size); + + break; + } + + case GridBinaryMarshaller.MAP: { + int size = in.readInt() * 2; + + skipBytes(Byte.BYTES); // Map type. + + skipCortege(size); + + break; + } + + case GridBinaryMarshaller.CLASS: { + skipTypeId(); + + break; + } + + case GridBinaryMarshaller.PROXY: { + int size = in.readInt(); + + for (int i = 0; i < size; i++) + skipTypeId(); // Interface type. + + skipObject(); + + break; + } + + default: + throw new BinaryObjectException("Unsupported binary type [type=" + type + ']'); + } + } + + /** {@inheritDoc} */ + @Override public byte readBytePositioned(int pos) { + return in.readBytePositioned(pos); + } + + /** {@inheritDoc} */ + @Override public short readShortPositioned(int pos) { + return in.readShortPositioned(pos); + } + + /** {@inheritDoc} */ + @Override public int readIntPositioned(int pos) { + return in.readIntPositioned(pos); + } + + /** */ + public int position() { + return in.position(); + } + + /** */ + public void position(int position) { + in.position(position); + } + + /** */ + public void skipBytes(int count) { + int curPos = in.position(); + + in.position(curPos + count); + } + + /** */ + public void skipTypeId() { + int typeId = in.readInt(); + + if (typeId == GridBinaryMarshaller.UNREGISTERED_TYPE_ID) { + skipBytes(Byte.BYTES); // String type. + + skipBytes(in.readInt()); + } + } + + /** */ + public byte[] array() { + return in.array(); + } + + /** */ + private void skipCortege() { + skipCortege(in.readInt()); + } + + /** */ + private void skipCortege(int size) { + for (int i = 0; i < size; i++) + skipObject(); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryAbstractOutputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryAbstractOutputStream.java index e87309dab1a9c..165bb8801e288 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryAbstractOutputStream.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryAbstractOutputStream.java @@ -52,6 +52,13 @@ public abstract class BinaryAbstractOutputStream extends BinaryAbstractStream copyAndShift(val, GridUnsafe.BYTE_ARR_OFF, val.length); } + /** {@inheritDoc} */ + @Override public void writeByteArray(byte[] val, int off, int len) { + ensureCapacity(pos + len); + + copyAndShift(val, GridUnsafe.BYTE_ARR_OFF + off, len); + } + /** {@inheritDoc} */ @Override public void writeBoolean(boolean val) { writeByte(val ? BYTE_ONE : BYTE_ZERO); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOutputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOutputStream.java index 84941203acd72..16423e2153eb2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOutputStream.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryOutputStream.java @@ -35,6 +35,15 @@ public interface BinaryOutputStream extends BinaryStream, AutoCloseable { */ public void writeByteArray(byte[] val); + /** + * Write byte array. + * + * @param val Byte array. + * @param off Offset. + * @param len Array length. + */ + public void writeByteArray(byte[] val, int off, int len); + /** * Write boolean value. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java index d7e30463ed3d9..d333d6ad80203 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java @@ -573,29 +573,27 @@ T readObject(BinaryInputStream in, boolean keepBinary, Class clazz) { if (keepBinary) return (T)marsh.unmarshal(in); else { - BinaryReaderHandles hnds = new BinaryReaderHandles(); - - return (T)unwrapBinary(marsh.deserialize(in, hnds), hnds, clazz); + return (T)unwrapBinary(marsh.deserialize(in, new BinaryReaderHandles()), clazz); } } /** * Unwrap binary object. */ - private Object unwrapBinary(Object obj, BinaryReaderHandles hnds, Class clazz) { + private Object unwrapBinary(Object obj, Class clazz) { if (obj instanceof BinaryObjectImpl) { BinaryObjectImpl obj0 = (BinaryObjectImpl)obj; - return marsh.deserialize(BinaryHeapInputStream.create(obj0.array(), obj0.start()), hnds); + return marsh.deserialize(BinaryHeapInputStream.create(obj0.array(), obj0.start()), new BinaryReaderHandles()); } else if (obj instanceof BinaryObject) return ((BinaryObject)obj).deserialize(); else if (BinaryUtils.knownCollection(obj)) - return unwrapCollection((Collection)obj, hnds); + return unwrapCollection((Collection)obj); else if (BinaryUtils.knownMap(obj)) - return unwrapMap((Map)obj, hnds); + return unwrapMap((Map)obj); else if (obj instanceof Object[]) - return unwrapArray((Object[])obj, hnds, clazz); + return unwrapArray((Object[])obj, clazz); else return obj; } @@ -603,11 +601,11 @@ else if (obj instanceof Object[]) /** * Unwrap collection with binary objects. */ - private Collection unwrapCollection(Collection col, BinaryReaderHandles hnds) { + private Collection unwrapCollection(Collection col) { Collection col0 = BinaryUtils.newKnownCollection(col); for (Object obj0 : col) - col0.add(unwrapBinary(obj0, hnds, null)); + col0.add(unwrapBinary(obj0, null)); return (col0 instanceof MutableSingletonList) ? U.convertToSingletonList(col0) : col0; } @@ -615,11 +613,11 @@ private Collection unwrapCollection(Collection col, BinaryReader /** * Unwrap map with binary objects. */ - private Map unwrapMap(Map map, BinaryReaderHandles hnds) { + private Map unwrapMap(Map map) { Map map0 = BinaryUtils.newMap(map); for (Map.Entry e : map.entrySet()) - map0.put(unwrapBinary(e.getKey(), hnds, null), unwrapBinary(e.getValue(), hnds, null)); + map0.put(unwrapBinary(e.getKey(), null), unwrapBinary(e.getValue(), null)); return map0; } @@ -627,7 +625,7 @@ private Map unwrapMap(Map map, BinaryReaderHandl /** * Unwrap array with binary objects. */ - private Object[] unwrapArray(Object[] arr, BinaryReaderHandles hnds, Class arrayClass) { + private Object[] unwrapArray(Object[] arr, Class arrayClass) { if (BinaryUtils.knownArray(arr)) return arr; @@ -638,7 +636,7 @@ private Object[] unwrapArray(Object[] arr, BinaryReaderHandles hnds, Class ar Object[] res = (Object[])Array.newInstance(componentType, arr.length); for (int i = 0; i < arr.length; i++) - res[i] = unwrapBinary(arr[i], hnds, null); + res[i] = unwrapBinary(arr[i], null); return res; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java index 3f5e1d2935646..3fa70540d9ee0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/memory/PlatformOutputStreamImpl.java @@ -59,6 +59,11 @@ public PlatformOutputStreamImpl(PlatformMemory mem) { copyAndShift(val, GridUnsafe.BYTE_ARR_OFF, val.length); } + /** {@inheritDoc} */ + @Override public void writeByteArray(byte[] val, int off, int len) { + copyAndShift(val, GridUnsafe.BYTE_ARR_OFF + off, len); + } + /** {@inheritDoc} */ @Override public void writeBoolean(boolean val) { writeByte(val ? (byte)1 : (byte)0); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/CrossObjetReferenceSerializationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/CrossObjetReferenceSerializationTest.java new file mode 100644 index 0000000000000..7efcf5a6bfb5e --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/CrossObjetReferenceSerializationTest.java @@ -0,0 +1,594 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.binary.BinaryRawReader; +import org.apache.ignite.binary.BinaryRawWriter; +import org.apache.ignite.binary.BinaryReader; +import org.apache.ignite.binary.BinaryWriter; +import org.apache.ignite.binary.Binarylizable; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.IgniteClient; +import org.apache.ignite.configuration.BinaryConfiguration; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.ClientConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.binary.mutabletest.GridBinaryTestClasses.TestObjectAllTypes; +import org.apache.ignite.testframework.junits.WithSystemProperty; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_SORT_OBJECT_FIELDS; + +/** */ +@RunWith(Parameterized.class) +@WithSystemProperty(key = IGNITE_BINARY_SORT_OBJECT_FIELDS, value = "true") +public class CrossObjetReferenceSerializationTest extends GridCommonAbstractTest { + /** */ + private static Ignite srv; + + /** */ + private static IgniteClient cli; + + /** */ + private static IgniteCache srvCache; + + /** */ + private static ClientCache cliCache; + + /** */ + @Parameterized.Parameter + public ObjectType innerObjType; + + /** */ + @Parameterized.Parameter(1) + public ObjectType outerObjType; + + /** */ + @Parameterized.Parameter(2) + public boolean isCompactFooterEnabled; + + /** */ + @Parameterized.Parameter(3) + public SerializationMode serializationMode; + + /** Test parameters. */ + @Parameterized.Parameters(name = "innerObjectType={0}, outerObjectType={1}, isCompactFooterEnabled={2}, serializationMode={3}") + public static Iterable parameters() { + List res = new ArrayList<>(); + + for (ObjectType innerObjType : ObjectType.values()) { + for (ObjectType outerObjType : ObjectType.values()) { + for (boolean isCompactFooterEnabled : new boolean[] {true, false}) { + for (SerializationMode serializationMode : SerializationMode.values()) + res.add(new Object[] {innerObjType, outerObjType, isCompactFooterEnabled, serializationMode}); + } + } + } + + return res; + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setCacheConfiguration(new CacheConfiguration<>(DEFAULT_CACHE_NAME)) + .setBinaryConfiguration(new BinaryConfiguration() + .setCompactFooter(isCompactFooterEnabled)); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + srv = startGrid(0); + cli = Ignition.startClient(new ClientConfiguration() + .setAddresses("127.0.0.1:10800") + .setBinaryConfiguration(new BinaryConfiguration() + .setCompactFooter(isCompactFooterEnabled))); + + srvCache = srv.cache(DEFAULT_CACHE_NAME); + cliCache = cli.cache(DEFAULT_CACHE_NAME); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + cli.close(); + srv.close(); + } + + /** */ + @Test + public void testArray() { + Object outerObj = createObject(outerObjType); + + Object[] arr = new Object[] {createReferencesHolder(outerObj), createReferencesHolder(outerObj)}; + + checkPutGetRemove(arr, arr); + } + + /** */ + @Test + public void testInnerArray() { + Object outerObj = createObject(outerObjType); + + Object[] innerArr = new Object[] {new TestObject(), createReferencesHolder(outerObj)}; + + Object[] arr = new Object[] {createReferencesHolder(outerObj), innerArr}; + + checkPutGetRemove(arr, arr); + } + + /** */ + @Test + public void testCollection() { + Object outerObj = createObject(outerObjType); + + Collection col = new ArrayList<>(); + + col.add(createReferencesHolder(outerObj)); + col.add(createReferencesHolder(outerObj)); + + checkPutGetRemove(col, col); + } + + /** */ + @Test + public void testInnerCollection() { + Object outerObj = createObject(outerObjType); + + Collection col = new ArrayList<>(); + + Collection innerCol = new ArrayList<>(); + + innerCol.add(new TestObject()); + innerCol.add(createReferencesHolder(outerObj)); + + col.add(createReferencesHolder(outerObj)); + col.add(innerCol); + + checkPutGetRemove(col, col); + } + + /** */ + @Test + public void testMapReferenceBetweenKeyAndValue() { + Object outerObj = createObject(outerObjType); + + Map map = new HashMap<>(); + + map.put(createReferencesHolder(outerObj), createReferencesHolder(outerObj)); + + checkPutGetRemove(map, map); + } + + /** */ + @Test + public void testMapReferenceBetweenEntries() { + Object outerObj = createObject(outerObjType); + + Map map = new HashMap<>(); + + map.put(0, createReferencesHolder(outerObj)); + map.put(1, createReferencesHolder(outerObj)); + + checkPutGetRemove(map, map); + } + + /** */ + @Test + public void testMapInnerCollection() { + Object outerObj = createObject(outerObjType); + + Collection col = new ArrayList<>(); + + col.add(createReferencesHolder(outerObj)); + col.add(createReferencesHolder(outerObj)); + + Map map = new HashMap<>(); + + map.put(0, col); + + checkPutGetRemove(map, map); + } + + /** */ + @Test + public void testMapInnerArray() { + Object outerObj = createObject(outerObjType); + + Map map = new HashMap<>(); + + map.put(0, new Object[] {createReferencesHolder(outerObj), createReferencesHolder(outerObj)}); + + checkPutGetRemove(0, map); + } + + /** */ + @Test + public void testConsecutiveCrossObjectReferences() { + Object outerObj = createObject(outerObjType); + + Object holder = createReferencesHolder(outerObj); + + Object enclosingHolder = createReferencesHolder(holder); + + Object doubleEnclosingHolder = createReferencesHolder(enclosingHolder); + + Object[] arr = new Object[] {createReferencesHolder(outerObj), holder, enclosingHolder, doubleEnclosingHolder}; + + checkPutGetRemove(arr, arr); + } + + /** */ + @Test + public void testMultipleCrossObjectReferences() { + Object firstOuterObj = createObject(outerObjType); + Object secondOuterObj = createObject(outerObjType); + + Object[] arr = new Object[] { + createReferencesHolder(new Object[] {firstOuterObj, secondOuterObj}), + createReferencesHolder(new Object[] {firstOuterObj, secondOuterObj}) + }; + + checkPutGetRemove(arr, arr); + } + + /** */ + private Object createReferencesHolder(Object outerObj) { + switch (serializationMode) { + case RAW: + return new RawObject(createObject(innerObjType), outerObj); + case MIXED: + return new MixedObject(createObject(innerObjType), outerObj); + case SCHEMA: + return new SchemaObject(createObject(innerObjType), outerObj); + default: + throw new IllegalStateException(); + } + } + + /** */ + private Object createObject(ObjectType type) { + switch (type) { + case OBJECT: { + return new TestObject(); + } + + case ARRAY: { + TestObjectAllTypes obj = new TestObject(); + + return new Object[] {obj, obj}; + } + + case COLLECTION: { + TestObjectAllTypes obj = new TestObject(); + + Collection col = new ArrayList<>(); + + col.add(obj); + col.add(obj); + + return col; + } + + case MAP: { + TestObjectAllTypes obj = new TestObject(); + + Map map = new HashMap<>(); + + map.put(0, obj); + map.put(1, obj); + + return map; + } + + default: + throw new IllegalStateException(); + } + } + + /** */ + private void checkPutGetRemove(Object key, Object val) { + srvCache.put(key, val); + + assertDeepEquals(val, srvCache.get(key)); +// assertDeepEquals(val, cliCache.get(key)); + + srvCache.remove(key); + + assertNull(srvCache.get(key)); +// assertNull(cliCache.get(key)); + + cliCache.put(key, val); + + assertDeepEquals(val, cliCache.get(key)); +// assertDeepEquals(val, srvCache.get(key)); + + cliCache.remove(key); + + assertNull(srvCache.get(key)); + assertNull(cliCache.get(key)); + } + + /** */ + private static void assertDeepEquals(Object exp, Object actual) { + assertTrue(deepEquals(exp, actual)); + } + + /** */ + private static boolean deepEquals(Object lhs, Object rhs) { + if (lhs instanceof Map && rhs instanceof Map) { + Map lhsMap = (Map)lhs; + Map rhsMap = (Map)rhs; + + assertEquals(lhsMap.size(), rhsMap.size()); + + return lhsMap.entrySet().stream().allMatch(e -> deepEquals(e.getValue(), rhsMap.get(e.getKey()))); + } + else if (lhs instanceof List && rhs instanceof List) { + List lhsList = (List)lhs; + List rhsList = (List)rhs; + + assertEquals(lhsList.size(), rhsList.size()); + + boolean res = true; + + for (int i = 0; i < lhsList.size(); i++) { + if (!deepEquals(lhsList.get(i), rhsList.get(i))) { + res = false; + + break; + } + } + + return res; + } + else + return Objects.deepEquals(lhs, rhs); + } + + /** */ + private static int hashCodeArraysAware(Object obj) { + return obj != null && obj.getClass().isArray() ? Arrays.deepHashCode((Object[])obj) : Objects.hash(obj); + } + + /** */ + public enum ObjectType { + /** */ + OBJECT, + + /** */ + ARRAY, + + /** */ + COLLECTION, + + /** */ + MAP + } + + /** */ + public enum SerializationMode { + /** */ + RAW, + + /** */ + MIXED, + + /** */ + SCHEMA + } + + /** */ + private static class MixedObject extends SchemaObject implements Binarylizable { + /** */ + public MixedObject() { + // No-op. + } + + /** */ + public MixedObject(Object innerObj, Object outerObj) { + super(innerObj, outerObj); + } + + /** {@inheritDoc} */ + @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException { + writer.writeObject("aWrapperOfOuterRefToReplaceWithObj", aWrapperOfOuterRefToReplaceWithObj); + writer.writeObject("bInnerObj", bInnerObj); + writer.writeObject("cRefToOuterObjToReplaceWithInnerRef", cRefToOuterObjToReplaceWithInnerRef); + writer.writeObject("dRefToInnerObjToRecalculate", dRefToInnerObjToRecalculate); + + BinaryRawWriter rawWriter = writer.rawWriter(); + + rawWriter.writeByte(eInnerBytePrimitive); + } + + /** {@inheritDoc} */ + @Override public void readBinary(BinaryReader reader) throws BinaryObjectException { + aWrapperOfOuterRefToReplaceWithObj = reader.readObject("aWrapperOfOuterRefToReplaceWithObj"); + bInnerObj = reader.readObject("bInnerObj"); + cRefToOuterObjToReplaceWithInnerRef = reader.readObject("cRefToOuterObjToReplaceWithInnerRef"); + dRefToInnerObjToRecalculate = reader.readObject("dRefToInnerObjToRecalculate"); + + BinaryRawReader rawReader = reader.rawReader(); + + eInnerBytePrimitive = rawReader.readByte(); + } + } + + /** */ + private static class RawObject extends SchemaObject implements Binarylizable { + /** */ + public RawObject() { + // No-op. + } + + /** */ + public RawObject(Object innerObj, Object outerObj) { + super(innerObj, outerObj); + + // Reference recalculation for objects written with Raw Binary Writer currently is not supported. + aWrapperOfOuterRefToReplaceWithObj = null; + cRefToOuterObjToReplaceWithInnerRef = null; + } + + /** {@inheritDoc} */ + @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException { + BinaryRawWriter rawWriter = writer.rawWriter(); + + rawWriter.writeObject(aWrapperOfOuterRefToReplaceWithObj); + rawWriter.writeObject(bInnerObj); + rawWriter.writeObject(cRefToOuterObjToReplaceWithInnerRef); + rawWriter.writeObject(dRefToInnerObjToRecalculate); + rawWriter.writeByte(eInnerBytePrimitive); + } + + /** {@inheritDoc} */ + @Override public void readBinary(BinaryReader reader) throws BinaryObjectException { + BinaryRawReader rawReader = reader.rawReader(); + + aWrapperOfOuterRefToReplaceWithObj = rawReader.readObject(); + bInnerObj = rawReader.readObject(); + cRefToOuterObjToReplaceWithInnerRef = rawReader.readObject(); + dRefToInnerObjToRecalculate = rawReader.readObject(); + eInnerBytePrimitive = rawReader.readByte(); + } + } + + /** */ + private static class SchemaObject { + /** */ + protected Object aWrapperOfOuterRefToReplaceWithObj; + + /** */ + protected Object bInnerObj; + + /** */ + protected Object cRefToOuterObjToReplaceWithInnerRef; + + /** */ + protected Object dRefToInnerObjToRecalculate; + + /** */ + protected byte eInnerBytePrimitive; + + /** */ + public SchemaObject() { + // No-op. + } + + /** */ + public SchemaObject(Object innerObj, Object outerObj) { + aWrapperOfOuterRefToReplaceWithObj = new ComplexWrapper(outerObj); + + bInnerObj = innerObj; + + cRefToOuterObjToReplaceWithInnerRef = outerObj; + + dRefToInnerObjToRecalculate = innerObj; + + eInnerBytePrimitive = 127; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (!(o instanceof SchemaObject)) + return false; + + SchemaObject that = (SchemaObject)o; + + return deepEquals(aWrapperOfOuterRefToReplaceWithObj, that.aWrapperOfOuterRefToReplaceWithObj) + && deepEquals(bInnerObj, that.bInnerObj) + && deepEquals(cRefToOuterObjToReplaceWithInnerRef, that.cRefToOuterObjToReplaceWithInnerRef) + && deepEquals(dRefToInnerObjToRecalculate, that.dRefToInnerObjToRecalculate) + && eInnerBytePrimitive == that.eInnerBytePrimitive; + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = 1; + + res = 31 * res + hashCodeArraysAware(aWrapperOfOuterRefToReplaceWithObj); + res = 31 * res + hashCodeArraysAware(bInnerObj); + res = 31 * res + hashCodeArraysAware(cRefToOuterObjToReplaceWithInnerRef); + res = 31 * res + hashCodeArraysAware(dRefToInnerObjToRecalculate); + res = 31 * res + hashCodeArraysAware(eInnerBytePrimitive); + + return res; + } + } + + /** */ + private static class TestObject extends TestObjectAllTypes { + /** */ + public TestObject() { + setDefaultData(); + + uuid = UUID.randomUUID(); + } + } + + /** */ + public static class ComplexWrapper extends TestObject { + /** */ + private final Object data; + + /** */ + public ComplexWrapper(Object data) { + this.data = data; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (!(o instanceof ComplexWrapper)) + return false; + + ComplexWrapper that = (ComplexWrapper)o; + + return super.equals(o) && deepEquals(data, that.data); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return Objects.hash(super.hashCode(), hashCodeArraysAware(data)); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/RawBytesObjectReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/RawBytesObjectReaderTest.java new file mode 100644 index 0000000000000..5b4a0e6156696 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/RawBytesObjectReaderTest.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.lang.reflect.Field; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl; +import org.apache.ignite.internal.binary.mutabletest.GridBinaryTestClasses.TestObjectAllTypes; +import org.apache.ignite.internal.binary.streams.BinaryHeapInputStream; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.marshaller.MarshallerContext; +import org.apache.ignite.marshaller.jdk.JdkMarshaller; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +/** */ +public class RawBytesObjectReaderTest extends GridCommonAbstractTest { + /** */ + @Test + public void test() throws Exception { + BinaryContext ctx = createTestBinaryContext(); + + Collection testObjects = createObjectsOfAllTypes(ctx); + + byte[] serializedTestObjectsBytes; + + try (BinaryWriterExImpl writer = new BinaryWriterExImpl(ctx)) { + testObjects.forEach(writer::writeObject); + + serializedTestObjectsBytes = writer.array(); + } + + RawBytesObjectReader rawReader = new RawBytesObjectReader(BinaryHeapInputStream.create(serializedTestObjectsBytes, 0)); + + for (Object testObj : testObjects) { + byte[] objRawBytes = rawReader.readObject(); + + try (BinaryReaderExImpl binReader = new BinaryReaderExImpl(ctx, BinaryHeapInputStream.create(objRawBytes, 0), null, false)) { + Object deserializedObj = binReader.readObject(); + + if (testObj instanceof Proxy) + assertEquals(String.valueOf(testObj), String.valueOf(deserializedObj)); + else + assertEqualsArraysAware(testObj, deserializedObj); + + assertEquals(objRawBytes.length, binReader.in().position()); + } + } + + assertEquals(serializedTestObjectsBytes.length, rawReader.position()); + } + + /** */ + public static BinaryContext createTestBinaryContext() { + BinaryContext ctx = new BinaryContext(BinaryCachingMetadataHandler.create(), new IgniteConfiguration(), null); + + BinaryMarshaller marsh = new BinaryMarshaller(); + + marsh.setContext(new TestMarshallerContext()); + + ctx.configure(marsh); + + return ctx; + } + + /** */ + private static Collection createObjectsOfAllTypes(BinaryContext ctx) throws IllegalAccessException { + Collection res = new ArrayList<>(); + + TestObjectAllTypesEx allTypesObj = new TestObjectAllTypesEx(ctx); + + for (Field field : TestObjectAllTypesEx.class.getFields()) + res.add(field.get(allTypesObj)); + + return res; + } + + /** */ + private Object createTestObject() { + TestObjectAllTypes res = new TestObjectAllTypes(); + + res.setDefaultData(); + + return res; + } + + /** */ + private interface RegisteredClass { } + + /** */ + private interface UnregisteredClass { } + + /** */ + private static class TestMarshallerContext implements MarshallerContext { + /** */ + Map clsNamesByTypeId = new HashMap<>(); + + /** {@inheritDoc} */ + @Override public boolean registerClassName( + byte platformId, + int typeId, + String clsName + ) { + if (Objects.equals(clsName, UnregisteredClass.class.getName())) + return false; + + clsNamesByTypeId.put(typeId, clsName); + + return true; + } + + /** {@inheritDoc} */ + @Override public boolean registerClassNameLocally(byte platformId, int typeId, String clsName) { + return registerClassName(platformId, typeId, clsName); + } + + /** {@inheritDoc} */ + @Override public Class getClass(int typeId, ClassLoader ldr) throws ClassNotFoundException { + return U.forName(clsNamesByTypeId.get(typeId), ldr); + } + + /** {@inheritDoc} */ + @Override public String getClassName(byte platformId, int typeId) { + return clsNamesByTypeId.get(typeId); + } + + /** {@inheritDoc} */ + @Override public boolean isSystemType(String typeName) { + return false; + } + + /** {@inheritDoc} */ + @Override public IgnitePredicate classNameFilter() { + return null; + } + + /** {@inheritDoc} */ + @Override public JdkMarshaller jdkMarshaller() { + return new JdkMarshaller(); + } + } + + /** */ + public static class TestInvocationHandler implements InvocationHandler { + /** */ + private final String proxyClsName; + + /** */ + public TestInvocationHandler(Class proxyCls) { + proxyClsName = proxyCls.getName(); + } + + /** {@inheritDoc} */ + @Override public Object invoke(Object p, Method method, Object[] methodArgs) throws Throwable { + if ("toString".equals(method.getName())) + return proxyClsName; + + throw new IllegalStateException(); + } + } + + /** */ + public static class TestObjectAllTypesEx extends TestObjectAllTypes { + /** */ + public Object nullObj; + + /** */ + public Object obj; + + /** */ + public Object[] emptyArr; + + /** */ + public Object[] objArr; + + /** */ + public Collection col; + + /** */ + public Map map; + + /** */ + public Class registeredClass; + + /** */ + public Class unregisteredClass; + + /** */ + public Object registeredClsProxy; + + /** */ + public Object unregisteredClsProxy; + + /** */ + public BinaryObject binObj; + + /** */ + public TestObjectAllTypesEx(BinaryContext ctx) { + setDefaultData(); + + nullObj = null; + obj = new TestObjectAllTypes(); + + emptyArr = new Object[] {}; + objArr = new Object[] {"test", new TestObjectAllTypes(), new Object[]{new TestObjectAllTypes()}}; + + col = new ArrayList<>(); + + col.add(0); + col.add(new TestObjectAllTypes()); + col.add(new ArrayList<>()); + col.add(new HashMap<>()); + + map = new HashMap<>(); + + map.put(0, 0); + map.put(1, new TestObjectAllTypes()); + map.put(2, new ArrayList<>()); + map.put(3, new HashMap<>()); + + registeredClass = RegisteredClass.class; + unregisteredClass = UnregisteredClass.class; + + registeredClsProxy = Proxy.newProxyInstance( + RawBytesObjectReaderTest.class.getClassLoader(), + new Class[] { RegisteredClass.class }, + new TestInvocationHandler(RegisteredClass.class)); + + unregisteredClsProxy = Proxy.newProxyInstance( + RawBytesObjectReaderTest.class.getClassLoader(), + new Class[] { UnregisteredClass.class }, + new TestInvocationHandler(UnregisteredClass.class)); + + binObj = new BinaryObjectBuilderImpl(ctx, "TestBinaryType").setField("test-field", "test-value").build(); + } + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java index 067866c094adf..8d1ff662782f2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/mutabletest/GridBinaryTestClasses.java @@ -26,9 +26,11 @@ import java.math.BigInteger; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.TreeMap; import java.util.UUID; import com.google.common.base.Throwables; @@ -317,6 +319,79 @@ public void setDefaultData() { entry = new GridMapEntry<>(1, "a"); } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (!(o instanceof TestObjectAllTypes)) + return false; + + TestObjectAllTypes allTypesObj = (TestObjectAllTypes)o; + + return b == allTypesObj.b + && s == allTypesObj.s + && i == allTypesObj.i + && l == allTypesObj.l + && Float.compare(f, allTypesObj.f) == 0 + && Double.compare(d, allTypesObj.d) == 0 + && c == allTypesObj.c + && z == allTypesObj.z + && Objects.equals(b_, allTypesObj.b_) + && Objects.equals(s_, allTypesObj.s_) + && Objects.equals(i_, allTypesObj.i_) + && Objects.equals(bi_, allTypesObj.bi_) + && Objects.equals(l_, allTypesObj.l_) + && Objects.equals(f_, allTypesObj.f_) + && Objects.equals(d_, allTypesObj.d_) + && (bd_ == null ? allTypesObj.bd_ == null : bd_.compareTo(allTypesObj.bd_) == 0) + && Objects.equals(c_, allTypesObj.c_) + && Objects.equals(z_, allTypesObj.z_) + && Objects.equals(str, allTypesObj.str) + && Objects.equals(uuid, allTypesObj.uuid) + && Objects.equals(date, allTypesObj.date) + && Objects.equals(ts, allTypesObj.ts) + && Arrays.equals(bArr, allTypesObj.bArr) + && Arrays.equals(sArr, allTypesObj.sArr) + && Arrays.equals(iArr, allTypesObj.iArr) + && Arrays.equals(lArr, allTypesObj.lArr) + && Arrays.equals(fArr, allTypesObj.fArr) + && Arrays.equals(dArr, allTypesObj.dArr) + && Arrays.equals(cArr, allTypesObj.cArr) + && Arrays.equals(zArr, allTypesObj.zArr) + && Arrays.equals(bdArr, allTypesObj.bdArr) + && Arrays.equals(strArr, allTypesObj.strArr) + && Arrays.equals(uuidArr, allTypesObj.uuidArr) + && Arrays.equals(dateArr, allTypesObj.dateArr) + && Arrays.equals(tsArr, allTypesObj.tsArr) + && anEnum == allTypesObj.anEnum + && Arrays.equals(enumArr, allTypesObj.enumArr) + && Objects.equals(entry, allTypesObj.entry); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = + Objects.hash(b_, s_, i_, bi_, l_, f_, d_, bd_, c_, z_, b, s, i, l, f, d, c, z, str, uuid, date, ts, anEnum, entry); + + res = 31 * res + Arrays.hashCode(bArr); + res = 31 * res + Arrays.hashCode(sArr); + res = 31 * res + Arrays.hashCode(iArr); + res = 31 * res + Arrays.hashCode(lArr); + res = 31 * res + Arrays.hashCode(fArr); + res = 31 * res + Arrays.hashCode(dArr); + res = 31 * res + Arrays.hashCode(cArr); + res = 31 * res + Arrays.hashCode(zArr); + res = 31 * res + Arrays.hashCode(bdArr); + res = 31 * res + Arrays.hashCode(strArr); + res = 31 * res + Arrays.hashCode(uuidArr); + res = 31 * res + Arrays.hashCode(dateArr); + res = 31 * res + Arrays.hashCode(tsArr); + res = 31 * res + Arrays.hashCode(enumArr); + + return res; + } } /** diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java index b52df8fe8752c..b7dc67e9ab3b6 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java @@ -41,11 +41,13 @@ import org.apache.ignite.internal.binary.BinarySerialiedFieldComparatorSelfTest; import org.apache.ignite.internal.binary.BinarySimpleNameTestPropertySelfTest; import org.apache.ignite.internal.binary.BinaryTreeSelfTest; +import org.apache.ignite.internal.binary.CrossObjetReferenceSerializationTest; import org.apache.ignite.internal.binary.GridBinaryAffinityKeySelfTest; import org.apache.ignite.internal.binary.GridBinaryMarshallerCtxDisabledSelfTest; import org.apache.ignite.internal.binary.GridBinaryWildcardsSelfTest; import org.apache.ignite.internal.binary.GridDefaultBinaryMappersBinaryMetaDataSelfTest; import org.apache.ignite.internal.binary.GridSimpleLowerCaseBinaryMappersBinaryMetaDataSelfTest; +import org.apache.ignite.internal.binary.RawBytesObjectReaderTest; import org.apache.ignite.internal.binary.noncompact.BinaryFieldsHeapNonCompactSelfTest; import org.apache.ignite.internal.binary.noncompact.BinaryFieldsOffheapNonCompactSelfTest; import org.apache.ignite.internal.binary.noncompact.BinaryFooterOffsetsHeapNonCompactSelfTest; @@ -179,6 +181,9 @@ BinaryMetadataMoveLegacyFolderTest.class, BinaryContextPredefinedTypesTest.class, + + RawBytesObjectReaderTest.class, + CrossObjetReferenceSerializationTest.class, }) public class IgniteBinaryObjectsTestSuite { } From 0a38c74691cf9232086768fcfacb2b13032961e4 Mon Sep 17 00:00:00 2001 From: Mikhail Petrov Date: Sun, 5 May 2024 21:01:55 +0300 Subject: [PATCH 2/3] wip --- .../org/apache/ignite/internal/binary/ObjectDetachHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java index 6d19f020bcea6..7f6ed1f48e341 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java @@ -137,7 +137,7 @@ private void readNextObject() { int handleObjPos = objStartPos - offset; if (handleObjPos < rootObjStartPos) - saveCrossObjectReferenceData(handleObjPos, objStartPos); + saveCrossObjectReferenceData(handleObjPos, objStartPos); break; } From d3a4f1f4337e924c4f914602bfbc070adbb95541 Mon Sep 17 00:00:00 2001 From: Mikhail Petrov Date: Wed, 15 May 2024 23:46:11 +0300 Subject: [PATCH 3/3] wip --- .../org/apache/ignite/internal/binary/ObjectDetachHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java index 7f6ed1f48e341..bcff3016ffcd0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java @@ -269,7 +269,7 @@ public int objectStartOffset() { /** {@inheritDoc} */ @Override public int compareTo(@NotNull ObjectDetachHelper.ObjectDescriptor other) { - return Integer.compare(objStartOffset, other.objDataStartOffset); + return Integer.compare(objStartOffset, other.objStartOffset); } /** */