Skip to content

Commit

Permalink
INTERNAL: Refactor transcoder compress logic.
Browse files Browse the repository at this point in the history
  • Loading branch information
brido4125 committed Nov 10, 2023
1 parent b21b786 commit c5d5e94
Show file tree
Hide file tree
Showing 7 changed files with 156 additions and 100 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,36 +22,35 @@
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.UnsupportedEncodingException;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import net.spy.memcached.CachedData;
import net.spy.memcached.compat.CloseUtil;
import net.spy.memcached.compat.SpyObject;

/**
* Base class for any transcoders that may want to work with serialized or
* compressed data.
*/
public abstract class BaseSerializingTranscoder extends SpyObject {

/**
* Default compression threshold value.
*/
public static final int DEFAULT_COMPRESSION_THRESHOLD = 16384;

private static final String DEFAULT_CHARSET = "UTF-8";

protected int compressionThreshold = DEFAULT_COMPRESSION_THRESHOLD;
protected String charset = DEFAULT_CHARSET;

protected final Compressor compressor;
private final int maxSize;

/**
* Initialize a serializing transcoder with the given maximum data size.
*/
public BaseSerializingTranscoder(int max) {
super();
this.compressor = new GzipCompressor();
maxSize = max;
}

/**
* Initialize a serializing transcoder with the given maximum data size and compressor.
*/
public BaseSerializingTranscoder(int max, Compressor compressor) {
super();
this.compressor = compressor;
maxSize = max;
}

Expand All @@ -67,7 +66,7 @@ public boolean asyncDecode(CachedData d) {
* @param to the number of bytes
*/
public void setCompressionThreshold(int to) {
compressionThreshold = to;
compressor.setCompressionThreshold(to);
}

/**
Expand Down Expand Up @@ -127,56 +126,6 @@ protected Object deserialize(byte[] in) {
return rv;
}

/**
* Compress the given array of bytes.
*/
protected byte[] compress(byte[] in) {
if (in == null) {
throw new NullPointerException("Can't compress null");
}
ByteArrayOutputStream bos = new ByteArrayOutputStream();
GZIPOutputStream gz = null;
try {
gz = new GZIPOutputStream(bos);
gz.write(in);
} catch (IOException e) {
throw new RuntimeException("IO exception compressing data", e);
} finally {
CloseUtil.close(gz);
CloseUtil.close(bos);
}
byte[] rv = bos.toByteArray();
getLogger().debug("Compressed %d bytes to %d", in.length, rv.length);
return rv;
}

/**
* Decompress the given array of bytes.
*
* @return null if the bytes cannot be decompressed
*/
protected byte[] decompress(byte[] in) {
ByteArrayOutputStream bos = null;
if (in != null) {
ByteArrayInputStream bis = new ByteArrayInputStream(in);
bos = new ByteArrayOutputStream();
GZIPInputStream gis;
try {
gis = new GZIPInputStream(bis);

byte[] buf = new byte[8192];
int r = -1;
while ((r = gis.read(buf)) > 0) {
bos.write(buf, 0, r);
}
} catch (IOException e) {
getLogger().warn("Failed to decompress data", e);
bos = null;
}
}
return bos == null ? null : bos.toByteArray();
}

/**
* Decode the string with the current character set.
*/
Expand Down
24 changes: 24 additions & 0 deletions src/main/java/net/spy/memcached/transcoders/Compressor.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package net.spy.memcached.transcoders;

public interface Compressor {
/**
* Default compression threshold value.
*/
int DEFAULT_COMPRESSION_THRESHOLD = 16384;

/**
* Compress the given array of bytes.
*/
public byte[] compress(byte[] in);

/**
* Decompress the given array of bytes.
*
* @return null if the bytes cannot be decompressed
*/
public byte[] decompress(byte[] in);

public int getCompressionThreshold();

public void setCompressionThreshold(int to);
}
66 changes: 66 additions & 0 deletions src/main/java/net/spy/memcached/transcoders/GzipCompressor.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package net.spy.memcached.transcoders;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import net.spy.memcached.compat.CloseUtil;
import net.spy.memcached.compat.SpyObject;

public class GzipCompressor extends SpyObject implements Compressor {
private int compressionThreshold = DEFAULT_COMPRESSION_THRESHOLD;

@Override
public byte[] compress(byte[] in) {
if (in == null) {
throw new NullPointerException("Can't compress null");
}
ByteArrayOutputStream bos = new ByteArrayOutputStream();
GZIPOutputStream gz = null;
try {
gz = new GZIPOutputStream(bos);
gz.write(in);
} catch (IOException e) {
throw new RuntimeException("IO exception compressing data", e);
} finally {
CloseUtil.close(gz);
CloseUtil.close(bos);
}
return bos.toByteArray();
}

@Override
public byte[] decompress(byte[] in) {
ByteArrayOutputStream bos = null;
if (in != null) {
ByteArrayInputStream bis = new ByteArrayInputStream(in);
bos = new ByteArrayOutputStream();
GZIPInputStream gis;
try {
gis = new GZIPInputStream(bis);
byte[] buf = new byte[8192];
int r = -1;
while ((r = gis.read(buf)) > 0) {
bos.write(buf, 0, r);
}
} catch (IOException e) {
getLogger().warn("Failed to decompress data", e);
bos = null;
}
}
return bos == null ? null : bos.toByteArray();
}

@Override
public int getCompressionThreshold() {
return compressionThreshold;
}

@Override
public void setCompressionThreshold(int compressionThreshold) {
this.compressionThreshold = compressionThreshold;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,20 @@ public SerializingTranscoder(int max) {
super(max);
}

/**
* Get a serializing transcoder customized compressor.
*/
public SerializingTranscoder(Compressor compressor) {
super(CachedData.MAX_SIZE, compressor);
}

/**
* Get a serializing transcoder that specifies the max data size and customized compressor.
*/
public SerializingTranscoder(int max, Compressor compressor) {
super(max, compressor);
}

@Override
public boolean asyncDecode(CachedData d) {
if ((d.getFlags() & COMPRESSED) != 0
Expand All @@ -72,7 +86,7 @@ public Object decode(CachedData d) {
byte[] data = d.getData();
Object rv = null;
if ((d.getFlags() & COMPRESSED) != 0) {
data = decompress(d.getData());
data = compressor.decompress(d.getData());
}
int flags = d.getFlags() & SPECIAL_MASK;
if ((d.getFlags() & SERIALIZED) != 0 && data != null) {
Expand Down Expand Up @@ -146,20 +160,21 @@ public CachedData encode(Object o) {
flags |= SERIALIZED;
}
assert b != null;
if (b.length > compressionThreshold) {
byte[] compressed = compress(b);
if (b.length > compressor.getCompressionThreshold()) {
byte[] compressed = compressor.compress(b);
if (compressed.length < b.length) {
getLogger().debug("Compressed %s from %d to %d",
o.getClass().getName(), b.length, compressed.length);
getLogger().debug("Compressed %s from %d to %d", o.getClass().getName(),
b.length, compressed.length);
b = compressed;
flags |= COMPRESSED;
} else {
getLogger().info(
"Compression increased the size of %s from %d to %d",
} else if (compressed.length > b.length) {
getLogger().info("Compression increased the size of %s from %d to %d",
o.getClass().getName(), b.length, compressed.length);
} else {
getLogger().info("Compression makes same length of %s : %d",
o.getClass().getName(), b.length);
}
}
return new CachedData(flags, b, getMaxSize());
}

}
22 changes: 14 additions & 8 deletions src/main/java/net/spy/memcached/transcoders/WhalinTranscoder.java
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,15 @@ public WhalinTranscoder() {
super(CachedData.MAX_SIZE);
}

public WhalinTranscoder(Compressor compressor) {
super(CachedData.MAX_SIZE, compressor);
}

public Object decode(CachedData d) {
byte[] data = d.getData();
Object rv = null;
if ((d.getFlags() & COMPRESSED) != 0) {
data = decompress(d.getData());
data = compressor.decompress(d.getData());
}
if ((d.getFlags() & SERIALIZED) != 0) {
rv = deserialize(data);
Expand Down Expand Up @@ -152,17 +156,19 @@ public CachedData encode(Object o) {
flags |= SERIALIZED;
}
assert b != null;
if (b.length > compressionThreshold) {
byte[] compressed = compress(b);
if (b.length > compressor.getCompressionThreshold()) {
byte[] compressed = compressor.compress(b);
if (compressed.length < b.length) {
getLogger().debug("Compressed %s from %d to %d",
o.getClass().getName(), b.length, compressed.length);
getLogger().debug("Compressed %s from %d to %d", o.getClass().getName(),
b.length, compressed.length);
b = compressed;
flags |= COMPRESSED;
} else {
getLogger().info(
"Compression increased the size of %s from %d to %d",
} else if (compressed.length > b.length) {
getLogger().info("Compression increased the size of %s from %d to %d",
o.getClass().getName(), b.length, compressed.length);
} else {
getLogger().info("Compression makes same length of %s : %d",
o.getClass().getName(), b.length);
}
}
return new CachedData(flags, b, getMaxSize());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ public WhalinV1Transcoder() {
super(CachedData.MAX_SIZE);
}

public WhalinV1Transcoder(Compressor compressor) {
super(CachedData.MAX_SIZE, compressor);
}

public CachedData encode(Object o) {
byte[] b = null;
int flags = 0;
Expand Down Expand Up @@ -66,17 +70,19 @@ public CachedData encode(Object o) {
flags |= SERIALIZED;
}
assert b != null;
if (b.length > compressionThreshold) {
byte[] compressed = compress(b);
if (b.length > compressor.getCompressionThreshold()) {
byte[] compressed = compressor.compress(b);
if (compressed.length < b.length) {
getLogger().info("Compressed %s from %d to %d",
o.getClass().getName(), b.length, compressed.length);
getLogger().debug("Compressed %s from %d to %d", o.getClass().getName(),
b.length, compressed.length);
b = compressed;
flags |= COMPRESSED;
} else {
getLogger().info(
"Compression increased the size of %s from %d to %d",
} else if (compressed.length > b.length) {
getLogger().info("Compression increased the size of %s from %d to %d",
o.getClass().getName(), b.length, compressed.length);
} else {
getLogger().info("Compression makes same length of %s : %d",
o.getClass().getName(), b.length);
}
}
return new CachedData(flags, b, getMaxSize());
Expand All @@ -86,7 +92,7 @@ public Object decode(CachedData d) {
byte[] data = d.getData();
Object rv = null;
if ((d.getFlags() & COMPRESSED) != 0) {
data = decompress(d.getData());
data = compressor.decompress(d.getData());
}
if ((d.getFlags() & SERIALIZED) != 0) {
rv = deserialize(data);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public void testInvalidCharacterSet() {

public void testCompressNull() {
try {
ex.compress(null);
ex.compressor.compress(null);
fail("Expected an assertion error");
} catch (NullPointerException e) {
// pass
Expand Down Expand Up @@ -67,7 +67,7 @@ public void testSerializeNull() {
}

public void testDecompressNull() {
assertNull(ex.decompress(null));
assertNull(ex.compressor.decompress(null));
}

public void testUndeserializable() throws Exception {
Expand Down Expand Up @@ -117,21 +117,11 @@ public void overrideCharsetSet(String to) {
charset = to;
}

@Override
public byte[] compress(byte[] in) {
return super.compress(in);
}

@Override
public String decodeString(byte[] data) {
return super.decodeString(data);
}

@Override
public byte[] decompress(byte[] in) {
return super.decompress(in);
}

@Override
public Object deserialize(byte[] in) {
return super.deserialize(in);
Expand Down

0 comments on commit c5d5e94

Please sign in to comment.