From e1480aadb223295545e41da804d891fee011b9d2 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Wed, 5 Oct 2016 10:57:26 -0400 Subject: [PATCH 01/55] Add streaming api to operations/futures --- .../riak/client/core/FutureOperation.java | 20 ++++++++-- .../basho/riak/client/core/RiakCluster.java | 11 ++++++ .../client/core/StreamingFutureOperation.java | 34 +++++++++++++++++ .../riak/client/core/StreamingRiakFuture.java | 37 +++++++++++++++++++ 4 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java create mode 100644 src/main/java/com/basho/riak/client/core/StreamingRiakFuture.java diff --git a/src/main/java/com/basho/riak/client/core/FutureOperation.java b/src/main/java/com/basho/riak/client/core/FutureOperation.java index 3e0195aea..1747f837b 100644 --- a/src/main/java/com/basho/riak/client/core/FutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/FutureOperation.java @@ -91,7 +91,7 @@ private enum State private final CountDownLatch latch = new CountDownLatch(1); private volatile OperationRetrier retrier; private volatile int remainingTries = 1; - private volatile List rawResponse = new LinkedList<>(); + private volatile List rawResponses = new LinkedList<>(); private volatile Throwable exception; private volatile T converted; private volatile State state = State.CREATED; @@ -194,7 +194,9 @@ public synchronized final void setResponse(RiakMessage rawResponse) { stateCheck(State.CREATED, State.WRITTEN, State.RETRY); U decodedMessage = decode(rawResponse); - this.rawResponse.add(decodedMessage); + + processMessage(decodedMessage); + exception = null; if (done(decodedMessage)) { @@ -208,6 +210,16 @@ public synchronized final void setResponse(RiakMessage rawResponse) } } + protected void processMessage(U decodedMessage) + { + processBatchMessage(decodedMessage); + } + + private void processBatchMessage(U decodedMessage) + { + this.rawResponses.add(decodedMessage); + } + public synchronized final void setComplete() { logger.debug("Setting Complete on future"); @@ -322,7 +334,7 @@ private void tryConvertResponse() throws ExecutionException { try { - converted = convert(rawResponse); + converted = convert(rawResponses); } catch (IllegalArgumentException ex) { @@ -358,7 +370,7 @@ public final T getNow() { if (null == converted) { - converted = convert(rawResponse); + converted = convert(rawResponses); } return converted; diff --git a/src/main/java/com/basho/riak/client/core/RiakCluster.java b/src/main/java/com/basho/riak/client/core/RiakCluster.java index e49a84e83..93e1640a0 100644 --- a/src/main/java/com/basho/riak/client/core/RiakCluster.java +++ b/src/main/java/com/basho/riak/client/core/RiakCluster.java @@ -220,6 +220,17 @@ public boolean isDone() } public RiakFuture execute(FutureOperation operation) + { + return executeFutureOperation(operation); + } + + public StreamingRiakFuture execute(StreamingFutureOperation operation) + { + executeFutureOperation(operation); + return operation; + } + + private RiakFuture executeFutureOperation(FutureOperation operation) { stateCheck(State.RUNNING, State.QUEUING); operation.setRetrier(this, executionAttempts); diff --git a/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java b/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java new file mode 100644 index 000000000..bf2429738 --- /dev/null +++ b/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java @@ -0,0 +1,34 @@ +package com.basho.riak.client.core; + +/** + * @author Alex Moore + * @param The type returned by the streaming and non-streaming operation versions + * @param The protocol type returned + * @param Query info type + * @since 2.1.0 + */ +public abstract class StreamingFutureOperation + extends FutureOperation + implements StreamingRiakFuture +{ + private boolean streamResults; + + protected StreamingFutureOperation(boolean streamResults) + { + this.streamResults = streamResults; + } + + @Override + protected void processMessage(ResponseType decodedMessage) + { + if(!streamResults) + { + super.processMessage(decodedMessage); + return; + } + + processStreamingChunk(decodedMessage); + } + + abstract protected void processStreamingChunk(ResponseType rawResponseChunk); +} diff --git a/src/main/java/com/basho/riak/client/core/StreamingRiakFuture.java b/src/main/java/com/basho/riak/client/core/StreamingRiakFuture.java new file mode 100644 index 000000000..895ba1d6d --- /dev/null +++ b/src/main/java/com/basho/riak/client/core/StreamingRiakFuture.java @@ -0,0 +1,37 @@ +/* + * Copyright 2013 Basho Technologies Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.basho.riak.client.core; + +import java.util.concurrent.BlockingQueue; + +/** + * The result of an asynchronous streaming (chunked) Riak operation. + * + * @author Alex Moore + * @param the streaming and non-streaming (response) return type + * @param The query info type + * + * @since 2.1.0 + */ +public interface StreamingRiakFuture + extends RiakFuture +{ + /** + * An Queue that provides the stream of results as they return from Riak. + * @return An Queue. + */ + BlockingQueue getResultsQueue(); +} From af3d5ba3a9f9419a7a14eb0c011bcc48d6ffbbe8 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Wed, 5 Oct 2016 14:00:37 -0400 Subject: [PATCH 02/55] Add Streaming List Buckets --- .../core/operations/ListBucketsOperation.java | 74 +++++- .../api/commands/buckets/ListBucketsTest.java | 8 +- .../itest/ITestListBucketsOperation.java | 224 ++++++++++-------- 3 files changed, 195 insertions(+), 111 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java index ef31851f3..97d3f911c 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java @@ -15,27 +15,34 @@ */ package com.basho.riak.client.core.operations; -import com.basho.riak.client.core.FutureOperation; import com.basho.riak.client.core.RiakMessage; +import com.basho.riak.client.core.StreamingFutureOperation; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.util.BinaryValue; -import com.basho.riak.protobuf.RiakMessageCodes; import com.basho.riak.protobuf.RiakKvPB; +import com.basho.riak.protobuf.RiakMessageCodes; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; -public class ListBucketsOperation extends FutureOperation +public class ListBucketsOperation extends StreamingFutureOperation { private final RiakKvPB.RpbListBucketsReq.Builder reqBuilder; private final BinaryValue bucketType; + private final BlockingQueue responseQueue; private ListBucketsOperation(Builder builder) { + super(builder.streamResults); this.reqBuilder = builder.reqBuilder; this.bucketType = builder.bucketType; + this.responseQueue = new LinkedBlockingQueue<>(); } @Override @@ -50,14 +57,30 @@ protected ListBucketsOperation.Response convert(List buckets = new ArrayList<>(rawResponse.size()); for (RiakKvPB.RpbListBucketsResp resp : rawResponse) { - for (ByteString bucket : resp.getBucketsList()) - { - buckets.add(BinaryValue.unsafeCreate(bucket.toByteArray())); - } + buckets.addAll(convertSingleResponse(resp)); } + return new Response(bucketType, buckets); } + private List convertSingleResponse(RiakKvPB.RpbListBucketsResp resp) + { + List buckets = new ArrayList<>(resp.getBucketsCount()); + + for (ByteString bucket : resp.getBucketsList()) + { + buckets.add(BinaryValue.unsafeCreate(bucket.toByteArray())); + } + return buckets; + } + + @Override + protected void processStreamingChunk(RiakKvPB.RpbListBucketsResp rawResponseChunk) + { + final List buckets = convertSingleResponse(rawResponseChunk); + this.responseQueue.add(new Response(bucketType, buckets)); + } + @Override protected RiakMessage createChannelMessage() { @@ -84,10 +107,17 @@ public BinaryValue getQueryInfo() return bucketType; } + @Override + public BlockingQueue getResultsQueue() + { + return this.responseQueue; + } + public static class Builder { private final RiakKvPB.RpbListBucketsReq.Builder reqBuilder = RiakKvPB.RpbListBucketsReq.newBuilder().setStream(true); + private boolean streamResults = false; private BinaryValue bucketType = BinaryValue.create(Namespace.DEFAULT_BUCKET_TYPE); /** @@ -98,6 +128,7 @@ public Builder() /** * Provide a timeout for this operation. + * * @param timeout value in milliseconds * @return a reference to this object */ @@ -112,11 +143,12 @@ public Builder withTimeout(int timeout) } /** - * Set the bucket type. - * If unset {@link Namespace#DEFAULT_BUCKET_TYPE} is used. - * @param bucketType the bucket type to use - * @return A reference to this object. - */ + * Set the bucket type. + * If unset {@link Namespace#DEFAULT_BUCKET_TYPE} is used. + * + * @param bucketType the bucket type to use + * @return A reference to this object. + */ public Builder withBucketType(BinaryValue bucketType) { if (null == bucketType || bucketType.length() == 0) @@ -128,6 +160,24 @@ public Builder withBucketType(BinaryValue bucketType) return this; } + /** + * Set the streamResults flag. + * + * If unset or false, the entire result set will be available through the {@link ListBucketsOperation#get()} + * method once the operation is complete. + * + * If set to true, results will be pushed to the queue available through the {@link ListBucketsOperation#getResultsQueue()} + * method as soon as they are available. + * + * @param streamResults whether to stream results to {@link ListBucketsOperation#get()}(false), or {@link ListBucketsOperation#getResultsQueue()}(true) + * @return A reference to this object. + */ + public Builder streamResults(boolean streamResults) + { + this.streamResults = streamResults; + return this; + } + public ListBucketsOperation build() { return new ListBucketsOperation(this); diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/ListBucketsTest.java b/src/test/java/com/basho/riak/client/api/commands/buckets/ListBucketsTest.java index 51d959b29..532ed7e30 100644 --- a/src/test/java/com/basho/riak/client/api/commands/buckets/ListBucketsTest.java +++ b/src/test/java/com/basho/riak/client/api/commands/buckets/ListBucketsTest.java @@ -1,9 +1,7 @@ package com.basho.riak.client.api.commands.buckets; import com.basho.riak.client.api.RiakClient; -import com.basho.riak.client.core.FutureOperation; -import com.basho.riak.client.core.RiakCluster; -import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.*; import com.basho.riak.client.core.operations.ListBucketsOperation; import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; @@ -33,7 +31,7 @@ public class ListBucketsTest { @Mock RiakCluster mockCluster; - @Mock RiakFuture mockFuture; + @Mock StreamingRiakFuture mockFuture; @Mock ListBucketsOperation.Response mockResponse; RiakClient client; @@ -49,7 +47,7 @@ public void init() throws Exception when(mockFuture.isCancelled()).thenReturn(false); when(mockFuture.isDone()).thenReturn(true); when(mockFuture.isSuccess()).thenReturn(true); - when(mockCluster.execute(any(FutureOperation.class))).thenReturn(mockFuture); + when(mockCluster.execute(any(StreamingFutureOperation.class))).thenReturn(mockFuture); client = new RiakClient(mockCluster); } diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListBucketsOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListBucketsOperation.java index 36f8993ea..4e646c318 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListBucketsOperation.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListBucketsOperation.java @@ -16,8 +16,9 @@ package com.basho.riak.client.core.operations.itest; import com.basho.riak.client.core.RiakFuture; -import com.basho.riak.client.core.RiakFutureListener; +import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.ListBucketsOperation; +import com.basho.riak.client.core.operations.ListKeysOperation; import com.basho.riak.client.core.operations.StoreOperation; import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; @@ -26,12 +27,12 @@ import org.junit.Test; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; -import java.util.concurrent.CountDownLatch; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; @@ -42,136 +43,171 @@ */ public class ITestListBucketsOperation extends ITestAutoCleanupBase { + private final String defaultBucketType = Namespace.DEFAULT_BUCKET_TYPE; + private final String namedBucketType = ITestBase.bucketType.toStringUtf8(); + @Test public void testListBucketsDefaultType() throws InterruptedException, ExecutionException { - testListBuckets(Namespace.DEFAULT_BUCKET_TYPE); + testBucketList(defaultBucketType, 1); + } + + @Test + public void testListBucketsDefaultTypeStreaming() throws ExecutionException, InterruptedException + { + testBucketListStreaming(namedBucketType, 1); } @Test public void testListBucketsTestType() throws InterruptedException, ExecutionException { assumeTrue(testBucketType); - testListBuckets(bucketType.toString()); + testBucketList(namedBucketType, 1); } - private void testListBuckets(String bucketType) throws InterruptedException, ExecutionException + @Test + public void testListBucketsTestTypeStreaming() throws InterruptedException, ExecutionException { - // Empty buckets do not show up - final BinaryValue key = BinaryValue.unsafeCreate("my_key".getBytes()); - final String value = "{\"value\":\"value\"}"; - - RiakObject rObj = new RiakObject().setValue(BinaryValue.create(value)); - Location location = new Location(new Namespace(bucketType, bucketName.toString()), key); - StoreOperation storeOp = - new StoreOperation.Builder(location) - .withContent(rObj) - .build(); - - cluster.execute(storeOp); - storeOp.get(); - - ListBucketsOperation listOp = new ListBucketsOperation.Builder() - .withBucketType(BinaryValue.createFromUtf8(bucketType)) - .build(); - cluster.execute(listOp); - List bucketList = listOp.get().getBuckets(); - assertTrue(bucketList.size() > 0); - - boolean found = false; - for (BinaryValue baw : bucketList) - { - if (baw.toString().equals(bucketName.toString())) - { - found = true; - } - } - - assertTrue(found); + assumeTrue(testBucketType); + testBucketListStreaming(namedBucketType, 1); } @Test public void testLargeBucketListDefaultType() throws InterruptedException, ExecutionException { - testLargeBucketList(Namespace.DEFAULT_BUCKET_TYPE); + testBucketList(defaultBucketType, 10); + } + + @Test + public void testLargeBucketListDefaultTypeStreaming() throws InterruptedException, ExecutionException + { + assumeTrue(testBucketType); + testBucketListStreaming(defaultBucketType, 10); } @Test public void testLargeBucketListTestType() throws InterruptedException, ExecutionException { assumeTrue(testBucketType); - testLargeBucketList(bucketType.toString()); + testBucketList(namedBucketType, 10); } - private void testLargeBucketList(String bucketType) throws InterruptedException, ExecutionException + @Test + public void testLargeBucketListTestTypeStreaming() throws InterruptedException, ExecutionException { - final BinaryValue key = BinaryValue.unsafeCreate("my_key".getBytes()); - final String value = "{\"value\":\"value\"}"; - final Semaphore semaphore = new Semaphore(10); - final CountDownLatch latch = new CountDownLatch(1); - final int bucketCount = 10; - final List bucketNames = new ArrayList<>(); + assumeTrue(testBucketType); + testBucketListStreaming(namedBucketType, 10); + } - RiakFutureListener listener = - new RiakFutureListener() - { - private final AtomicInteger received = new AtomicInteger(); - - @Override - public void handle(RiakFuture f) - { - try - { - f.get(); - } - catch (InterruptedException | ExecutionException ex) - { - throw new RuntimeException(ex); - } - - semaphore.release(); - received.incrementAndGet(); - if (received.intValue() == bucketCount) - { - latch.countDown(); - } - } - }; - for (int i = 0; i < bucketCount; i++) + private void testBucketList(String bucketType, int bucketCount) throws InterruptedException, ExecutionException + { + final List expectedBuckets = storeObjects(bucketType, bucketCount); + + List actualBucketNames = getAllBucketListResults(bucketType); + + assertContainsAll(expectedBuckets, actualBucketNames); + + resetAndEmptyBuckets(bucketType, expectedBuckets); + } + + private void testBucketListStreaming(String bucketType, int bucketCount) throws InterruptedException, ExecutionException + { + final List expectedBuckets = storeObjects(bucketType, bucketCount); + final ListBucketsOperation listOp = new ListBucketsOperation.Builder() + .withBucketType(BinaryValue.createFromUtf8(bucketType)) + .streamResults(true) + .build(); + + final StreamingRiakFuture execute = cluster.execute(listOp); + final BlockingQueue resultsQueue = execute.getResultsQueue(); + + List actualBuckets = new LinkedList<>(); + int timeouts = 0; + + while(!execute.isDone()) { - String testBucketName = bucketName.toString() + i; - bucketNames.add(BinaryValue.create(testBucketName)); + final ListBucketsOperation.Response response = resultsQueue.poll(5, TimeUnit.MILLISECONDS); - Namespace ns = new Namespace(bucketType, testBucketName); - RiakObject rObj = new RiakObject().setValue(BinaryValue.create(value)); - Location location = new Location(ns, key); - StoreOperation storeOp = - new StoreOperation.Builder(location) - .withContent(rObj) - .build(); - storeOp.addListener(listener); - semaphore.acquire(); - cluster.execute(storeOp); + if(response != null) + { + actualBuckets.addAll(response.getBuckets()); + continue; + } + + timeouts++; + if(timeouts == 10) + { + break; + } } - latch.await(2, TimeUnit.MINUTES); + execute.await(); - ListBucketsOperation listOp = new ListBucketsOperation.Builder() - .withBucketType(BinaryValue.createFromUtf8(bucketType)) - .build(); - cluster.execute(listOp); - List bucketList = listOp.get().getBuckets(); + // Grab any last buckets that came in on the last message + actualBuckets.addAll(resultsQueue.stream() + .map(ListBucketsOperation.Response::getBuckets) + .flatMap(List::stream) + .collect(Collectors.toList())); + resultsQueue.clear(); + + assertTrue(resultsQueue.isEmpty()); + + assertContainsAll(expectedBuckets, actualBuckets); + + resetAndEmptyBuckets(bucketType, expectedBuckets); + } - for (BinaryValue name : bucketNames) + private void assertContainsAll(List expectedSet, List actualSet) + { + for (BinaryValue name : expectedSet) { - assertTrue(bucketList.contains(name)); + assertTrue(actualSet.contains(name)); } + } - for (BinaryValue name : bucketNames) + private void resetAndEmptyBuckets(String bucketType, List expectedBucketNames) + throws InterruptedException, ExecutionException + { + for (BinaryValue name : expectedBucketNames) { Namespace ns = new Namespace(bucketType, name.toString()); resetAndEmptyBucket(ns); } } + + private List getAllBucketListResults(String bucketType) throws InterruptedException, ExecutionException + { + final ListBucketsOperation listOp = new ListBucketsOperation.Builder() + .withBucketType(BinaryValue.createFromUtf8(bucketType)) + .build(); + cluster.execute(listOp); + return listOp.get().getBuckets(); + } + + private List storeObjects(String bucketType, int bucketCount) throws InterruptedException + { + final List bucketNames = new ArrayList<>(); + final BinaryValue key = BinaryValue.unsafeCreate("my_key".getBytes()); + final String value = "{\"value\":\"value\"}"; + final RiakObject rObj = new RiakObject().setValue(BinaryValue.create(value)); + + for (int i = 0; i < bucketCount; i++) + { + final String testBucketName = testName.getMethodName() + "_" + i; + bucketNames.add(BinaryValue.create(testBucketName)); + + final Location location = new Location(new Namespace(bucketType, testBucketName), key); + final StoreOperation storeOp = new StoreOperation.Builder(location) + .withContent(rObj) + .build(); + + final RiakFuture execute = cluster.execute(storeOp); + execute.await(); + assertTrue(execute.isSuccess()); + } + + return bucketNames; + } + } From 5c56d9f4406c5ce51f1cfb43d53a10fdc46f1446 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Wed, 5 Oct 2016 15:56:31 -0400 Subject: [PATCH 03/55] Add streaming list keys support --- .../core/operations/ListKeysOperation.java | 58 +++- .../itest/ITestListKeysOperation.java | 251 +++++++++++------- 2 files changed, 205 insertions(+), 104 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java index 923d258cc..1aad8b11d 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java @@ -15,8 +15,8 @@ */ package com.basho.riak.client.core.operations; -import com.basho.riak.client.core.FutureOperation; import com.basho.riak.client.core.RiakMessage; +import com.basho.riak.client.core.StreamingFutureOperation; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.util.BinaryValue; import com.basho.riak.protobuf.RiakMessageCodes; @@ -28,17 +28,23 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; -public class ListKeysOperation extends FutureOperation +public class ListKeysOperation extends StreamingFutureOperation { private final Logger logger = LoggerFactory.getLogger("ListKeysOperation"); private final Namespace namespace; private final RiakKvPB.RpbListKeysReq.Builder reqBuilder; + private final BlockingQueue responseQueue; + private ListKeysOperation(Builder builder) { + super(builder.streamResults); this.reqBuilder = builder.reqBuilder; this.namespace = builder.namespace; + this.responseQueue = new LinkedBlockingQueue<>(); } @Override @@ -47,14 +53,23 @@ protected Response convert(List rawResponse) Response.Builder builder = new Response.Builder(); for (RiakKvPB.RpbListKeysResp resp : rawResponse) { - for (ByteString bucket : resp.getKeysList()) - { - builder.addKey(BinaryValue.unsafeCreate(bucket.toByteArray())); - } + builder.addKeys(convertSingleResponse(resp)); } return builder.build(); } + private List convertSingleResponse(RiakKvPB.RpbListKeysResp resp) + { + List keys = new ArrayList<>(resp.getKeysCount()); + + for (ByteString key : resp.getKeysList()) + { + keys.add(BinaryValue.unsafeCreate(key.toByteArray())); + } + + return keys; + } + @Override protected RiakMessage createChannelMessage() { @@ -87,11 +102,24 @@ public Namespace getQueryInfo() return namespace; } + @Override + protected void processStreamingChunk(RiakKvPB.RpbListKeysResp rawResponseChunk) + { + this.responseQueue.add(new Response.Builder().addKeys(convertSingleResponse(rawResponseChunk)).build()); + } + + @Override + public BlockingQueue getResultsQueue() + { + return this.responseQueue; + } + public static class Builder { private final RiakKvPB.RpbListKeysReq.Builder reqBuilder = RiakKvPB.RpbListKeysReq.newBuilder(); private final Namespace namespace; + private boolean streamResults; /** * Construct a builder for a ListKeysOperaiton. @@ -118,6 +146,24 @@ public Builder withTimeout(int timeout) return this; } + /** + * Set the streamResults flag. + * + * If unset or false, the entire result set will be available through the {@link ListKeysOperation#get()} + * method once the operation is complete. + * + * If set to true, results will be pushed to the queue available through the {@link ListKeysOperation#getResultsQueue()} + * method as soon as they are available. + * + * @param streamResults whether to stream results to {@link ListKeysOperation#get()}(false), or {@link ListKeysOperation#getResultsQueue()}(true) + * @return A reference to this object. + */ + public Builder streamResults(boolean streamResults) + { + this.streamResults = streamResults; + return this; + } + public ListKeysOperation build() { return new ListKeysOperation(this); diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java index fa8fc9699..2a9f11c48 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java @@ -16,170 +16,225 @@ package com.basho.riak.client.core.operations.itest; import com.basho.riak.client.core.RiakFuture; -import com.basho.riak.client.core.RiakFutureListener; +import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.ListKeysOperation; import com.basho.riak.client.core.operations.StoreOperation; import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.query.RiakObject; import com.basho.riak.client.core.util.BinaryValue; +import org.junit.Test; + +import java.util.LinkedList; import java.util.List; -import java.util.concurrent.CountDownLatch; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import static org.junit.Assert.*; -import static org.junit.Assume.assumeTrue; -import org.junit.Ignore; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; /** * * @author Brian Roach + * @author Alex Moore */ public class ITestListKeysOperation extends ITestBase { - private Logger logger = LoggerFactory.getLogger("ITestListKeysOperation"); + private final String defaultBucketType = Namespace.DEFAULT_BUCKET_TYPE; + private final String namedBucketType = ITestBase.bucketType.toStringUtf8(); @Test public void testListNoKeysDefaultType() throws InterruptedException, ExecutionException { - testListNoKeys(Namespace.DEFAULT_BUCKET_TYPE); + testListNoKeys(defaultBucketType); + } + + @Test + public void testListNoKeysDefaultTypeStreaming() throws InterruptedException, ExecutionException + { + testListKeysStreaming(defaultBucketType, 0); } @Test public void testListNoKeysTestType() throws InterruptedException, ExecutionException { assumeTrue(testBucketType); - testListNoKeys(bucketType.toString()); + testListNoKeys(namedBucketType); } - private void testListNoKeys(String bucketType) throws InterruptedException, ExecutionException + @Test + public void testListNoKeysTestTypeStreaming() throws InterruptedException, ExecutionException { - Namespace ns = new Namespace(bucketType, bucketName.toString() + "_1"); - ListKeysOperation klistOp = new ListKeysOperation.Builder(ns).build(); - cluster.execute(klistOp); - List kList = klistOp.get().getKeys(); - assertTrue(kList.isEmpty()); - resetAndEmptyBucket(ns); + assumeTrue(testBucketType); + testListKeysStreaming(namedBucketType, 0); } @Test public void testListKeyDefaultType() throws InterruptedException, ExecutionException { - testListKey(Namespace.DEFAULT_BUCKET_TYPE); + testListSingleKey(defaultBucketType); } @Test - public void testListKeyTestType() throws InterruptedException, ExecutionException + public void testListKeyDefaultTypeStreaming() throws InterruptedException, ExecutionException { - assumeTrue(testBucketType); - testListKey(bucketType.toString()); + testListKeysStreaming(defaultBucketType, 1); } - private void testListKey(String bucketType) throws InterruptedException, ExecutionException + @Test + public void testListKeyTestType() throws InterruptedException, ExecutionException { - final Namespace ns = new Namespace(bucketType, bucketName.toString() + "_2"); - final BinaryValue key = BinaryValue.unsafeCreate("my_key".getBytes()); - final String value = "{\"value\":\"value\"}"; - - RiakObject rObj = new RiakObject().setValue(BinaryValue.create(value)); - Location location = new Location(ns, key); - StoreOperation storeOp = - new StoreOperation.Builder(location) - .withContent(rObj) - .build(); - - cluster.execute(storeOp); - storeOp.get(); + assumeTrue(testBucketType); + testListSingleKey(namedBucketType); + } - ListKeysOperation klistOp = new ListKeysOperation.Builder(ns).build(); - cluster.execute(klistOp); - List kList = klistOp.get().getKeys(); - assertEquals(kList.size(), 1); - assertEquals(kList.get(0), key); - resetAndEmptyBucket(ns); + @Test + public void testListKeyTestTypeStreaming() throws InterruptedException, ExecutionException + { + assumeTrue(testBucketType); + testListKeysStreaming(namedBucketType, 1); } @Test public void testLargeKeyListDefaultType() throws InterruptedException, ExecutionException { - testLargeKeyList(Namespace.DEFAULT_BUCKET_TYPE); + testManyKeyList(defaultBucketType, 1000); + } + + @Test + public void testLargeKeyListDefaultTypeStreaming() throws InterruptedException, ExecutionException + { + testListKeysStreaming(defaultBucketType, 1000); } @Test public void testLargeKeyListTestType() throws InterruptedException, ExecutionException { assumeTrue(testBucketType); - testLargeKeyList(bucketType.toString()); + testManyKeyList(namedBucketType, 1000); } - private void testLargeKeyList(String bucketType) throws InterruptedException, ExecutionException + @Test + public void testLargeKeyListTestTypeStreaming() throws InterruptedException, ExecutionException { - final String baseKey = "my_key"; - final String value = "{\"value\":\"value\"}"; - final Namespace ns = new Namespace(bucketType, bucketName.toString() + "_3"); - final Semaphore semaphore = new Semaphore(10); - final CountDownLatch latch = new CountDownLatch(1); - final int expected = 100; + assumeTrue(testBucketType); + testListKeysStreaming(namedBucketType, 1000); + } - RiakFutureListener listener = - new RiakFutureListener() + + + private void testListNoKeys(String bucketType) throws InterruptedException, ExecutionException + { + final Namespace ns = setupBucket(bucketType, 0); + + final List kList = getAllKeyListResults(ns); + + assertTrue(kList.isEmpty()); + resetAndEmptyBucket(ns); + } + + private void testListSingleKey(String bucketType) throws InterruptedException, ExecutionException + { + final Namespace ns = setupBucket(bucketType, 1); + + final List kList = getAllKeyListResults(ns); + + assertEquals(kList.size(), 1); + assertEquals(kList.get(0), createKeyName(0)); + resetAndEmptyBucket(ns); + } + + private void testManyKeyList(String bucketType, int numExpected) throws InterruptedException, ExecutionException + { + final Namespace ns = setupBucket(bucketType, numExpected); + + final List kList = getAllKeyListResults(ns); + + assertEquals(numExpected, kList.size()); + + resetAndEmptyBucket(ns); + } + + private void testListKeysStreaming(String bucketType, int numExpected) throws InterruptedException, ExecutionException + { + final Namespace ns = setupBucket(bucketType, numExpected); + + final ListKeysOperation slko = new ListKeysOperation.Builder(ns).streamResults(true).build(); + final StreamingRiakFuture execute = cluster.execute(slko); + + final BlockingQueue resultsQueue = execute.getResultsQueue(); + List actualKeys = new LinkedList<>(); + int timeouts = 0; + + while(!execute.isDone()) + { + final ListKeysOperation.Response response = resultsQueue.poll(5, TimeUnit.MILLISECONDS); + + if(response != null) { - private final AtomicInteger received = new AtomicInteger(); - - @Override - public void handle(RiakFuture f) - { - try - { - f.get(); - semaphore.release(); - received.incrementAndGet(); - - if (expected == received.intValue()) - { - logger.debug("Executing ListKeys"); - latch.countDown(); - } - } - catch (InterruptedException | ExecutionException ex) - { - throw new RuntimeException(ex); - } - } - }; - - logger.debug("Inserting data"); + actualKeys.addAll(response.getKeys()); + continue; + } - for (int i = 0; i < expected; i++) + timeouts++; + if(timeouts == 10) + { + break; + } + } + + // Grab any last buckets that came in on the last message + for (ListKeysOperation.Response response : resultsQueue) { - semaphore.acquire(); - BinaryValue key = BinaryValue.unsafeCreate((baseKey + i).getBytes()); - RiakObject rObj = new RiakObject().setValue(BinaryValue.create(value)); - Location location = new Location(ns, key); - StoreOperation storeOp = - new StoreOperation.Builder(location) - .withContent(rObj) - .build(); - - storeOp.addListener(listener); - cluster.execute(storeOp); + actualKeys.addAll(response.getKeys()); } - latch.await(2, TimeUnit.MINUTES); + assertEquals(numExpected, actualKeys.size()); - ListKeysOperation klistOp = new ListKeysOperation.Builder(ns).build(); + ITestBase.resetAndEmptyBucket(ns); + } + + private List getAllKeyListResults(Namespace ns) throws InterruptedException, ExecutionException + { + final ListKeysOperation klistOp = new ListKeysOperation.Builder(ns).build(); cluster.execute(klistOp); - List kList; - kList = klistOp.get().getKeys(); - assertEquals(expected, kList.size()); + return klistOp.get().getKeys(); + } - resetAndEmptyBucket(ns); + private Namespace setupBucket(String bucketType, int numExpected) throws InterruptedException + { + final Namespace ns = new Namespace(bucketType, bucketName + "_" + testName.getMethodName()); + storeObjects(ns, numExpected); + return ns; + } + + private void storeObjects(Namespace ns, int expected) throws InterruptedException + { + final String value = "{\"value\":\"value\"}"; + final RiakObject rObj = new RiakObject().setValue(BinaryValue.create(value)); + + for (int i = 0; i < expected; i++) + { + final BinaryValue key = createKeyName(i); + + final Location location = new Location(ns, key); + final StoreOperation storeOp = + new StoreOperation.Builder(location) + .withContent(rObj) + .build(); + + final RiakFuture execute = cluster.execute(storeOp); + execute.await(); + assertTrue(execute.isSuccess()); + } + } + + private BinaryValue createKeyName(int i) + { + final String keyBase = "my_key"; + return BinaryValue.unsafeCreate((keyBase + i).getBytes()); } } From 78c9076dd96865bc1bf3c93cfe30289b6314278d Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Wed, 5 Oct 2016 17:46:30 -0400 Subject: [PATCH 04/55] Add streaming 2i operations/results --- .../riak/client/core/FutureOperation.java | 2 +- .../SecondaryIndexQueryOperation.java | 178 +++++++++--- .../itest/ITestSecondaryIndexQueryOp.java | 257 ++++++++++++++++-- 3 files changed, 373 insertions(+), 64 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/FutureOperation.java b/src/main/java/com/basho/riak/client/core/FutureOperation.java index 1747f837b..8756c70c9 100644 --- a/src/main/java/com/basho/riak/client/core/FutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/FutureOperation.java @@ -215,7 +215,7 @@ protected void processMessage(U decodedMessage) processBatchMessage(decodedMessage); } - private void processBatchMessage(U decodedMessage) + protected void processBatchMessage(U decodedMessage) { this.rawResponses.add(decodedMessage); } diff --git a/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java b/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java index 39097366a..f486787e1 100644 --- a/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java @@ -15,21 +15,22 @@ */ package com.basho.riak.client.core.operations; -import com.basho.riak.client.core.FutureOperation; import com.basho.riak.client.core.RiakMessage; +import com.basho.riak.client.core.StreamingFutureOperation; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.query.indexes.IndexNames; import com.basho.riak.client.core.util.BinaryValue; -import com.basho.riak.protobuf.RiakMessageCodes; import com.basho.riak.protobuf.RiakKvPB; +import com.basho.riak.protobuf.RiakMessageCodes; import com.basho.riak.protobuf.RiakPB.RpbPair; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; /** * @@ -38,18 +39,25 @@ * @since 2.0 */ public class SecondaryIndexQueryOperation - extends FutureOperation + extends StreamingFutureOperation { - private final static Logger logger = LoggerFactory.getLogger(SecondaryIndexQueryOperation.class); private final RiakKvPB.RpbIndexReq pbReq; private final Query query; + private final BlockingQueue responseQueue; + private SecondaryIndexQueryOperation(Builder builder) { + // Decide if we should release results as they come in (stream), or gather them all until the operation is + // done (not stream). + super(builder.streamResults); + // Yo dawg, we don't ever not want to use streaming. builder.pbReqBuilder.setStream(true); this.query = builder.query; this.pbReq = builder.pbReqBuilder.build(); + this.responseQueue = new LinkedBlockingQueue<>(); } @Override @@ -58,50 +66,77 @@ protected SecondaryIndexQueryOperation.Response convert(List rawResponse SecondaryIndexQueryOperation.Response.Builder responseBuilder = new SecondaryIndexQueryOperation.Response.Builder(); + final boolean isIndexBodyResp = rawResponse != null && + !rawResponse.isEmpty() && + objectIsIndexBodyResp(rawResponse.get(0)); + for (Object o : rawResponse) { - if (o instanceof RiakKvPB.RpbIndexBodyResp) - { - assert pbReq.getReturnBody(); - final RiakKvPB.RpbIndexBodyResp bodyResp = (RiakKvPB.RpbIndexBodyResp)o; - convertBodies(responseBuilder, bodyResp); + convertSingleResponse(responseBuilder, isIndexBodyResp, o); + } - if (bodyResp.hasContinuation()) - { - responseBuilder.withContinuation( - BinaryValue.unsafeCreate(bodyResp.getContinuation().toByteArray())); - } - continue; - } + return responseBuilder.build(); + } - final RiakKvPB.RpbIndexResp pbEntry = (RiakKvPB.RpbIndexResp) o; + private boolean objectIsIndexBodyResp(Object o) + { + return o instanceof RiakKvPB.RpbIndexBodyResp; + } - /** - * The 2i API is inconsistent on the Riak side. If it's not - * a range query, return_terms is ignored it only returns the - * list of object keys and you have to have - * preserved the index key if you want to return it to the user - * with the results. - * - * Also, the $key index queries just ignore return_terms altogether. - */ + private void convertSingleResponse(Response.Builder responseBuilder, boolean isIndexBodyResp, Object o) + { + if (isIndexBodyResp) + { + convertIndexBodyResp(responseBuilder, o); + } + else + { + convertIndexResp(responseBuilder, o); + } + } - if (pbReq.getReturnTerms() && !query.indexName.toString().equalsIgnoreCase(IndexNames.KEY)) - { - convertTerms(responseBuilder, pbEntry); - } - else - { - convertKeys(responseBuilder, pbEntry); - } + private void convertIndexBodyResp(Response.Builder responseBuilder, Object o) + { + assert pbReq.getReturnBody(); + final RiakKvPB.RpbIndexBodyResp bodyResp = (RiakKvPB.RpbIndexBodyResp)o; + convertBodies(responseBuilder, bodyResp); - if (pbEntry.hasContinuation()) - { - responseBuilder.withContinuation( - BinaryValue.unsafeCreate(pbEntry.getContinuation().toByteArray())); - } + if (bodyResp.hasContinuation()) + { + responseBuilder.withContinuation( + BinaryValue.unsafeCreate(bodyResp.getContinuation().toByteArray())); + } + } + + + private void convertIndexResp(Response.Builder responseBuilder, Object o) + { + final RiakKvPB.RpbIndexResp pbEntry = (RiakKvPB.RpbIndexResp) o; + + /** + * The 2i API is inconsistent on the Riak side. If it's not + * a range query, return_terms is ignored it only returns the + * list of object keys and you have to have + * preserved the index key if you want to return it to the user + * with the results. + * + * Also, the $key index queries just ignore return_terms altogether. + */ + + if (pbReq.getReturnTerms() && !query.indexName.toString().equalsIgnoreCase(IndexNames.KEY)) + { + convertTerms(responseBuilder, pbEntry); + } + else + { + convertKeys(responseBuilder, pbEntry); + } + + if (pbEntry.hasContinuation()) + { + responseBuilder.withContinuation( + BinaryValue.unsafeCreate(pbEntry.getContinuation().toByteArray())); } - return responseBuilder.build(); } private static void convertKeys(SecondaryIndexQueryOperation.Response.Builder builder, @@ -196,6 +231,38 @@ public Query getQueryInfo() return query; } + @Override + protected void processStreamingChunk(Object rawResponseChunk) + { + SecondaryIndexQueryOperation.Response.Builder responseBuilder = + new SecondaryIndexQueryOperation.Response.Builder(); + + final boolean bodyResp = objectIsIndexBodyResp(rawResponseChunk); + + convertSingleResponse(responseBuilder, bodyResp, rawResponseChunk); + + final Response response = responseBuilder.build(); + + if (response.hasContinuation()) + { + // Return the continuation in the normal fashion as well + final RiakKvPB.RpbIndexResp continuationOnlyResponse = + RiakKvPB.RpbIndexResp.newBuilder().setContinuation( + ByteString.copyFrom(response.getContinuation().unsafeGetValue())) + .build(); + + processBatchMessage(continuationOnlyResponse); + } + + this.responseQueue.add(response); + } + + @Override + public BlockingQueue getResultsQueue() + { + return this.responseQueue; + } + /** * Builder that constructs a QueryOperation. */ @@ -203,6 +270,7 @@ public static class Builder { private final RiakKvPB.RpbIndexReq.Builder pbReqBuilder = RiakKvPB.RpbIndexReq.newBuilder(); private final Query query; + private boolean streamResults = false; /** * Constructs a builder for a QueryOperation. @@ -287,6 +355,26 @@ else if (query.getRangeStart() != null) } } + /** + * Set the streamResults flag. + *

+ * If unset or false, the entire result set will be available through the {@link ListKeysOperation#get()} + * method once the operation is complete. + *

+ * If set to true, results will be pushed to the queue available through the + * {@link ListKeysOperation#getResultsQueue()} + * method as soon as they are available. + * + * @param streamResults whether to stream results to {@link ListKeysOperation#get()}(false), or + * {@link ListKeysOperation#getResultsQueue()}(true) + * @return A reference to this object. + */ + public Builder streamResults(boolean streamResults) + { + this.streamResults = streamResults; + return this; + } + /** * Construct a new QueryOperation. * @return a QueryOperation @@ -741,6 +829,12 @@ Builder addEntry(Response.Entry entry) return this; } + Builder addAllEntries(Collection entries) + { + entryList.addAll(entries); + return this; + } + Response build() { return new Response(this); diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestSecondaryIndexQueryOp.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestSecondaryIndexQueryOp.java index 5d7fa708f..e78e4aefe 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestSecondaryIndexQueryOp.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestSecondaryIndexQueryOp.java @@ -15,6 +15,7 @@ */ package com.basho.riak.client.core.operations.itest; +import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation; import com.basho.riak.client.core.operations.StoreOperation; import com.basho.riak.client.core.query.Location; @@ -24,10 +25,17 @@ import com.basho.riak.client.core.query.indexes.LongIntIndex; import com.basho.riak.client.core.query.indexes.StringBinIndex; import com.basho.riak.client.core.util.BinaryValue; -import org.junit.*; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; +import java.util.LinkedList; +import java.util.List; import java.util.Random; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.*; @@ -80,6 +88,13 @@ public void testSingleQuerySingleResponseDefaultType() throws InterruptedExcepti testSingleQuerySingleResponse(defaultTypeNamespace); } + @Test + public void testSingleQuerySingleResponseDefaultTypeStreaming() throws InterruptedException, ExecutionException + { + Assume.assumeTrue(test2i); + testSingleQuerySingleResponseStreaming(defaultTypeNamespace); + } + @Test public void testSingleQuerySingleResponseTestType() throws InterruptedException, ExecutionException { @@ -133,6 +148,12 @@ public void testNoSortWithNoPagingTestType() throws InterruptedException, Execut testNoSortWithNoPaging(typedNamespace); } + @Test + public void testNoSortWithNoPagingTestTypeStreaming() throws InterruptedException, ExecutionException + { + testNoSortWithNoPagingStreaming(typedNamespace); + } + @Test public void testSortWithNoPagingDefaultType() throws InterruptedException, ExecutionException { @@ -221,6 +242,14 @@ public void testSortWithPagingTestType() throws InterruptedException, ExecutionE testSortWithPaging(typedNamespace); } + @Test + public void testSortWithPagingTestTypeStreaming() throws InterruptedException, ExecutionException + { + Assume.assumeTrue(test2i); + Assume.assumeTrue(testBucketType); + testSortWithPagingStreaming(typedNamespace); + } + @Test public void testRegexTermFilterDefaultType() throws InterruptedException, ExecutionException { @@ -287,6 +316,70 @@ private void testSingleQuerySingleResponse(Namespace namespace) throws Interrupt assertEquals(keyBase + "5", response.getEntryList().get(0).getObjectKey().toString()); } + private void testSingleQuerySingleResponseStreaming(Namespace namespace) throws InterruptedException, ExecutionException + { + { + SecondaryIndexQueryOperation.Query query = + new SecondaryIndexQueryOperation.Query.Builder(namespace, incrementingIndexName) + .withIndexKey(BinaryValue.unsafeCreate(String.valueOf(5L).getBytes())) + .build(); + + SecondaryIndexQueryOperation queryOp = + new SecondaryIndexQueryOperation.Builder(query).streamResults(true).build(); + + final StreamingRiakFuture + future = cluster.execute(queryOp); + + SecondaryIndexQueryOperation.Response syncResponse = queryOp.get(); + + List streamedResponses = new LinkedList<>(); + for (SecondaryIndexQueryOperation.Response r : future.getResultsQueue()) + { + streamedResponses.addAll(r.getEntryList()); + } + + assertEquals(0, syncResponse.getEntryList().size()); + assertFalse(syncResponse.hasContinuation()); + + assertEquals(1, streamedResponses.size()); + final SecondaryIndexQueryOperation.Response.Entry entry = streamedResponses.get(0); + assertFalse(entry.hasIndexKey()); + assertEquals(keyBase + "5", entry.getObjectKey().toString()); + assertFalse(syncResponse.hasContinuation()); + } + + { + SecondaryIndexQueryOperation.Query query2 = + new SecondaryIndexQueryOperation.Query.Builder(namespace, incrementingIndexName) + .withIndexKey(BinaryValue.unsafeCreate(String.valueOf(5L).getBytes())) + .withReturnKeyAndIndex(true) + .build(); + + SecondaryIndexQueryOperation queryOp2 = + new SecondaryIndexQueryOperation.Builder(query2).streamResults(true).build(); + + final StreamingRiakFuture + future2 = cluster.execute(queryOp2); + + SecondaryIndexQueryOperation.Response syncResponse2 = queryOp2.get(); + + LinkedList streamedResponses2 = new LinkedList<>(); + for (SecondaryIndexQueryOperation.Response r : future2.getResultsQueue()) + { + streamedResponses2.addAll(r.getEntryList()); + } + + assertEquals(0, syncResponse2.getEntryList().size()); + assertFalse(syncResponse2.hasContinuation()); + + assertEquals(1, streamedResponses2.size()); + final SecondaryIndexQueryOperation.Response.Entry entry2 = streamedResponses2.get(0); + assertTrue(entry2.hasIndexKey()); + assertEquals(BinaryValue.unsafeCreate("5".getBytes()), entry2.getIndexKey()); + assertEquals(keyBase + "5", entry2.getObjectKey().toString()); + } + } + private void testSingleQueryMultipleResponse(Namespace namespace) throws InterruptedException, ExecutionException { SecondaryIndexQueryOperation.Query query = @@ -296,8 +389,7 @@ private void testSingleQueryMultipleResponse(Namespace namespace) throws Interru .build(); SecondaryIndexQueryOperation queryOp = - new SecondaryIndexQueryOperation.Builder(query) - .build(); + new SecondaryIndexQueryOperation.Builder(query).build(); cluster.execute(queryOp); SecondaryIndexQueryOperation.Response response = queryOp.get(); @@ -306,16 +398,13 @@ private void testSingleQueryMultipleResponse(Namespace namespace) throws Interru assertFalse(response.getEntryList().get(0).hasIndexKey()); assertEquals(keyBase + "0", response.getEntryList().get(0).getObjectKey().toString()); - query = - new SecondaryIndexQueryOperation.Query.Builder(namespace, allFivesIndexName) + query = new SecondaryIndexQueryOperation.Query.Builder(namespace, allFivesIndexName) .withIndexKey(BinaryValue.unsafeCreate(String.valueOf(5L).getBytes())) .withPaginationSort(true) .withReturnKeyAndIndex(true) .build(); - queryOp = - new SecondaryIndexQueryOperation.Builder(query) - .build(); + queryOp = new SecondaryIndexQueryOperation.Builder(query).build(); cluster.execute(queryOp); response = queryOp.get(); @@ -346,17 +435,14 @@ private void testRangeQuery(Namespace namespace) throws InterruptedException, Ex assertFalse(response.getEntryList().get(0).hasIndexKey()); assertEquals(keyBase + "5", response.getEntryList().get(0).getObjectKey().toString()); - query = - new SecondaryIndexQueryOperation.Query.Builder(namespace, incrementingIndexName) + query = new SecondaryIndexQueryOperation.Query.Builder(namespace, incrementingIndexName) .withRangeStart(BinaryValue.unsafeCreate(String.valueOf(5L).getBytes())) .withRangeEnd(BinaryValue.unsafeCreate(String.valueOf(20L).getBytes())) .withReturnKeyAndIndex(true) .withPaginationSort(true) .build(); - queryOp = - new SecondaryIndexQueryOperation.Builder(query) - .build(); + queryOp = new SecondaryIndexQueryOperation.Builder(query).build(); cluster.execute(queryOp); response = queryOp.get(); @@ -386,6 +472,33 @@ private void testNoSortWithNoPaging(Namespace namespace) throws InterruptedExcep assertEquals(100, response.getEntryList().size()); } + private void testNoSortWithNoPagingStreaming(Namespace namespace) throws InterruptedException, ExecutionException + { + SecondaryIndexQueryOperation.Query query = + new SecondaryIndexQueryOperation.Query.Builder(namespace, incrementingIndexName) + .withRangeStart(BinaryValue.unsafeCreate(String.valueOf(0L).getBytes())) + .withRangeEnd(BinaryValue.unsafeCreate(String.valueOf(100L).getBytes())) + .withPaginationSort(false) + .build(); + + SecondaryIndexQueryOperation queryOp = + new SecondaryIndexQueryOperation.Builder(query) + .streamResults(true) + .build(); + + final StreamingRiakFuture + future = cluster.execute(queryOp); + + StreamingResultsGatherer streamingResultsGatherer = new StreamingResultsGatherer(future).invoke(); + List resultsList = streamingResultsGatherer.getResultsList(); + BinaryValue continuation = streamingResultsGatherer.getContinuation(); + + assertEquals(100, resultsList.size()); + assertEquals(0, future.get().getEntryList().size()); + assertNull(continuation); + assertFalse(future.get().hasContinuation()); + } + private void testSortWithNoPaging(Namespace namespace) throws InterruptedException, ExecutionException { SecondaryIndexQueryOperation.Query query = @@ -404,7 +517,7 @@ private void testSortWithNoPaging(Namespace namespace) throws InterruptedExcepti assertEquals(100, response.getEntryList().size()); - AssertLongObjectsInOrder(response); + AssertLongObjectsInOrder(response.getEntryList()); } private void testNoSortWithPaging(Namespace namespace) throws InterruptedException, ExecutionException @@ -448,7 +561,38 @@ private void testSortWithPaging(Namespace namespace) throws InterruptedException assertEquals(20, response.getEntryList().size()); - AssertLongObjectsInOrder(response); + AssertLongObjectsInOrder(response.getEntryList()); + } + + private void testSortWithPagingStreaming(Namespace namespace) throws InterruptedException, ExecutionException + { + SecondaryIndexQueryOperation.Query query = + new SecondaryIndexQueryOperation.Query.Builder(namespace, incrementingIndexName) + .withRangeStart(BinaryValue.unsafeCreate(String.valueOf(0L).getBytes())) + .withRangeEnd(BinaryValue.unsafeCreate(String.valueOf(100L).getBytes())) + .withPaginationSort(true) + .withMaxResults(20) + .build(); + + SecondaryIndexQueryOperation queryOp = + new SecondaryIndexQueryOperation.Builder(query) + .streamResults(true) + .build(); + + final StreamingRiakFuture + future = cluster.execute(queryOp); + + StreamingResultsGatherer streamingResultsGatherer = new StreamingResultsGatherer(future).invoke(); + List resultsList = streamingResultsGatherer.getResultsList(); + BinaryValue continuation = streamingResultsGatherer.getContinuation(); + + assertEquals(20, resultsList.size()); + assertEquals(0, future.get().getEntryList().size()); + assertTrue(future.get().hasContinuation()); + assertNotNull(continuation); + assertEquals(continuation, future.get().getContinuation()); + + AssertLongObjectsInOrder(resultsList); } private void testRegexTermFilter(Namespace namespace) throws InterruptedException, ExecutionException @@ -463,8 +607,7 @@ private void testRegexTermFilter(Namespace namespace) throws InterruptedExceptio .build(); SecondaryIndexQueryOperation queryOp = - new SecondaryIndexQueryOperation.Builder(query) - .build(); + new SecondaryIndexQueryOperation.Builder(query).build(); cluster.execute(queryOp); SecondaryIndexQueryOperation.Response response = queryOp.get(); @@ -521,17 +664,89 @@ private static void setupIndexTestData(Namespace ns) } } - private void AssertLongObjectsInOrder(SecondaryIndexQueryOperation.Response response) + private void AssertLongObjectsInOrder(List entryList) { - final String firstKey = response.getEntryList().get(0).getObjectKey().toString(); + final String firstKey = entryList.get(0).getObjectKey().toString(); Long previousKey = Long.parseLong(firstKey.substring(keyBase.length())); - for (int j = 1; j < response.getEntryList().size(); j++) + for (int j = 1; j < entryList.size(); j++) { - String fullKey = response.getEntryList().get(j).getObjectKey().toString(); + String fullKey = entryList.get(j).getObjectKey().toString(); Long currentKey = Long.parseLong(fullKey.substring(keyBase.length())); assertTrue(previousKey <= currentKey); previousKey = currentKey; } } + + private class StreamingResultsGatherer + { + private StreamingRiakFuture future; + private List resultsList; + private BinaryValue continuation; + + public StreamingResultsGatherer(StreamingRiakFuture future) + { + this.future = future; + } + + public List getResultsList() + { + return resultsList; + } + + public BinaryValue getContinuation() + { + return continuation; + } + + public StreamingResultsGatherer invoke() throws InterruptedException + { + final BlockingQueue resultsQueue = future.getResultsQueue(); + + resultsList = new LinkedList<>(); + + continuation = null; + int timeouts = 0; + + while(!future.isDone()) + { + final SecondaryIndexQueryOperation.Response response = resultsQueue.poll(5, TimeUnit.MILLISECONDS); + + if(response == null) + { + if (++timeouts == 10) + { + break; + } + else + { + continue; + } + } + + continuation = addResponseToResults(resultsList, continuation, response); + } + + for (SecondaryIndexQueryOperation.Response response : resultsQueue) + { + continuation = addResponseToResults(resultsList, continuation, response); + } + return this; + } + + private BinaryValue addResponseToResults(List resultsList, + BinaryValue continuation, + SecondaryIndexQueryOperation.Response response) + { + resultsList.addAll(response.getEntryList()); + + if(response.hasContinuation() && continuation == null) + { + continuation = response.getContinuation(); + } + + return continuation; + } + } } From 590bcdeb191eda5b7cf93cd520f06749f363fe03 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Mon, 10 Oct 2016 15:15:06 -0400 Subject: [PATCH 05/55] Use TransferQueues instead of BlockingQueues --- .../basho/riak/client/core/StreamingRiakFuture.java | 3 ++- .../core/operations/ListBucketsOperation.java | 10 +++++----- .../client/core/operations/ListKeysOperation.java | 13 ++++++------- .../operations/SecondaryIndexQueryOperation.java | 11 +++++------ 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/StreamingRiakFuture.java b/src/main/java/com/basho/riak/client/core/StreamingRiakFuture.java index 895ba1d6d..8e4043186 100644 --- a/src/main/java/com/basho/riak/client/core/StreamingRiakFuture.java +++ b/src/main/java/com/basho/riak/client/core/StreamingRiakFuture.java @@ -16,6 +16,7 @@ package com.basho.riak.client.core; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TransferQueue; /** * The result of an asynchronous streaming (chunked) Riak operation. @@ -33,5 +34,5 @@ public interface StreamingRiakFuture * An Queue that provides the stream of results as they return from Riak. * @return An Queue. */ - BlockingQueue getResultsQueue(); + TransferQueue getResultsQueue(); } diff --git a/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java index 97d3f911c..358a39c6b 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java @@ -26,8 +26,8 @@ import java.util.ArrayList; import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.TransferQueue; public class ListBucketsOperation extends StreamingFutureOperation responseQueue; + private final TransferQueue responseQueue; private ListBucketsOperation(Builder builder) { super(builder.streamResults); this.reqBuilder = builder.reqBuilder; this.bucketType = builder.bucketType; - this.responseQueue = new LinkedBlockingQueue<>(); + this.responseQueue = new LinkedTransferQueue<>(); } @Override @@ -108,7 +108,7 @@ public BinaryValue getQueryInfo() } @Override - public BlockingQueue getResultsQueue() + public TransferQueue getResultsQueue() { return this.responseQueue; } diff --git a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java index 1aad8b11d..45f95f3a6 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java @@ -19,8 +19,8 @@ import com.basho.riak.client.core.StreamingFutureOperation; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.util.BinaryValue; -import com.basho.riak.protobuf.RiakMessageCodes; import com.basho.riak.protobuf.RiakKvPB; +import com.basho.riak.protobuf.RiakMessageCodes; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import org.slf4j.Logger; @@ -28,15 +28,14 @@ import java.util.ArrayList; import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.TransferQueue; public class ListKeysOperation extends StreamingFutureOperation { - private final Logger logger = LoggerFactory.getLogger("ListKeysOperation"); private final Namespace namespace; private final RiakKvPB.RpbListKeysReq.Builder reqBuilder; - private final BlockingQueue responseQueue; + private final LinkedTransferQueue responseQueue; private ListKeysOperation(Builder builder) @@ -44,7 +43,7 @@ private ListKeysOperation(Builder builder) super(builder.streamResults); this.reqBuilder = builder.reqBuilder; this.namespace = builder.namespace; - this.responseQueue = new LinkedBlockingQueue<>(); + this.responseQueue = new LinkedTransferQueue<>(); } @Override @@ -109,7 +108,7 @@ protected void processStreamingChunk(RiakKvPB.RpbListKeysResp rawResponseChunk) } @Override - public BlockingQueue getResultsQueue() + public TransferQueue getResultsQueue() { return this.responseQueue; } diff --git a/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java b/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java index f486787e1..7969f16af 100644 --- a/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java @@ -29,8 +29,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.TransferQueue; /** * @@ -44,8 +44,7 @@ public class SecondaryIndexQueryOperation { private final RiakKvPB.RpbIndexReq pbReq; private final Query query; - private final BlockingQueue responseQueue; - + private final TransferQueue responseQueue; private SecondaryIndexQueryOperation(Builder builder) { @@ -57,7 +56,7 @@ private SecondaryIndexQueryOperation(Builder builder) builder.pbReqBuilder.setStream(true); this.query = builder.query; this.pbReq = builder.pbReqBuilder.build(); - this.responseQueue = new LinkedBlockingQueue<>(); + this.responseQueue = new LinkedTransferQueue<>(); } @Override @@ -258,7 +257,7 @@ protected void processStreamingChunk(Object rawResponseChunk) } @Override - public BlockingQueue getResultsQueue() + public TransferQueue getResultsQueue() { return this.responseQueue; } From 5b87e248ac4ccf2475a3779202284b646fff5fd3 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 11 Oct 2016 13:10:15 -0400 Subject: [PATCH 06/55] Add Streaming MapReduce Operations --- .../itest/ITestMapReduceOperation.java | 181 ++++++++++++++---- 1 file changed, 145 insertions(+), 36 deletions(-) diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestMapReduceOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestMapReduceOperation.java index 31936c492..9eacaf272 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestMapReduceOperation.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestMapReduceOperation.java @@ -15,19 +15,26 @@ */ package com.basho.riak.client.core.operations.itest; +import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.MapReduceOperation; import com.basho.riak.client.core.operations.StoreOperation; import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.query.RiakObject; import com.basho.riak.client.core.util.BinaryValue; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import java.io.IOException; +import java.util.LinkedHashMap; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TransferQueue; import static org.junit.Assert.*; import static org.junit.Assume.assumeTrue; @@ -37,8 +44,31 @@ * @author Brian Roach * @since 2.0 */ -public class ITestMapReduceOperation extends ITestAutoCleanupBase +public class ITestMapReduceOperation extends ITestBase { + @BeforeClass + public static void setup() throws ExecutionException, InterruptedException + { + insertData(new Namespace(Namespace.DEFAULT_BUCKET_TYPE, bucketName.toString())); + + if(testBucketType) + { + insertData(new Namespace(bucketType, bucketName)); + } + Thread.sleep(2000); + } + + @AfterClass + public static void cleanup() throws ExecutionException, InterruptedException + { + resetAndEmptyBucket(new Namespace(Namespace.DEFAULT_BUCKET_TYPE, bucketName.toString())); + + if(testBucketType) + { + resetAndEmptyBucket(new Namespace(bucketType, bucketName)); + } + } + @Test public void testBasicMRDefaultType() throws InterruptedException, ExecutionException, IOException { @@ -58,7 +88,121 @@ public void testBasicMRTestType() throws InterruptedException, ExecutionExceptio assertEquals(Integer.valueOf(8), resultMap.get("the")); } + @Test + public void testBasicStreamingMRTestType() throws IOException, InterruptedException + { + assumeTrue(testBucketType); + Map> phaseResultMap = testBasicStreamingMR(new Namespace(bucketType, bucketName)); + Map resultMap = phaseResultMap.get(1); + assertNotNull(resultMap.containsKey("the")); + assertEquals(Integer.valueOf(8), resultMap.get("the")); + + } + private Map testBasicMR(Namespace namespace) throws InterruptedException, ExecutionException, IOException + { + final ObjectMapper objectMapper = new ObjectMapper(); + String query = createMapReduceQuery(namespace); + + MapReduceOperation mrOp = + new MapReduceOperation.Builder(BinaryValue.unsafeCreate(query.getBytes())) + .build(); + + cluster.execute(mrOp); + mrOp.await(); + assertTrue(mrOp.isSuccess()); + ArrayNode resultList = mrOp.get().getResults().get(1); + + // The query should return one result which is a JSON array containing a + // single JSON object that is a asSet of word counts. + assertEquals(resultList.size(), 1); + + String json = resultList.get(0).toString(); + @SuppressWarnings("unchecked") + Map resultMap = objectMapper.readValue(json, Map.class); + + return resultMap; + } + + private Map> testBasicStreamingMR(Namespace namespace) throws InterruptedException, IOException + { + final String query = createMapReduceQuery(namespace); + MapReduceOperation mrOp = new MapReduceOperation.Builder(BinaryValue.unsafeCreate(query.getBytes())) + .streamResults(true) + .build(); + + final StreamingRiakFuture streamFuture = cluster.execute(mrOp); + + final TransferQueue resultsQueue = streamFuture.getResultsQueue(); + + + Map> resultMap = new LinkedHashMap<>(); + + while(!streamFuture.isDone()) + { + final MapReduceOperation.Response response = resultsQueue.poll(10, TimeUnit.MILLISECONDS); + if(response == null) + { + continue; + } + mergeWordCountMaps(resultMap, response); + } + + for (MapReduceOperation.Response response : resultsQueue) + { + mergeWordCountMaps(resultMap, response); + } + + return resultMap; + } + + private void mergeWordCountMaps(Map> resultMap, + MapReduceOperation.Response result) throws IOException + { + final ObjectMapper objectMapper = new ObjectMapper(); + + for (Integer phaseNumber : result.getResults().keySet()) + { + resultMap.putIfAbsent(phaseNumber, new LinkedHashMap<>()); + final Map currentPhaseMap = resultMap.get(phaseNumber); + + final ArrayNode resultList = result.getResults().get(phaseNumber); + final String wordCountMapJsonString = resultList.get(0).toString(); + @SuppressWarnings("unchecked") + Map wordCountMap = objectMapper.readValue(wordCountMapJsonString, Map.class); + + for(String wordCountKey: wordCountMap.keySet()) + { + if(currentPhaseMap.containsKey(wordCountKey)) + { + final int newWordCountSum = currentPhaseMap.get(wordCountKey) + wordCountMap.get(wordCountKey); + currentPhaseMap.put(wordCountKey, newWordCountSum); + } + else + { + currentPhaseMap.put(wordCountKey, wordCountMap.get(wordCountKey)); + } + } + } + } + + private String createMapReduceQuery(Namespace namespace) + { + final String bucketType = namespace.getBucketTypeAsString(); + final String bucketName = namespace.getBucketNameAsString(); + + return "{\"inputs\":[[\"" + bucketName + "\",\"p1\",\"\",\"" + bucketType + "\"]," + + "[\"" + bucketName + "\",\"p2\",\"\",\"" + bucketType + "\"]," + + "[\"" + bucketName + "\",\"p3\",\"\",\"" + bucketType + "\"]]," + + "\"query\":[{\"map\":{\"language\":\"javascript\",\"source\":\"" + + "function(v) {var m = v.values[0].data.toLowerCase().match(/\\w*/g); var r = [];" + + "for (var i in m) {if (m[i] != '') {var o = {};o[m[i]] = 1;r.push(o);}}return r;}" + + "\"}},{\"reduce\":{\"language\":\"javascript\",\"source\":\"" + + "function(v) {var r = {};for (var i in v) {for(var w in v[i]) {if (w in r) r[w] += v[i][w];" + + "else r[w] = v[i][w];}}return [r];}\"}}]}"; + } + + private static void insertData(Namespace namespace) throws InterruptedException, ExecutionException { RiakObject obj = new RiakObject(); @@ -103,40 +247,5 @@ private Map testBasicMR(Namespace namespace) throws Interrupted cluster.execute(storeOp); storeOp.get(); - - String bName = namespace.getBucketNameAsString(); - String bType = namespace.getBucketTypeAsString(); - - String query = "{\"inputs\":[[\"" + bName + "\",\"p1\",\"\",\"" + bType + "\"]," + - "[\"" + bName + "\",\"p2\",\"\",\"" + bType + "\"]," + - "[\"" + bName + "\",\"p3\",\"\",\"" + bType + "\"]]," + - "\"query\":[{\"map\":{\"language\":\"javascript\",\"source\":\"" + - "function(v) {var m = v.values[0].data.toLowerCase().match(/\\w*/g); var r = [];" + - "for (var i in m) {if (m[i] != '') {var o = {};o[m[i]] = 1;r.push(o);}}return r;}" + - "\"}},{\"reduce\":{\"language\":\"javascript\",\"source\":\"" + - "function(v) {var r = {};for (var i in v) {for(var w in v[i]) {if (w in r) r[w] += v[i][w];" + - "else r[w] = v[i][w];}}return [r];}\"}}]}"; - - MapReduceOperation mrOp = - new MapReduceOperation.Builder(BinaryValue.unsafeCreate(query.getBytes())) - .build(); - - cluster.execute(mrOp); - mrOp.await(); - assertTrue(mrOp.isSuccess()); - ArrayNode resultList = mrOp.get().getResults().get(1); - - // The query should return one result which is a JSON array containing a - // single JSON object that is a asSet of word counts. - assertEquals(resultList.size(), 1); - - String json = resultList.get(0).toString(); - ObjectMapper oMapper = new ObjectMapper(); - @SuppressWarnings("unchecked") - Map resultMap = oMapper.readValue(json, Map.class); - - resetAndEmptyBucket(namespace); - - return resultMap; } } From 69aed60bd39bf4ab8a40b34d29e6e7092ee62f29 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 11 Oct 2016 13:16:45 -0400 Subject: [PATCH 07/55] Forgot a file --- .../core/operations/MapReduceOperation.java | 132 ++++++++++++------ 1 file changed, 90 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java b/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java index da810298c..7b7d99090 100644 --- a/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java @@ -15,8 +15,8 @@ */ package com.basho.riak.client.core.operations; -import com.basho.riak.client.core.FutureOperation; import com.basho.riak.client.core.RiakMessage; +import com.basho.riak.client.core.StreamingFutureOperation; import com.basho.riak.client.core.util.BinaryValue; import com.basho.riak.protobuf.RiakMessageCodes; import com.basho.riak.protobuf.RiakKvPB; @@ -31,6 +31,9 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.LinkedTransferQueue; +import java.util.concurrent.TransferQueue; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,72 +41,81 @@ * A Map/Reduce Operation on Riak. No error checking is done on the content type of the content itself * with the exception to making sure they are provided. */ -public class MapReduceOperation extends FutureOperation +public class MapReduceOperation extends StreamingFutureOperation { private final RiakKvPB.RpbMapRedReq.Builder reqBuilder; private final BinaryValue mapReduce; private final Logger logger = LoggerFactory.getLogger(MapReduceOperation.class); + private final TransferQueue responseQueue; + private final ObjectMapper objectMapper = new ObjectMapper(); + private final JsonNodeFactory jsonNodeFactory = JsonNodeFactory.instance; + private MapReduceOperation(Builder builder) { + super(builder.streamResults); this.reqBuilder = builder.reqBuilder; this.mapReduce = builder.mapReduce; + this.responseQueue = new LinkedTransferQueue<>(); } @Override protected Response convert(List rawResponse) + { + final Map resultMap = new LinkedHashMap<>(); + + for (RiakKvPB.RpbMapRedResp response : rawResponse) + { + convertSingleResponse(resultMap, response); + } + return new Response(resultMap); + } + + private void convertSingleResponse(Map resultMap, + RiakKvPB.RpbMapRedResp response) { // Riak streams the result back. Each message from Riak contains a int // that tells you what phase the result is from. The result from a phase // can span multiple messages. Each result chunk is a JSON array. - final JsonNodeFactory factory = JsonNodeFactory.instance; - final ObjectMapper mapper = new ObjectMapper(); - final Map resultMap = new LinkedHashMap<>(); - int phase = 0; - - for (RiakKvPB.RpbMapRedResp response : rawResponse) + if (response.hasPhase()) { - if (response.hasPhase()) + phase = response.getPhase(); + } + if (response.hasResponse()) + { + ArrayNode jsonArray; + if (resultMap.containsKey(phase)) + { + jsonArray = resultMap.get(phase); + } + else + { + jsonArray = jsonNodeFactory.arrayNode(); + resultMap.put(phase, jsonArray); + } + + JsonNode responseJson; + try { - phase = response.getPhase(); + responseJson = objectMapper.readTree(response.getResponse().toStringUtf8()); } - if (response.hasResponse()) + catch (IOException ex) { - ArrayNode jsonArray; - if (resultMap.containsKey(phase)) - { - jsonArray = resultMap.get(phase); - } - else - { - jsonArray = factory.arrayNode(); - resultMap.put(phase, jsonArray); - } - - JsonNode responseJson; - try - { - responseJson = mapper.readTree(response.getResponse().toStringUtf8()); - } - catch (IOException ex) - { - logger.error("Mapreduce job returned non-JSON; {}",response.getResponse().toStringUtf8()); - throw new RuntimeException("Non-JSON response from MR job", ex); - } - - if (responseJson.isArray()) - { - jsonArray.addAll((ArrayNode)responseJson); - } - else - { - logger.error("Mapreduce job returned JSON that wasn't an array; {}", response.getResponse().toStringUtf8()); - } + logger.error("Mapreduce job returned non-JSON; {}",response.getResponse().toStringUtf8()); + throw new RuntimeException("Non-JSON response from MR job", ex); + } + + if (responseJson.isArray()) + { + jsonArray.addAll((ArrayNode)responseJson); + } + else + { + logger.error("Mapreduce job returned JSON that wasn't an array; {}", response.getResponse().toStringUtf8()); } } - return new Response(resultMap); } @Override @@ -139,11 +151,29 @@ public BinaryValue getQueryInfo() return mapReduce; } + @Override + protected void processStreamingChunk(RiakKvPB.RpbMapRedResp rawResponseChunk) + { + final Map resultMap = new LinkedHashMap<>(); + + convertSingleResponse(resultMap, rawResponseChunk); + + final Response chunkResponse = new Response(resultMap); + this.responseQueue.offer(chunkResponse); + } + + @Override + public TransferQueue getResultsQueue() + { + return this.responseQueue; + } + public static class Builder { private final RiakKvPB.RpbMapRedReq.Builder reqBuilder = RiakKvPB.RpbMapRedReq.newBuilder(); private final BinaryValue mapReduce; + private boolean streamResults; /** * Create a MapReduce operation builder with the given function. @@ -162,6 +192,24 @@ public Builder(BinaryValue mapReduce) this.mapReduce = mapReduce; } + /** + * Set the streamResults flag. + * + * If unset or false, the entire result set will be available through the {@link MapReduceOperation#get()} + * method once the operation is complete. + * + * If set to true, results will be pushed to the queue available through the {@link MapReduceOperation#getResultsQueue()} + * method as soon as they are available. + * + * @param streamResults whether to stream results to {@link MapReduceOperation#get()}(false), or {@link MapReduceOperation#getResultsQueue()}(true) + * @return A reference to this object. + */ + public Builder streamResults(boolean streamResults) + { + this.streamResults = streamResults; + return this; + } + public MapReduceOperation build() { return new MapReduceOperation(this); From c7c40dd1b55b5403142e7eacb35cfb3b73bb1c09 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Thu, 13 Oct 2016 13:54:49 -0400 Subject: [PATCH 08/55] Command level streaming for ListKeys and ListBuckets --- .../com/basho/riak/client/api/RiakClient.java | 22 ++++ .../client/api/StreamableRiakCommand.java | 22 ++++ .../api/commands/ChunkedQueueIterator.java | 96 ++++++++++++++++ .../commands/ImmediateCoreFutureAdapter.java | 69 ++++++++++++ .../api/commands/buckets/ListBuckets.java | 51 ++++++++- .../riak/client/api/commands/kv/ListKeys.java | 52 ++++++++- .../core/operations/ListBucketsOperation.java | 9 +- .../core/operations/ListKeysOperation.java | 9 +- .../buckets/itest/ITestListBuckets.java | 102 +++++++++++++---- .../commands/buckets/itest/ITestListKeys.java | 106 ++++++++++++++++++ 10 files changed, 508 insertions(+), 30 deletions(-) create mode 100644 src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java create mode 100644 src/main/java/com/basho/riak/client/api/commands/ChunkedQueueIterator.java create mode 100644 src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java create mode 100644 src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java diff --git a/src/main/java/com/basho/riak/client/api/RiakClient.java b/src/main/java/com/basho/riak/client/api/RiakClient.java index 114c31b38..346a097fd 100644 --- a/src/main/java/com/basho/riak/client/api/RiakClient.java +++ b/src/main/java/com/basho/riak/client/api/RiakClient.java @@ -402,6 +402,28 @@ public RiakFuture executeAsync(RiakCommand command) return command.executeAsync(cluster); } + /** + * Execute a StreamableRiakCommand asynchronously, and stream the results back before the command is complete. + *

+ * Calling this method causes the client to execute the provided StreamableRiakCommand + * asynchronously. It will immediately return a RiakFuture that contains an immediately available result that data + * will be streamed to. The RiakFuture will also keep track of the overall operation's progress with the + * {@see RiakFuture#isDone}, etc methods. + * + * @param StreamableRiakCommand's immediate return type, available before the command/operation is complete. + * @param The RiakCommand's query info type. + * @param command The RiakCommand to execute. + * @param timeoutMS The loading timeout in milliseconds for each result chunk. + * If the timeout is reached a {@see null} will be returned from the result's iterator, + * instead of blocking indefinitely. + * @return a RiakFuture for the operation + * @see RiakFuture + */ + public RiakFuture executeAsyncStreaming(StreamableRiakCommand command, int timeoutMS) + { + return command.executeAsyncStreaming(cluster, timeoutMS); + } + /** * Shut down the client and the underlying RiakCluster. *

diff --git a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java new file mode 100644 index 000000000..dfa2e9918 --- /dev/null +++ b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java @@ -0,0 +1,22 @@ +package com.basho.riak.client.api; + +import com.basho.riak.client.core.RiakCluster; +import com.basho.riak.client.core.RiakFuture; + +/* + * The base class for all Streamable Riak Commands. + * Allows the user to either use {@link RiakCommand#executeAsync} and return a "batch-mode" result + * that is only available after the command is complete, or + * use {@link StreamableRiakCommand#executeAsyncStreaming} and return a "immediate" or "stream-mode" result + * that data will flow into. + * @param The response type returned by "streaming mode" {@link executeAsyncStreaming} + * @param The response type returned by the "batch mode" @{link executeAsync} + * @param The query info type + * @author Dave Rusek + * @author Brian Roach + * @since 2.0 + */ +public abstract class StreamableRiakCommand extends RiakCommand +{ + protected abstract RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout); +} diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedQueueIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedQueueIterator.java new file mode 100644 index 000000000..7fbf851e2 --- /dev/null +++ b/src/main/java/com/basho/riak/client/api/commands/ChunkedQueueIterator.java @@ -0,0 +1,96 @@ +package com.basho.riak.client.api.commands; + +import com.basho.riak.client.core.StreamingRiakFuture; + +import java.util.Iterator; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TransferQueue; +import java.util.function.Function; + +public class ChunkedQueueIterator, CoreT> implements Iterator +{ + private final int timeout; + private final StreamingRiakFuture coreFuture; + private final TransferQueue chunkQueue; + private final Function createNext; + private final Function> getNextIterator; + + private Iterator currentIterator = null; + + public ChunkedQueueIterator(StreamingRiakFuture coreFuture, + int pollTimeout, + Function createNext, + Function> getNextIterator) + { + this.timeout = pollTimeout; + this.coreFuture = coreFuture; + this.chunkQueue = coreFuture.getResultsQueue(); + this.createNext = createNext; + this.getNextIterator = getNextIterator; + } + + @Override + public boolean hasNext() + { + return currentIteratorHasNext() || possibleChunksRemaining(); + } + + private boolean currentIteratorHasNext() + { + return currentIterator != null && currentIterator.hasNext(); + } + + private boolean possibleChunksRemaining() + { + // Chunks may remain if : + // Core Operation Not Done OR items still in chunk Queue + return !coreFuture.isDone() || !chunkQueue.isEmpty(); + } + + @Override + public synchronized FinalT next() + { + if(!hasNext()) + { + return null; + } + + if(!currentIteratorHasNext()) + { + try + { + loadNextChunkIterator(); + } + catch (InterruptedException e) + { + // Catch InterruptedException at this level so that the chunk is fully loaded and + // ready to run next(), etc again in the future. + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + + if(!hasNext()) + { + return null; + } + } + + return createNext.apply(currentIterator.next()); + } + + private void loadNextChunkIterator() throws InterruptedException + { + this.currentIterator = null; + boolean populatedChunkLoaded = false; + + while(!populatedChunkLoaded && possibleChunksRemaining()) + { + final ChunkT nextChunk = chunkQueue.poll(timeout, TimeUnit.MILLISECONDS); + if(nextChunk != null) + { + this.currentIterator = getNextIterator.apply(nextChunk); + populatedChunkLoaded = currentIteratorHasNext(); + } + } + } +} diff --git a/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java b/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java new file mode 100644 index 000000000..4b8bf47a4 --- /dev/null +++ b/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java @@ -0,0 +1,69 @@ +/* + * Copyright 2014 Brian Roach . + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.basho.riak.client.api.commands; + +import com.basho.riak.client.core.RiakFuture; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Used when the converted response is available before the core future is complete. + * @author Alex Moore + * @since 2.1 + * @param The core response type. + * @param The core query and converted query info type. + * @param The converted response type. + */ +public abstract class ImmediateCoreFutureAdapter extends CoreFutureAdapter +{ + private final T2 immediateResponse; + + protected ImmediateCoreFutureAdapter(RiakFuture coreFuture, T2 immediateResponse) + { + super(coreFuture); + this.immediateResponse = immediateResponse; + } + + @Override + public T2 get() throws InterruptedException, ExecutionException + { + return immediateResponse; + } + + @Override + public T2 get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException + { + return immediateResponse; + } + + @Override + public T2 getNow() + { + return immediateResponse; + } + + @Override + protected T2 convertResponse(T unused) { return null; } + + @Override + protected S convertQueryInfo(S coreQueryInfo) + { + return coreQueryInfo; + } +} diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java index a83bf4c07..411cf376a 100644 --- a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java +++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java @@ -15,10 +15,13 @@ */ package com.basho.riak.client.api.commands.buckets; -import com.basho.riak.client.api.RiakCommand; +import com.basho.riak.client.api.commands.ChunkedQueueIterator; import com.basho.riak.client.api.commands.CoreFutureAdapter; +import com.basho.riak.client.api.StreamableRiakCommand; +import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.ListBucketsOperation; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.util.BinaryValue; @@ -42,7 +45,7 @@ * @author Dave Rusek * @since 2.0 */ -public final class ListBuckets extends RiakCommand +public final class ListBuckets extends StreamableRiakCommand { private final int timeout; private final BinaryValue type; @@ -57,7 +60,7 @@ public final class ListBuckets extends RiakCommand executeAsync(RiakCluster cluster) { RiakFuture coreFuture = - cluster.execute(buildCoreOperation()); + cluster.execute(buildCoreOperation(false)); CoreFutureAdapter future = new CoreFutureAdapter(coreFuture) @@ -78,7 +81,22 @@ protected BinaryValue convertQueryInfo(BinaryValue coreQueryInfo) return future; } - private ListBucketsOperation buildCoreOperation() + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) + { + StreamingRiakFuture coreFuture = + cluster.execute(buildCoreOperation(true)); + + final StreamingResponse streamingResponse = new StreamingResponse(type, timeout, coreFuture); + + ImmediateCoreFutureAdapter future = + new ImmediateCoreFutureAdapter + (coreFuture, streamingResponse) {}; + + coreFuture.addListener(future); + return future; + } + + private ListBucketsOperation buildCoreOperation(boolean streamResults) { ListBucketsOperation.Builder builder = new ListBucketsOperation.Builder(); if (timeout > 0) @@ -91,6 +109,8 @@ private ListBucketsOperation buildCoreOperation() builder.withBucketType(type); } + builder.streamResults(streamResults); + return builder.build(); } @@ -126,6 +146,28 @@ public Iterator iterator() } } + public static class StreamingResponse extends Response + { + private final ChunkedQueueIterator chunkedQueueIterator; + + StreamingResponse(BinaryValue type, + int pollTimeout, + StreamingRiakFuture coreFuture) + { + super(type, null); + chunkedQueueIterator = new ChunkedQueueIterator<>(coreFuture, + pollTimeout, + (bucketName) -> new Namespace(super.type, bucketName), + (response) -> response.getBuckets().iterator()); + } + + @Override + public Iterator iterator() + { + return chunkedQueueIterator; + } + } + private static class Itr implements Iterator { private final Iterator iterator; @@ -207,4 +249,5 @@ public ListBuckets build() return new ListBuckets(this); } } + } diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java index f0a12f511..c2249c255 100644 --- a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java +++ b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java @@ -15,9 +15,12 @@ */ package com.basho.riak.client.api.commands.kv; -import com.basho.riak.client.api.RiakCommand; +import com.basho.riak.client.api.StreamableRiakCommand; +import com.basho.riak.client.api.commands.ChunkedQueueIterator; +import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.ListKeysOperation; import com.basho.riak.client.api.commands.CoreFutureAdapter; import com.basho.riak.client.core.query.Location; @@ -50,7 +53,7 @@ * @author Dave Rusek * @since 2.0 */ -public final class ListKeys extends RiakCommand +public final class ListKeys extends StreamableRiakCommand { private final Namespace namespace; private final int timeout; @@ -65,7 +68,7 @@ public final class ListKeys extends RiakCommand protected final RiakFuture executeAsync(RiakCluster cluster) { RiakFuture coreFuture = - cluster.execute(buildCoreOperation()); + cluster.execute(buildCoreOperation(false)); CoreFutureAdapter future = new CoreFutureAdapter(coreFuture) @@ -86,7 +89,23 @@ protected Namespace convertQueryInfo(Namespace coreQueryInfo) return future; } - private ListKeysOperation buildCoreOperation() + @Override + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) + { + StreamingRiakFuture coreFuture = + cluster.execute(buildCoreOperation(true)); + + final ListKeys.StreamingResponse streamingResponse = new ListKeys.StreamingResponse(namespace, timeout, coreFuture); + + ImmediateCoreFutureAdapter future = + new ImmediateCoreFutureAdapter( + coreFuture, streamingResponse) {}; + + coreFuture.addListener(future); + return future; + } + + private ListKeysOperation buildCoreOperation(boolean streamResults) { ListKeysOperation.Builder builder = new ListKeysOperation.Builder(namespace); @@ -95,6 +114,8 @@ private ListKeysOperation buildCoreOperation() builder.withTimeout(timeout); } + builder.streamResults(streamResults); + return builder.build(); } @@ -147,6 +168,29 @@ public void remove() } } + public static class StreamingResponse extends Response + { + private final ChunkedQueueIterator chunkedQueueIterator; + + StreamingResponse(Namespace namespace, + int pollTimeout, + StreamingRiakFuture coreFuture) + { + super(namespace, null); + chunkedQueueIterator = new ChunkedQueueIterator<>(coreFuture, + pollTimeout, + (key) -> new Location(super.namespace, key), + (nextChunk) -> nextChunk.getKeys().iterator()); + } + + @Override + public Iterator iterator() + { + return chunkedQueueIterator; + } + + } + /** * Used to construct a ListKeys command. */ diff --git a/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java index 358a39c6b..5297af5eb 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java @@ -25,6 +25,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.concurrent.LinkedTransferQueue; import java.util.concurrent.TransferQueue; @@ -184,7 +185,7 @@ public ListBucketsOperation build() } } - public static class Response + public static class Response implements Iterable { private final BinaryValue bucketType; private final List buckets; @@ -204,5 +205,11 @@ public List getBuckets() { return buckets; } + + @Override + public Iterator iterator() + { + return getBuckets().iterator(); + } } } diff --git a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java index 45f95f3a6..d1bd57f11 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java @@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.concurrent.LinkedTransferQueue; import java.util.concurrent.TransferQueue; @@ -169,7 +170,7 @@ public ListKeysOperation build() } } - public static class Response + public static class Response implements Iterable { private final List keys; private Response(Builder builder) @@ -182,6 +183,12 @@ public List getKeys() return keys; } + @Override + public Iterator iterator() + { + return keys.iterator(); + } + static class Builder { private List keys = new ArrayList<>(); diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java index 0735501aa..3bb5353ef 100644 --- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java +++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java @@ -3,11 +3,15 @@ import com.basho.riak.client.api.RiakClient; import com.basho.riak.client.api.commands.buckets.ListBuckets; import com.basho.riak.client.api.commands.kv.StoreValue; +import com.basho.riak.client.core.RiakFuture; import com.basho.riak.client.core.operations.itest.ITestAutoCleanupBase; +import com.basho.riak.client.core.operations.itest.ITestBase; import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.query.RiakObject; import com.basho.riak.client.core.util.BinaryValue; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import java.util.Iterator; @@ -20,40 +24,56 @@ * @author empovit * @since 2.0.3 */ -public class ITestListBuckets extends ITestAutoCleanupBase +public class ITestListBuckets extends ITestBase { - private final RiakClient client = new RiakClient(cluster); + private static final RiakClient client = new RiakClient(cluster); + private static final String bucketName = "ITestListBuckets"; + private static final Namespace defaultNamespace = new Namespace(bucketName); + private static final Namespace typedNamespace = new Namespace(bucketType.toString(), bucketName); + + @BeforeClass + public static void setup() throws ExecutionException, InterruptedException + { + storeTestObject(defaultNamespace); + if(testBucketType) + { + storeTestObject(typedNamespace); + } + } + + @AfterClass + public static void cleanup() throws ExecutionException, InterruptedException + { + resetAndEmptyBucket(defaultNamespace); + if(testBucketType) + { + resetAndEmptyBucket(typedNamespace); + } + } @Test public void testListBucketsDefaultType() throws InterruptedException, ExecutionException { - testListBuckets(Namespace.DEFAULT_BUCKET_TYPE); + testListBuckets(defaultNamespace); } @Test public void testListBucketsTestType() throws InterruptedException, ExecutionException { assumeTrue(testBucketType); - testListBuckets(bucketType.toString()); + testListBuckets(typedNamespace); } - private void testListBuckets(String bucketType) throws InterruptedException, ExecutionException + @Test + public void testListBucketsStreamingTestType() throws InterruptedException, ExecutionException { - // Empty buckets do not show up - final BinaryValue key = BinaryValue.unsafeCreate("temp_key".getBytes()); - - RiakObject value = new RiakObject().setValue(BinaryValue.create("{\"value\":\"value\"}")); - - // Since bucket type in response is populated from the command's context, - // need a way to make sure the type is indeed as expected - use bucket type for bucket name - Location location = new Location(new Namespace(bucketType, bucketType), key); - StoreValue storeCommand = new StoreValue.Builder(value).withLocation(location).build(); - - client.execute(storeCommand); - - final BinaryValue typeBinary = BinaryValue.createFromUtf8(bucketType); + assumeTrue(testBucketType); + testListBucketsStreaming(typedNamespace); + } - ListBuckets listBucketsCommand = new ListBuckets.Builder(typeBinary).build(); + private void testListBuckets(Namespace namespace) throws InterruptedException, ExecutionException + { + ListBuckets listBucketsCommand = new ListBuckets.Builder(namespace.getBucketType()).build(); final ListBuckets.Response listResponse = client.execute(listBucketsCommand); @@ -63,9 +83,51 @@ private void testListBuckets(String bucketType) throws InterruptedException, Exe while (!found && iterator.hasNext()) { - found = iterator.next().getBucketName().equals(typeBinary); + found = iterator.next().getBucketName().toString().equals(bucketName); } assertTrue(found); } + + private void testListBucketsStreaming(Namespace namespace) throws InterruptedException, ExecutionException + { + ListBuckets listBucketsCommand = new ListBuckets.Builder(namespace.getBucketType()).build(); + + final RiakFuture streamingFuture = + client.executeAsyncStreaming(listBucketsCommand, 500); + + Iterator iterator = streamingFuture.get().iterator(); + assumeTrue(iterator.hasNext()); + boolean found = false; + + while(!found && iterator.hasNext()) + { + final Namespace next = iterator.next(); + if(next == null) + { + continue; + } + found = next.getBucketName().toString().equals(bucketName); + } + + streamingFuture.await(); // Wait for command to finish, even if we've found our data + assumeTrue(streamingFuture.isDone()); + + assertTrue(found); + } + + private static void storeTestObject(Namespace namespace) throws ExecutionException, InterruptedException + { + // Empty buckets do not show up + final BinaryValue key = BinaryValue.unsafeCreate("temp_key".getBytes()); + + RiakObject value = new RiakObject().setValue(BinaryValue.create("{\"value\":\"value\"}")); + + // Since bucket type in response is populated from the command's context, + // need a way to make sure the type is indeed as expected - use bucket type for bucket name + Location location = new Location(namespace, key); + StoreValue storeCommand = new StoreValue.Builder(value).withLocation(location).build(); + + client.execute(storeCommand); + } } diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java new file mode 100644 index 000000000..d4262128c --- /dev/null +++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java @@ -0,0 +1,106 @@ +package com.basho.riak.client.api.commands.buckets.itest; + +import com.basho.riak.client.api.RiakClient; +import com.basho.riak.client.api.commands.kv.ListKeys; +import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.operations.StoreOperation; +import com.basho.riak.client.core.operations.itest.ITestBase; +import com.basho.riak.client.core.query.Location; +import com.basho.riak.client.core.query.Namespace; +import com.basho.riak.client.core.query.RiakObject; +import com.basho.riak.client.core.util.BinaryValue; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Iterator; +import java.util.Objects; +import java.util.Random; +import java.util.concurrent.ExecutionException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + + +public class ITestListKeys extends ITestBase +{ + private static final RiakClient client = new RiakClient(cluster); + private static final String bucketName = "ITestListBuckets" + new Random().nextLong(); + private static final Namespace typedNamespace = new Namespace(bucketType.toString(), bucketName); + + @BeforeClass + public static void setup() throws ExecutionException, InterruptedException + { + if(testBucketType) + { + storeTestObjects(typedNamespace); + } + } + + @AfterClass + public static void cleanup() throws ExecutionException, InterruptedException + { + if(testBucketType) + { + resetAndEmptyBucket(typedNamespace); + } + } + + @Test + public void testLargeStreamingListKeys() throws ExecutionException, InterruptedException + { + assumeTrue(testBucketType); + + ListKeys lk = new ListKeys.Builder(typedNamespace).build(); + + final RiakFuture streamFuture = + client.executeAsyncStreaming(lk, 200); + + final ListKeys.StreamingResponse streamingResponse = streamFuture.get(); + + int count = 0; + boolean foundLastKey = false; + + for (Location location : streamingResponse) + { + if(location == null) + { + continue; + } + + count++; + + if(!foundLastKey) + { + foundLastKey = location.getKeyAsString().equals("9999"); + } + } + + streamFuture.await(); + assertTrue(foundLastKey); + assertTrue(streamFuture.isDone()); + assertEquals(10000, count); + } + + private static void storeTestObjects(Namespace namespace) throws InterruptedException + { + final String value = "{\"value\":\"value\"}"; + final RiakObject rObj = new RiakObject().setValue(BinaryValue.create(value)); + + for (int i = 0; i < 10000; i++) + { + final String key = Integer.toString(i); + + final Location location = new Location(namespace, key); + final StoreOperation storeOp = + new StoreOperation.Builder(location) + .withContent(rObj) + .build(); + + final RiakFuture execute = cluster.execute(storeOp); + execute.await(); + assertTrue(execute.isSuccess()); + } + } +} From ca0aaf4b272019a431c7ab29d734d656db020173 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Fri, 14 Oct 2016 22:49:01 -0400 Subject: [PATCH 09/55] Cleanup & name things betterer --- .../api/commands/ChunkedQueueIterator.java | 96 ------------- .../api/commands/ChunkedResponseIterator.java | 130 ++++++++++++++++++ .../commands/ImmediateCoreFutureAdapter.java | 17 ++- .../api/commands/buckets/ListBuckets.java | 19 +-- .../riak/client/api/commands/kv/ListKeys.java | 19 +-- 5 files changed, 164 insertions(+), 117 deletions(-) delete mode 100644 src/main/java/com/basho/riak/client/api/commands/ChunkedQueueIterator.java create mode 100644 src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedQueueIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedQueueIterator.java deleted file mode 100644 index 7fbf851e2..000000000 --- a/src/main/java/com/basho/riak/client/api/commands/ChunkedQueueIterator.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.basho.riak.client.api.commands; - -import com.basho.riak.client.core.StreamingRiakFuture; - -import java.util.Iterator; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TransferQueue; -import java.util.function.Function; - -public class ChunkedQueueIterator, CoreT> implements Iterator -{ - private final int timeout; - private final StreamingRiakFuture coreFuture; - private final TransferQueue chunkQueue; - private final Function createNext; - private final Function> getNextIterator; - - private Iterator currentIterator = null; - - public ChunkedQueueIterator(StreamingRiakFuture coreFuture, - int pollTimeout, - Function createNext, - Function> getNextIterator) - { - this.timeout = pollTimeout; - this.coreFuture = coreFuture; - this.chunkQueue = coreFuture.getResultsQueue(); - this.createNext = createNext; - this.getNextIterator = getNextIterator; - } - - @Override - public boolean hasNext() - { - return currentIteratorHasNext() || possibleChunksRemaining(); - } - - private boolean currentIteratorHasNext() - { - return currentIterator != null && currentIterator.hasNext(); - } - - private boolean possibleChunksRemaining() - { - // Chunks may remain if : - // Core Operation Not Done OR items still in chunk Queue - return !coreFuture.isDone() || !chunkQueue.isEmpty(); - } - - @Override - public synchronized FinalT next() - { - if(!hasNext()) - { - return null; - } - - if(!currentIteratorHasNext()) - { - try - { - loadNextChunkIterator(); - } - catch (InterruptedException e) - { - // Catch InterruptedException at this level so that the chunk is fully loaded and - // ready to run next(), etc again in the future. - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - - if(!hasNext()) - { - return null; - } - } - - return createNext.apply(currentIterator.next()); - } - - private void loadNextChunkIterator() throws InterruptedException - { - this.currentIterator = null; - boolean populatedChunkLoaded = false; - - while(!populatedChunkLoaded && possibleChunksRemaining()) - { - final ChunkT nextChunk = chunkQueue.poll(timeout, TimeUnit.MILLISECONDS); - if(nextChunk != null) - { - this.currentIterator = getNextIterator.apply(nextChunk); - populatedChunkLoaded = currentIteratorHasNext(); - } - } - } -} diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java new file mode 100644 index 000000000..38d0c9c52 --- /dev/null +++ b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java @@ -0,0 +1,130 @@ +package com.basho.riak.client.api.commands; + +import com.basho.riak.client.core.StreamingRiakFuture; +import com.basho.riak.client.core.util.BinaryValue; + +import java.util.Iterator; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TransferQueue; +import java.util.function.Function; + +public class ChunkedResponseIterator, CoreT> implements Iterator +{ + private final int timeout; + private volatile BinaryValue continuation = null; + private final StreamingRiakFuture coreFuture; + private final TransferQueue chunkQueue; + private final Function createNext; + private final Function> getNextIterator; + private final Function getContinuationFn; + + private Iterator currentIterator = null; + + public ChunkedResponseIterator(StreamingRiakFuture coreFuture, + int pollTimeout, + Function createNextFn, + Function> getNextIteratorFn) + { + this(coreFuture, pollTimeout, createNextFn, getNextIteratorFn, (x) -> null); + } + + public ChunkedResponseIterator(StreamingRiakFuture coreFuture, + int pollTimeout, + Function createNextFn, + Function> getNextIteratorFn, + Function getContinuationFn) + { + this.timeout = pollTimeout; + this.coreFuture = coreFuture; + this.chunkQueue = coreFuture.getResultsQueue(); + this.createNext = createNextFn; + this.getNextIterator = getNextIteratorFn; + this.getContinuationFn = getContinuationFn; + loadNextChunkIterator(); + } + + @Override + public boolean hasNext() + { + return currentIteratorHasNext() || possibleChunksRemaining(); + } + + private boolean currentIteratorHasNext() + { + return currentIterator != null && currentIterator.hasNext(); + } + + private boolean possibleChunksRemaining() + { + // Chunks may remain if : + // Core Operation Not Done OR items still in chunk Queue + return !coreFuture.isDone() || !chunkQueue.isEmpty(); + } + + public boolean hasContinuation() + { + return continuation != null || possibleChunksRemaining(); + } + + public BinaryValue getContinuation() + { + return continuation; + } + + @Override + public synchronized FinalT next() + { + if(!hasNext()) + { + return null; + } + + if(!currentIteratorHasNext()) + { + loadNextChunkIterator(); + + if(!hasNext()) + { + return null; + } + } + + return createNext.apply(currentIterator.next()); + } + + private void loadNextChunkIterator() + { + this.currentIterator = null; + boolean populatedChunkLoaded = false; + + try + { + while (!populatedChunkLoaded && possibleChunksRemaining()) + { + final ChunkT nextChunk = chunkQueue.poll(timeout, TimeUnit.MILLISECONDS); + + if (nextChunk != null) + { + this.currentIterator = getNextIterator.apply(nextChunk); + populatedChunkLoaded = currentIteratorHasNext(); + + loadContinuation(nextChunk); + } + } + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private void loadContinuation(ChunkT nextChunk) + { + final BinaryValue fetchedContinuation = getContinuationFn.apply(nextChunk); + if(this.continuation == null && fetchedContinuation != null) + { + this.continuation = fetchedContinuation; + } + } +} diff --git a/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java b/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java index 4b8bf47a4..83d216ee7 100644 --- a/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java +++ b/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java @@ -30,7 +30,7 @@ * @param The core query and converted query info type. * @param The converted response type. */ -public abstract class ImmediateCoreFutureAdapter extends CoreFutureAdapter +public abstract class ImmediateCoreFutureAdapter extends CoreFutureAdapter { private final T2 immediateResponse; @@ -62,8 +62,19 @@ public T2 getNow() protected T2 convertResponse(T unused) { return null; } @Override - protected S convertQueryInfo(S coreQueryInfo) + protected abstract S2 convertQueryInfo(S coreQueryInfo); + + public static abstract class SameQueryInfo extends ImmediateCoreFutureAdapter { - return coreQueryInfo; + protected SameQueryInfo(RiakFuture coreFuture, T2 immediateResponse) + { + super(coreFuture, immediateResponse); + } + + @Override + protected S convertQueryInfo(S coreQueryInfo) + { + return coreQueryInfo; + } } } diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java index 411cf376a..56648c62f 100644 --- a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java +++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java @@ -15,7 +15,7 @@ */ package com.basho.riak.client.api.commands.buckets; -import com.basho.riak.client.api.commands.ChunkedQueueIterator; +import com.basho.riak.client.api.commands.ChunkedResponseIterator; import com.basho.riak.client.api.commands.CoreFutureAdapter; import com.basho.riak.client.api.StreamableRiakCommand; import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter; @@ -88,8 +88,8 @@ protected RiakFuture executeAsyncStreaming(RiakC final StreamingResponse streamingResponse = new StreamingResponse(type, timeout, coreFuture); - ImmediateCoreFutureAdapter future = - new ImmediateCoreFutureAdapter + ImmediateCoreFutureAdapter.SameQueryInfo future = + new ImmediateCoreFutureAdapter.SameQueryInfo (coreFuture, streamingResponse) {}; coreFuture.addListener(future); @@ -148,23 +148,24 @@ public Iterator iterator() public static class StreamingResponse extends Response { - private final ChunkedQueueIterator chunkedQueueIterator; + private final ChunkedResponseIterator + chunkedResponseIterator; StreamingResponse(BinaryValue type, int pollTimeout, StreamingRiakFuture coreFuture) { super(type, null); - chunkedQueueIterator = new ChunkedQueueIterator<>(coreFuture, - pollTimeout, - (bucketName) -> new Namespace(super.type, bucketName), - (response) -> response.getBuckets().iterator()); + chunkedResponseIterator = new ChunkedResponseIterator<>(coreFuture, + pollTimeout, + (bucketName) -> new Namespace(type, bucketName), + (response) -> response.getBuckets().iterator()); } @Override public Iterator iterator() { - return chunkedQueueIterator; + return chunkedResponseIterator; } } diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java index c2249c255..b0d69e4d1 100644 --- a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java +++ b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java @@ -16,7 +16,7 @@ package com.basho.riak.client.api.commands.kv; import com.basho.riak.client.api.StreamableRiakCommand; -import com.basho.riak.client.api.commands.ChunkedQueueIterator; +import com.basho.riak.client.api.commands.ChunkedResponseIterator; import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; @@ -97,8 +97,8 @@ protected RiakFuture executeAsyncStreaming(RiakClu final ListKeys.StreamingResponse streamingResponse = new ListKeys.StreamingResponse(namespace, timeout, coreFuture); - ImmediateCoreFutureAdapter future = - new ImmediateCoreFutureAdapter( + ImmediateCoreFutureAdapter.SameQueryInfo future = + new ImmediateCoreFutureAdapter.SameQueryInfo( coreFuture, streamingResponse) {}; coreFuture.addListener(future); @@ -170,23 +170,24 @@ public void remove() public static class StreamingResponse extends Response { - private final ChunkedQueueIterator chunkedQueueIterator; + private final ChunkedResponseIterator + chunkedResponseIterator; StreamingResponse(Namespace namespace, int pollTimeout, StreamingRiakFuture coreFuture) { super(namespace, null); - chunkedQueueIterator = new ChunkedQueueIterator<>(coreFuture, - pollTimeout, - (key) -> new Location(super.namespace, key), - (nextChunk) -> nextChunk.getKeys().iterator()); + chunkedResponseIterator = new ChunkedResponseIterator<>(coreFuture, + pollTimeout, + (key) -> new Location(namespace, key), + (nextChunk) -> nextChunk.getKeys().iterator()); } @Override public Iterator iterator() { - return chunkedQueueIterator; + return chunkedResponseIterator; } } From ca29f1a8de0455ea1d12cfb89aa717bd8f30a563 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Fri, 14 Oct 2016 22:49:17 -0400 Subject: [PATCH 10/55] Start into Streaming 2i special commands. --- .../commands/indexes/BigIntIndexQuery.java | 82 ++++++++++++++++++- .../api/commands/indexes/BinIndexQuery.java | 9 +- .../api/commands/indexes/IntIndexQuery.java | 9 +- .../api/commands/indexes/RawIndexQuery.java | 9 +- .../commands/indexes/SecondaryIndexQuery.java | 60 +++++++++++++- .../api/commands/kv/FullBucketRead.java | 9 +- .../SecondaryIndexQueryOperation.java | 9 +- 7 files changed, 177 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java index c01ba6ee1..e98a49b41 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java @@ -16,8 +16,11 @@ package com.basho.riak.client.api.commands.indexes; +import com.basho.riak.client.api.commands.ChunkedResponseIterator; +import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation; import com.basho.riak.client.api.commands.CoreFutureAdapter; import com.basho.riak.client.core.query.Location; @@ -25,6 +28,7 @@ import com.basho.riak.client.core.util.BinaryValue; import java.math.BigInteger; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; /** @@ -45,7 +49,11 @@ * @author Brian Roach * @since 2.0 */ -public class BigIntIndexQuery extends SecondaryIndexQuery +public class BigIntIndexQuery + extends SecondaryIndexQuery { private final IndexConverter converter; @@ -90,6 +98,21 @@ protected RiakFuture executeAsync(RiakCluster cluste return future; } + @Override + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, + int timeout) + { + StreamingRiakFuture coreFuture = + executeCoreAsyncStreaming(cluster); + + StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout); + + BigIntStreamingQueryFuture future = new BigIntStreamingQueryFuture(coreFuture, response); + coreFuture.addListener(future); + + return future; + } + protected final class BigIntQueryFuture extends CoreFutureAdapter + { + + protected BigIntStreamingQueryFuture(StreamingRiakFuture coreFuture, + StreamingResponse immediateResponse) + { + super(coreFuture, immediateResponse); + } + + @Override + protected BigIntIndexQuery convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo) + { + return BigIntIndexQuery.this; + } + } + protected static abstract class Init> extends SecondaryIndexQuery.Init { public Init(Namespace namespace, String indexName, S start, S end) @@ -228,7 +272,7 @@ public List getEntries() return convertedList; } - public class Entry extends SecondaryIndexQuery.Response.Entry + public static class Entry extends SecondaryIndexQuery.Response.Entry { protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverter converter) { @@ -236,4 +280,38 @@ protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverte } } } + + public static class StreamingResponse extends SecondaryIndexQuery.StreamingResponse + { + StreamingResponse(Namespace namespace, IndexConverter converter, + StreamingRiakFuture coreFuture, + int pollTimeout) + { + super(namespace, converter, createResponseIterator(namespace, converter, coreFuture, pollTimeout)); + } + + private static ChunkedResponseIterator createResponseIterator( + Namespace namespace, + IndexConverter converter, + StreamingRiakFuture coreFuture, + int pollTimeout) + { + return new ChunkedResponseIterator<>( + coreFuture, + pollTimeout, + (e) -> createExternalResponseEntry(namespace, e, converter), + SecondaryIndexQueryOperation.Response::iterator, + SecondaryIndexQueryOperation.Response::getContinuation); + } + + private static Response.Entry createExternalResponseEntry(Namespace namespace, + SecondaryIndexQueryOperation.Response.Entry baseEntry, + IndexConverter converter) + { + return new Response.Entry(SecondaryIndexQuery.Response.getLocationFromCoreEntry(namespace, baseEntry), + baseEntry.getIndexKey(), + converter); + } + } } diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java index c039bf003..6561aa953 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java @@ -47,7 +47,7 @@ * @author Brian Roach * @since 2.0 */ -public class BinIndexQuery extends SecondaryIndexQuery +public class BinIndexQuery extends SecondaryIndexQuery { private final Charset charset; private final IndexConverter converter; @@ -76,6 +76,13 @@ protected RiakFuture executeAsync(RiakCluster cluster) return future; } + @Override + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) + { + // TODO + return null; + } + @Override public boolean equals(Object o) { diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java index b28e93fe2..1a97ce397 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java @@ -42,7 +42,7 @@ * @author Brian Roach * @since 2.0 */ -public class IntIndexQuery extends SecondaryIndexQuery +public class IntIndexQuery extends SecondaryIndexQuery { private final IndexConverter converter; @@ -88,6 +88,13 @@ protected RiakFuture executeAsync(RiakCluster cluster) return future; } + @Override + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) + { + // TODO + return null; + } + protected final class IntQueryFuture extends CoreFutureAdapter * @since 2.0 */ -public class RawIndexQuery extends SecondaryIndexQuery +public class RawIndexQuery extends SecondaryIndexQuery { private final IndexConverter converter; @@ -77,6 +77,13 @@ protected RiakFuture executeAsync(RiakCluster cluster) return future; } + @Override + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) + { + // TODO + return null; + } + protected final class RawQueryFuture extends CoreFutureAdapter * @since 2.0 */ -public abstract class SecondaryIndexQuery extends RiakCommand +public abstract class SecondaryIndexQuery extends StreamableRiakCommand { protected final Namespace namespace; protected final String indexName; @@ -232,6 +236,15 @@ SecondaryIndexQueryOperation.Query> executeCoreAsync(RiakCluster cluster) return cluster.execute(builder.build()); } + protected StreamingRiakFuture executeCoreAsyncStreaming(RiakCluster cluster) + { + SecondaryIndexQueryOperation.Builder builder = + new SecondaryIndexQueryOperation.Builder(this.createCoreQuery()).streamResults(true); + + return cluster.execute(builder.build()); + } + @Override public boolean equals(Object o) { @@ -244,7 +257,7 @@ public boolean equals(Object o) return false; } - SecondaryIndexQuery that = (SecondaryIndexQuery) o; + SecondaryIndexQuery that = (SecondaryIndexQuery) o; if (returnTerms != that.returnTerms) { @@ -592,7 +605,12 @@ public boolean hasEntries() protected final Location getLocationFromCoreEntry(SecondaryIndexQueryOperation.Response.Entry e) { - Location loc = new Location(queryLocation, e.getObjectKey()); + return Response.getLocationFromCoreEntry(this.queryLocation, e); + } + + protected static Location getLocationFromCoreEntry(Namespace queryLocation, SecondaryIndexQueryOperation.Response.Entry e) + { + final Location loc = new Location(queryLocation, e.getObjectKey()); return loc; } @@ -634,4 +652,40 @@ public T getIndexKey() } } } + + public abstract static class StreamingResponse implements Iterable + { + final protected IndexConverter converter; + final protected Namespace queryLocation; + + private final ChunkedResponseIterator chunkedResponseIterator; + + StreamingResponse(Namespace queryLocation, IndexConverter converter, + ChunkedResponseIterator chunkedResponseIterator) + { + this.converter = converter; + this.queryLocation = queryLocation; + this.chunkedResponseIterator = chunkedResponseIterator; + } + + public Iterator iterator() + { + return chunkedResponseIterator; + } + + public boolean hasContinuation() + { + return chunkedResponseIterator.hasContinuation(); + } + + public BinaryValue getContinuation() + { + return chunkedResponseIterator.getContinuation(); + } + + public boolean hasEntries() + { + return chunkedResponseIterator.hasNext(); + } + } } diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java b/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java index b349840c4..15c2d3e49 100644 --- a/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java +++ b/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java @@ -49,7 +49,7 @@ * @author Sergey Galkin * @see CoveragePlan */ -public class FullBucketRead extends SecondaryIndexQuery +public class FullBucketRead extends SecondaryIndexQuery { private final IndexConverter converter; @@ -83,6 +83,13 @@ protected RiakFuture executeAsync(RiakCluster cluster) return future; } + @Override + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) + { + // TODO + return null; + } + protected final class RawQueryFuture extends CoreFutureAdapter { private final BinaryValue continuation; private final List entryList; @@ -757,6 +758,12 @@ public List getEntryList() return entryList; } + @Override + public Iterator iterator() + { + return getEntryList().iterator(); + } + public static class Entry { private final BinaryValue indexKey; From e6de1e081745577ffa33ca982ead305a003b53f4 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Mon, 17 Oct 2016 16:50:55 -0400 Subject: [PATCH 11/55] Stub out rest of streaming 2i commands --- .../commands/ImmediateCoreFutureAdapter.java | 3 +- .../commands/indexes/BigIntIndexQuery.java | 2 +- .../api/commands/indexes/BinIndexQuery.java | 55 ++++++++++- .../api/commands/indexes/IntIndexQuery.java | 56 ++++++++++- .../api/commands/indexes/RawIndexQuery.java | 55 ++++++++++- .../commands/indexes/SecondaryIndexQuery.java | 8 +- .../indexes/StreamingQueryFuture.java | 26 +++++ .../api/commands/kv/FullBucketRead.java | 99 ++++++++++++++----- 8 files changed, 259 insertions(+), 45 deletions(-) create mode 100644 src/main/java/com/basho/riak/client/api/commands/indexes/StreamingQueryFuture.java diff --git a/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java b/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java index 83d216ee7..5b12ec032 100644 --- a/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java +++ b/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java @@ -27,8 +27,9 @@ * @author Alex Moore * @since 2.1 * @param The core response type. - * @param The core query and converted query info type. + * @param The core query info type. * @param The converted response type. + * @param The converted query info type. */ public abstract class ImmediateCoreFutureAdapter extends CoreFutureAdapter { diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java index e98a49b41..0b20812ab 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java @@ -99,7 +99,7 @@ protected RiakFuture executeAsync(RiakCluster cluste } @Override - protected RiakFuture executeAsyncStreaming(RiakCluster cluster, + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) { StreamingRiakFuture coreFuture = diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java index 6561aa953..b12672b79 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java @@ -16,9 +16,11 @@ package com.basho.riak.client.api.commands.indexes; +import com.basho.riak.client.api.commands.ChunkedResponseIterator; import com.basho.riak.client.api.commands.CoreFutureAdapter; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation; import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; @@ -47,7 +49,7 @@ * @author Brian Roach * @since 2.0 */ -public class BinIndexQuery extends SecondaryIndexQuery +public class BinIndexQuery extends SecondaryIndexQuery { private final Charset charset; private final IndexConverter converter; @@ -77,10 +79,19 @@ protected RiakFuture executeAsync(RiakCluster cluster) } @Override - protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) { - // TODO - return null; + StreamingRiakFuture coreFuture = + executeCoreAsyncStreaming(cluster); + + StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout); + + StreamingQueryFuture future = + new StreamingQueryFuture<>(coreFuture, response, this); + + coreFuture.addListener(future); + + return future; } @Override @@ -251,7 +262,7 @@ public List getEntries() return convertedList; } - public class Entry extends SecondaryIndexQuery.Response.Entry + public static class Entry extends SecondaryIndexQuery.Response.Entry { protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverter converter) { @@ -260,6 +271,40 @@ protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverte } } + public static class StreamingResponse extends SecondaryIndexQuery.StreamingResponse + { + StreamingResponse(Namespace namespace, IndexConverter converter, + StreamingRiakFuture coreFuture, + int pollTimeout) + { + super(namespace, converter, createResponseIterator(namespace, converter, coreFuture, pollTimeout)); + } + + private static ChunkedResponseIterator createResponseIterator( + Namespace namespace, + IndexConverter converter, + StreamingRiakFuture coreFuture, + int pollTimeout) + { + return new ChunkedResponseIterator<>( + coreFuture, + pollTimeout, + (e) -> createExternalResponseEntry(namespace, e, converter), + SecondaryIndexQueryOperation.Response::iterator, + SecondaryIndexQueryOperation.Response::getContinuation); + } + + private static Response.Entry createExternalResponseEntry(Namespace namespace, + SecondaryIndexQueryOperation.Response.Entry baseEntry, + IndexConverter converter) + { + return new Response.Entry(SecondaryIndexQuery.Response.getLocationFromCoreEntry(namespace, baseEntry), + baseEntry.getIndexKey(), + converter); + } + } + protected final class BinQueryFuture extends CoreFutureAdapter diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java index 1a97ce397..5cbd47fb6 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java @@ -16,8 +16,10 @@ package com.basho.riak.client.api.commands.indexes; +import com.basho.riak.client.api.commands.ChunkedResponseIterator; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation; import com.basho.riak.client.api.commands.CoreFutureAdapter; import com.basho.riak.client.core.query.Location; @@ -42,7 +44,8 @@ * @author Brian Roach * @since 2.0 */ -public class IntIndexQuery extends SecondaryIndexQuery +public class IntIndexQuery extends SecondaryIndexQuery { private final IndexConverter converter; @@ -89,10 +92,19 @@ protected RiakFuture executeAsync(RiakCluster cluster) } @Override - protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) { - // TODO - return null; + StreamingRiakFuture coreFuture = + executeCoreAsyncStreaming(cluster); + + StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout); + + StreamingQueryFuture future = + new StreamingQueryFuture<>(coreFuture, response, this); + + coreFuture.addListener(future); + + return future; } protected final class IntQueryFuture @@ -230,7 +242,7 @@ public List getEntries() return convertedList; } - public class Entry extends SecondaryIndexQuery.Response.Entry + public static class Entry extends SecondaryIndexQuery.Response.Entry { protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverter converter) { @@ -238,4 +250,38 @@ protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverte } } } + + public static class StreamingResponse extends SecondaryIndexQuery.StreamingResponse + { + StreamingResponse(Namespace namespace, IndexConverter converter, + StreamingRiakFuture coreFuture, + int pollTimeout) + { + super(namespace, converter, createResponseIterator(namespace, converter, coreFuture, pollTimeout)); + } + + private static ChunkedResponseIterator createResponseIterator( + Namespace namespace, + IndexConverter converter, + StreamingRiakFuture coreFuture, + int pollTimeout) + { + return new ChunkedResponseIterator<>( + coreFuture, + pollTimeout, + (e) -> createExternalResponseEntry(namespace, e, converter), + SecondaryIndexQueryOperation.Response::iterator, + SecondaryIndexQueryOperation.Response::getContinuation); + } + + private static Response.Entry createExternalResponseEntry(Namespace namespace, + SecondaryIndexQueryOperation.Response.Entry baseEntry, + IndexConverter converter) + { + return new Response.Entry(SecondaryIndexQuery.Response.getLocationFromCoreEntry(namespace, baseEntry), + baseEntry.getIndexKey(), + converter); + } + } } diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java index b8a93d857..a66576ef0 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java @@ -16,8 +16,10 @@ package com.basho.riak.client.api.commands.indexes; +import com.basho.riak.client.api.commands.ChunkedResponseIterator; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation; import com.basho.riak.client.api.commands.CoreFutureAdapter; import com.basho.riak.client.core.query.Location; @@ -43,7 +45,7 @@ * @author Brian Roach * @since 2.0 */ -public class RawIndexQuery extends SecondaryIndexQuery +public class RawIndexQuery extends SecondaryIndexQuery { private final IndexConverter converter; @@ -78,10 +80,19 @@ protected RiakFuture executeAsync(RiakCluster cluster) } @Override - protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) { - // TODO - return null; + StreamingRiakFuture coreFuture = + executeCoreAsyncStreaming(cluster); + + StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout); + + StreamingQueryFuture future = + new StreamingQueryFuture<>(coreFuture, response, this); + + coreFuture.addListener(future); + + return future; } protected final class RawQueryFuture @@ -185,7 +196,7 @@ public List getEntries() return convertedList; } - public class Entry extends SecondaryIndexQuery.Response.Entry + public static class Entry extends SecondaryIndexQuery.Response.Entry { protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverter converter) { @@ -193,4 +204,38 @@ protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverte } } } + + public static class StreamingResponse extends SecondaryIndexQuery.StreamingResponse + { + StreamingResponse(Namespace namespace, IndexConverter converter, + StreamingRiakFuture coreFuture, + int pollTimeout) + { + super(namespace, converter, createResponseIterator(namespace, converter, coreFuture, pollTimeout)); + } + + private static ChunkedResponseIterator createResponseIterator( + Namespace namespace, + IndexConverter converter, + StreamingRiakFuture coreFuture, + int pollTimeout) + { + return new ChunkedResponseIterator<>( + coreFuture, + pollTimeout, + (e) -> createExternalResponseEntry(namespace, e, converter), + SecondaryIndexQueryOperation.Response::iterator, + SecondaryIndexQueryOperation.Response::getContinuation); + } + + private static Response.Entry createExternalResponseEntry(Namespace namespace, + SecondaryIndexQueryOperation.Response.Entry baseEntry, + IndexConverter converter) + { + return new Response.Entry(SecondaryIndexQuery.Response.getLocationFromCoreEntry(namespace, baseEntry), + baseEntry.getIndexKey(), + converter); + } + } } diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java index 167deb6e3..01463c1a0 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java @@ -608,7 +608,8 @@ protected final Location getLocationFromCoreEntry(SecondaryIndexQueryOperation.R return Response.getLocationFromCoreEntry(this.queryLocation, e); } - protected static Location getLocationFromCoreEntry(Namespace queryLocation, SecondaryIndexQueryOperation.Response.Entry e) + public static Location getLocationFromCoreEntry(Namespace queryLocation, + SecondaryIndexQueryOperation.Response.Entry e) { final Location loc = new Location(queryLocation, e.getObjectKey()); return loc; @@ -660,8 +661,9 @@ public abstract static class StreamingResponse implements Iterable private final ChunkedResponseIterator chunkedResponseIterator; - StreamingResponse(Namespace queryLocation, IndexConverter converter, - ChunkedResponseIterator chunkedResponseIterator) + protected StreamingResponse(Namespace queryLocation, + IndexConverter converter, + ChunkedResponseIterator chunkedResponseIterator) { this.converter = converter; this.queryLocation = queryLocation; diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/StreamingQueryFuture.java b/src/main/java/com/basho/riak/client/api/commands/indexes/StreamingQueryFuture.java new file mode 100644 index 000000000..96e8750c7 --- /dev/null +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/StreamingQueryFuture.java @@ -0,0 +1,26 @@ +package com.basho.riak.client.api.commands.indexes; + +import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter; +import com.basho.riak.client.core.StreamingRiakFuture; +import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation; + +public class StreamingQueryFuture + extends ImmediateCoreFutureAdapter +{ + private S indexQuery; + + public StreamingQueryFuture(StreamingRiakFuture coreFuture, + T immediateResponse, + S indexQuery) + { + super(coreFuture, immediateResponse); + this.indexQuery = indexQuery; + } + + @Override + protected S convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo) + { + return indexQuery; + } +} diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java b/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java index 15c2d3e49..fc6173b55 100644 --- a/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java +++ b/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java @@ -15,10 +15,13 @@ */ package com.basho.riak.client.api.commands.kv; +import com.basho.riak.client.api.commands.ChunkedResponseIterator; import com.basho.riak.client.api.commands.CoreFutureAdapter; import com.basho.riak.client.api.commands.indexes.SecondaryIndexQuery; +import com.basho.riak.client.api.commands.indexes.StreamingQueryFuture; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.FetchOperation; import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation; import com.basho.riak.client.core.query.Location; @@ -49,7 +52,7 @@ * @author Sergey Galkin * @see CoveragePlan */ -public class FullBucketRead extends SecondaryIndexQuery +public class FullBucketRead extends SecondaryIndexQuery { private final IndexConverter converter; @@ -84,10 +87,19 @@ protected RiakFuture executeAsync(RiakCluster cluster) } @Override - protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) { - // TODO - return null; + StreamingRiakFuture coreFuture = + executeCoreAsyncStreaming(cluster); + + StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout); + + StreamingQueryFuture future = + new StreamingQueryFuture<>(coreFuture, response, this); + + coreFuture.addListener(future); + + return future; } protected final class RawQueryFuture @@ -264,32 +276,37 @@ public List getEntries() convertedList = new ArrayList<>(coreResponse.getEntryList().size()); for (SecondaryIndexQueryOperation.Response.Entry e : coreResponse.getEntryList()) { - final Location loc = getLocationFromCoreEntry(e); - - final FetchValue.Response fr; - if (e.hasBody()) - { - FetchOperation.Response resp = e.getBody(); - - // The following code has been copied from the FetchValue.executeAsync - CoreFutureAdapter - fr = new FetchValue.Response.Builder() - .withNotFound(resp.isNotFound()) - .withUnchanged(resp.isUnchanged()) - .withValues(resp.getObjectList()) - .withLocation(loc) // for ORM - .build(); - } - else - { - fr = null; - } - - Entry ce = new Entry(loc, fr); + Entry ce = createKeyValueEntry(queryLocation, e); convertedList.add(ce); } return convertedList; } + static Entry createKeyValueEntry(Namespace namespace, SecondaryIndexQueryOperation.Response.Entry e) + { + final Location loc = getLocationFromCoreEntry(namespace, e); + + final FetchValue.Response fr; + if (e.hasBody()) + { + FetchOperation.Response resp = e.getBody(); + + // The following code has been copied from the FetchValue.executeAsync - CoreFutureAdapter + fr = new FetchValue.Response.Builder() + .withNotFound(resp.isNotFound()) + .withUnchanged(resp.isUnchanged()) + .withValues(resp.getObjectList()) + .withLocation(loc) // for ORM + .build(); + } + else + { + fr = null; + } + + return new Entry(loc, fr); + } + public static class Entry { private final FetchValue.Response fetchedValue; @@ -331,4 +348,36 @@ public String toString() } } } + + public static class StreamingResponse extends SecondaryIndexQuery.StreamingResponse + { + StreamingResponse(Namespace namespace, IndexConverter converter, + StreamingRiakFuture coreFuture, + int pollTimeout) + { + super(namespace, converter, createResponseIterator(namespace, converter, coreFuture, pollTimeout)); + } + + private static ChunkedResponseIterator createResponseIterator( + Namespace namespace, + IndexConverter converter, + StreamingRiakFuture coreFuture, + int pollTimeout) + { + return new ChunkedResponseIterator<>( + coreFuture, + pollTimeout, + (e) -> createExternalResponseEntry(namespace, e, converter), + SecondaryIndexQueryOperation.Response::iterator, + SecondaryIndexQueryOperation.Response::getContinuation); + } + + private static Response.Entry createExternalResponseEntry(Namespace namespace, + SecondaryIndexQueryOperation.Response.Entry baseEntry, + IndexConverter unused) + { + return Response.createKeyValueEntry(namespace, baseEntry); + } + } } From 35e9c4f38bdec4393a86d46c7d659815ad1b1451 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 18 Oct 2016 16:47:21 -0400 Subject: [PATCH 12/55] Add Javadocs, remove need to check for nulls from streaming results --- .../api/commands/ChunkedResponseIterator.java | 24 +++++++------------ .../commands/ImmediateCoreFutureAdapter.java | 8 ++++--- .../commands/indexes/BigIntIndexQuery.java | 1 + .../api/commands/indexes/BinIndexQuery.java | 5 ++-- .../api/commands/indexes/IntIndexQuery.java | 5 ++-- .../api/commands/indexes/RawIndexQuery.java | 5 ++-- .../commands/indexes/SecondaryIndexQuery.java | 1 + ...uture.java => Streaming2iQueryFuture.java} | 18 ++++++++++---- .../api/commands/kv/FullBucketRead.java | 6 ++--- .../buckets/itest/ITestListBuckets.java | 5 +--- .../commands/buckets/itest/ITestListKeys.java | 12 ++++------ .../indexes/SecondaryIndexQueryTest.java | 2 ++ 12 files changed, 47 insertions(+), 45 deletions(-) rename src/main/java/com/basho/riak/client/api/commands/indexes/{StreamingQueryFuture.java => Streaming2iQueryFuture.java} (52%) diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java index 38d0c9c52..7ec733a3f 100644 --- a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java +++ b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java @@ -46,7 +46,14 @@ public ChunkedResponseIterator(StreamingRiakFuture coreFuture, @Override public boolean hasNext() { - return currentIteratorHasNext() || possibleChunksRemaining(); + if(currentIteratorHasNext()) + { + return true; + } + + loadNextChunkIterator(); + + return currentIteratorHasNext(); } private boolean currentIteratorHasNext() @@ -74,21 +81,6 @@ public BinaryValue getContinuation() @Override public synchronized FinalT next() { - if(!hasNext()) - { - return null; - } - - if(!currentIteratorHasNext()) - { - loadNextChunkIterator(); - - if(!hasNext()) - { - return null; - } - } - return createNext.apply(currentIterator.next()); } diff --git a/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java b/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java index 5b12ec032..1922ef1b3 100644 --- a/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java +++ b/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java @@ -24,12 +24,14 @@ /** * Used when the converted response is available before the core future is complete. - * @author Alex Moore - * @since 2.1 + * * @param The core response type. * @param The core query info type. * @param The converted response type. - * @param The converted query info type. + * @param The converted query info type. + * + * @author Alex Moore + * @since 2.1 */ public abstract class ImmediateCoreFutureAdapter extends CoreFutureAdapter { diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java index 0b20812ab..d5d146946 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java @@ -47,6 +47,7 @@ * BigIntIndexQuery q = new BigIntIndexQuery.Builder(ns, "my_index", key).build(); * BigIntIndexQuery.Response resp = client.execute(q);} * @author Brian Roach + * @author Alex Moore * @since 2.0 */ public class BigIntIndexQuery diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java index b12672b79..e3e4d88cd 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java @@ -47,6 +47,7 @@ * BinIndexQuery.Response resp = client.execute(q);} * * @author Brian Roach + * @author Alex Moore * @since 2.0 */ public class BinIndexQuery extends SecondaryIndexQuery @@ -86,8 +87,8 @@ protected RiakFuture executeAsyncStreaming(Ria StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout); - StreamingQueryFuture future = - new StreamingQueryFuture<>(coreFuture, response, this); + Streaming2iQueryFuture future = + new Streaming2iQueryFuture<>(coreFuture, response, this); coreFuture.addListener(future); diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java index 5cbd47fb6..28de516ff 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java @@ -42,6 +42,7 @@ * IntIndexQuery q = new IntIndexQuery.Builder(ns, "my_index", key).build(); * IntIndexQuery.Response resp = client.execute(q);} * @author Brian Roach + * @author Alex Moore * @since 2.0 */ public class IntIndexQuery extends SecondaryIndexQuery executeAsyncStreaming(Ria StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout); - StreamingQueryFuture future = - new StreamingQueryFuture<>(coreFuture, response, this); + Streaming2iQueryFuture future = + new Streaming2iQueryFuture<>(coreFuture, response, this); coreFuture.addListener(future); diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java index a66576ef0..4e88de13f 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java @@ -43,6 +43,7 @@ * RawIndexQuery q = new RawIndexQuery.Builder(ns, "my_index", Type._BIN, key).build(); * RawIndexquery.Response resp = client.execute(q);} * @author Brian Roach + * @author Alex Moore * @since 2.0 */ public class RawIndexQuery extends SecondaryIndexQuery @@ -87,8 +88,8 @@ protected RiakFuture executeAsyncStreaming(Ria StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout); - StreamingQueryFuture future = - new StreamingQueryFuture<>(coreFuture, response, this); + Streaming2iQueryFuture future = + new Streaming2iQueryFuture<>(coreFuture, response, this); coreFuture.addListener(future); diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java index 01463c1a0..abb1b8ffa 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java @@ -38,6 +38,7 @@ * * @param the type being used for the query. * @author Brian Roach + * @author Alex Moore * @since 2.0 */ public abstract class SecondaryIndexQuery extends StreamableRiakCommand diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/StreamingQueryFuture.java b/src/main/java/com/basho/riak/client/api/commands/indexes/Streaming2iQueryFuture.java similarity index 52% rename from src/main/java/com/basho/riak/client/api/commands/indexes/StreamingQueryFuture.java rename to src/main/java/com/basho/riak/client/api/commands/indexes/Streaming2iQueryFuture.java index 96e8750c7..909ddf217 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/StreamingQueryFuture.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/Streaming2iQueryFuture.java @@ -4,15 +4,23 @@ import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation; -public class StreamingQueryFuture +/** + * Streamlined ImmediateCoreFutureAdapter for converting streaming 2i operation results to command results. + * @param The converted response type. + * @param The converted query info type. + * + * @author Alex Moore + * @since 2.1.0 + */ +public class Streaming2iQueryFuture extends ImmediateCoreFutureAdapter { private S indexQuery; - public StreamingQueryFuture(StreamingRiakFuture coreFuture, - T immediateResponse, - S indexQuery) + public Streaming2iQueryFuture(StreamingRiakFuture coreFuture, + T immediateResponse, + S indexQuery) { super(coreFuture, immediateResponse); this.indexQuery = indexQuery; diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java b/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java index fc6173b55..f9bbf53c0 100644 --- a/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java +++ b/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java @@ -18,7 +18,7 @@ import com.basho.riak.client.api.commands.ChunkedResponseIterator; import com.basho.riak.client.api.commands.CoreFutureAdapter; import com.basho.riak.client.api.commands.indexes.SecondaryIndexQuery; -import com.basho.riak.client.api.commands.indexes.StreamingQueryFuture; +import com.basho.riak.client.api.commands.indexes.Streaming2iQueryFuture; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; import com.basho.riak.client.core.StreamingRiakFuture; @@ -94,8 +94,8 @@ protected RiakFuture executeAsyncStreaming(Ri StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout); - StreamingQueryFuture future = - new StreamingQueryFuture<>(coreFuture, response, this); + Streaming2iQueryFuture future = + new Streaming2iQueryFuture<>(coreFuture, response, this); coreFuture.addListener(future); diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java index 3bb5353ef..5af4e0eb8 100644 --- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java +++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java @@ -22,6 +22,7 @@ /** * @author empovit + * @author Alex Moore * @since 2.0.3 */ public class ITestListBuckets extends ITestBase @@ -103,10 +104,6 @@ private void testListBucketsStreaming(Namespace namespace) throws InterruptedExc while(!found && iterator.hasNext()) { final Namespace next = iterator.next(); - if(next == null) - { - continue; - } found = next.getBucketName().toString().equals(bucketName); } diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java index d4262128c..d74c23905 100644 --- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java +++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java @@ -13,8 +13,6 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.util.Iterator; -import java.util.Objects; import java.util.Random; import java.util.concurrent.ExecutionException; @@ -22,7 +20,10 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; - +/** + * + * @author Alex Moore + */ public class ITestListKeys extends ITestBase { private static final RiakClient client = new RiakClient(cluster); @@ -64,11 +65,6 @@ public void testLargeStreamingListKeys() throws ExecutionException, InterruptedE for (Location location : streamingResponse) { - if(location == null) - { - continue; - } - count++; if(!foundLastKey) diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQueryTest.java b/src/test/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQueryTest.java index 29eb320ab..8fc72bec6 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQueryTest.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQueryTest.java @@ -26,6 +26,7 @@ /** * * @author Brian Roach + * @author Alex Moore */ public class SecondaryIndexQueryTest { @@ -128,6 +129,7 @@ public void binIndexQueryBuildsCorrectly() assertEquals(true, query.isReturnKeyAndIndex()); } + @Test public void rawIndexQueryBuildsCorrectly() { Namespace ns = new Namespace("bucket_type", "bucket_name"); From 82fb33927464964400321a2f3d527cb798fb5618 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 18 Oct 2016 23:20:20 -0400 Subject: [PATCH 13/55] Streaming 2i Command Integration Tests --- .../indexes/itest/ITestBigIntIndexQuery.java | 174 +++++++++-------- .../indexes/itest/ITestBinIndexQuery.java | 176 +++++++++-------- .../indexes/itest/ITestIndexBase.java | 18 ++ .../indexes/itest/ITestIntIndexQuery.java | 180 ++++++++++-------- .../indexes/itest/ITestRawIndexQuery.java | 53 ++++-- 5 files changed, 336 insertions(+), 265 deletions(-) create mode 100644 src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIndexBase.java diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java index 4c74bf65b..b0f4db43f 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java @@ -20,27 +20,41 @@ import com.basho.riak.client.api.annotations.RiakBucketName; import com.basho.riak.client.api.annotations.RiakIndex; import com.basho.riak.client.api.annotations.RiakKey; -import com.basho.riak.client.api.annotations.RiakVClock; -import com.basho.riak.client.api.cap.VClock; -import com.basho.riak.client.core.RiakFuture; -import com.basho.riak.client.core.operations.itest.ITestAutoCleanupBase; import com.basho.riak.client.api.commands.indexes.BigIntIndexQuery; import com.basho.riak.client.api.commands.kv.StoreValue; +import com.basho.riak.client.core.RiakFuture; import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; + import java.math.BigInteger; import java.util.concurrent.ExecutionException; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import org.junit.Assume; -import org.junit.Test; /** - * * @author Brian Roach + * @author Alex Moore */ -public class ITestBigIntIndexQuery extends ITestAutoCleanupBase +public class ITestBigIntIndexQuery extends ITestIndexBase { + private static final String OBJECT_KEY_BASE = "index_test_object_key"; + private static final String INDEX_NAME = "test_index"; + private static final BigInteger INDEX_ENTRY = new BigInteger("91234567890123456789012345678901234567890"); + private static final BigInteger INDEX_ENTRY2 = new BigInteger("91234567890123456789012345678901234567898"); + private static final RiakClient client = new RiakClient(cluster); + + @BeforeClass + public static void setupData() throws InterruptedException + { + createIndexedPojo(0, INDEX_ENTRY); + createIndexedPojo(1, INDEX_ENTRY); + createIndexedPojo(2, INDEX_ENTRY2); + } + @Test public void testMatchQuery() throws InterruptedException, ExecutionException { @@ -48,64 +62,92 @@ public void testMatchQuery() throws InterruptedException, ExecutionException RiakClient client = new RiakClient(cluster); - IndexedPojo ip = new IndexedPojo(); - ip.key = "index_test_object_key"; - ip.bucketName = bucketName.toString(); - ip.indexKey = new BigInteger("91234567890123456789012345678901234567890"); - ip.value = "My Object Value!"; + Namespace ns = new Namespace(Namespace.DEFAULT_BUCKET_TYPE, bucketName.toString()); - StoreValue sv = new StoreValue.Builder(ip).build(); - RiakFuture svFuture = client.executeAsync(sv); + BigIntIndexQuery indexQuery = new BigIntIndexQuery.Builder(ns, INDEX_NAME, INDEX_ENTRY).withKeyAndIndex(true) + .build(); + BigIntIndexQuery.Response indexResponse = client.execute(indexQuery); - svFuture.await(); - assertTrue(svFuture.isSuccess()); + assertTrue(indexResponse.hasEntries()); + assertEquals(2, indexResponse.getEntries().size()); - IndexedPojo ip2 = new IndexedPojo(); - ip2.key = "index_test_object_key2"; - ip2.bucketName = bucketName.toString(); - ip2.indexKey = new BigInteger("91234567890123456789012345678901234567890"); - ip2.value = "My Object Value!"; + assertFirstObjectFound(indexResponse); + } - sv = new StoreValue.Builder(ip2).build(); - svFuture = client.executeAsync(sv); + @Test + public void testRangeQuery() throws InterruptedException, ExecutionException + { + Assume.assumeTrue(test2i); - svFuture.await(); - assertTrue(svFuture.isSuccess()); + BigIntIndexQuery indexQuery = new BigIntIndexQuery.Builder(namespace, + INDEX_NAME, + INDEX_ENTRY, + INDEX_ENTRY2).withKeyAndIndex(true).build(); - Namespace ns = new Namespace(Namespace.DEFAULT_BUCKET_TYPE, bucketName.toString()); + BigIntIndexQuery.Response indexResponse = client.execute(indexQuery); + assertTrue(indexResponse.hasEntries()); + assertEquals(3, indexResponse.getEntries().size()); + + assertFirstObjectFound(indexResponse); + } + + @Test + public void testStreamingRangeQuery() throws ExecutionException, InterruptedException + { + Assume.assumeTrue(test2i); + + BigIntIndexQuery indexQuery = new BigIntIndexQuery.Builder(namespace, + INDEX_NAME, + INDEX_ENTRY, + INDEX_ENTRY2).withKeyAndIndex(true).build(); - BigIntIndexQuery iiq = - new BigIntIndexQuery.Builder(ns, "test_index", new BigInteger("91234567890123456789012345678901234567890")) - .withKeyAndIndex(true).build(); - BigIntIndexQuery.Response iResp = client.execute(iiq); + final RiakFuture streamingFuture = + client.executeAsyncStreaming(indexQuery, 200); - assertTrue(iResp.hasEntries()); - assertEquals(2, iResp.getEntries().size()); + final BigIntIndexQuery.StreamingResponse streamingResponse = streamingFuture.get(); + assertTrue(streamingResponse.hasEntries()); + + final String expectedObjectKey = objectKey(1); boolean found = false; - for (BigIntIndexQuery.Response.Entry e : iResp.getEntries()) + int size = 0; + + for (BigIntIndexQuery.Response.Entry e : streamingResponse) { - if (e.getRiakObjectLocation().getKey().toString().equals("index_test_object_key")) + size++; + if (e.getRiakObjectLocation().getKey().toString().equals(expectedObjectKey)) { found = true; - assertEquals(ip.indexKey, e.getIndexKey()); + assertEquals(INDEX_ENTRY, e.getIndexKey()); } } assertTrue(found); + assertEquals(3, size); } - @Test - public void testRangeQuery() throws InterruptedException, ExecutionException + private boolean assertFirstObjectFound(BigIntIndexQuery.Response response) { - Assume.assumeTrue(test2i); + final String expectedObjectKey = objectKey(1); + boolean found = false; - RiakClient client = new RiakClient(cluster); + for (BigIntIndexQuery.Response.Entry e : response.getEntries()) + { + if (e.getRiakObjectLocation().getKey().toString().equals(expectedObjectKey)) + { + found = true; + assertEquals(INDEX_ENTRY, e.getIndexKey()); + } + } + return found; + } + private static void createIndexedPojo(int keySuffix, BigInteger indexValue) throws InterruptedException + { IndexedPojo ip = new IndexedPojo(); - ip.key = "index_test_object_key1"; - ip.bucketName = bucketName.toString(); - ip.indexKey = new BigInteger("91234567890123456789012345678901234567890"); + ip.key = objectKey(keySuffix); + ip.bucketName = namespace.getBucketNameAsString(); + ip.indexKey = indexValue; ip.value = "My Object Value!"; StoreValue sv = new StoreValue.Builder(ip).build(); @@ -113,45 +155,14 @@ public void testRangeQuery() throws InterruptedException, ExecutionException svFuture.await(); assertTrue(svFuture.isSuccess()); + } - IndexedPojo ip2 = new IndexedPojo(); - ip2.key = "index_test_object_key2"; - ip2.bucketName = bucketName.toString(); - ip2.indexKey = new BigInteger("91234567890123456789012345678901234567898"); - ip2.value = "My Object Value!"; - - sv = new StoreValue.Builder(ip2).build(); - svFuture = client.executeAsync(sv); - - svFuture.await(); - assertTrue(svFuture.isSuccess()); - - Namespace ns = new Namespace(Namespace.DEFAULT_BUCKET_TYPE, bucketName.toString()); - BigIntIndexQuery iiq = - new BigIntIndexQuery.Builder(ns, "test_index", - new BigInteger("91234567890123456789012345678901234567890"), - new BigInteger("91234567890123456789012345678901234567898")) - .withKeyAndIndex(true) - .build(); - - BigIntIndexQuery.Response iResp = client.execute(iiq); - assertTrue(iResp.hasEntries()); - assertEquals(2, iResp.getEntries().size()); - - boolean found = false; - for (BigIntIndexQuery.Response.Entry e : iResp.getEntries()) - { - if (e.getRiakObjectLocation().getKey().toString().equals("index_test_object_key1")) - { - found = true; - assertEquals(ip.indexKey, e.getIndexKey()); - } - } - - assertTrue(found); + private static String objectKey(int suffix) + { + return OBJECT_KEY_BASE + suffix; } - public static class IndexedPojo + private static class IndexedPojo { @RiakKey public String key; @@ -159,12 +170,9 @@ public static class IndexedPojo @RiakBucketName public String bucketName; - @RiakIndex(name="test_index") + @RiakIndex(name = INDEX_NAME) BigInteger indexKey; - @RiakVClock - VClock vclock; - public String value; } } diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java index 979872335..19874fd49 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java @@ -20,91 +20,134 @@ import com.basho.riak.client.api.annotations.RiakBucketName; import com.basho.riak.client.api.annotations.RiakIndex; import com.basho.riak.client.api.annotations.RiakKey; -import com.basho.riak.client.api.annotations.RiakVClock; -import com.basho.riak.client.api.cap.VClock; -import com.basho.riak.client.core.RiakFuture; -import com.basho.riak.client.core.operations.itest.ITestAutoCleanupBase; import com.basho.riak.client.api.commands.indexes.BinIndexQuery; import com.basho.riak.client.api.commands.kv.StoreValue; +import com.basho.riak.client.core.RiakFuture; import com.basho.riak.client.core.query.Location; -import com.basho.riak.client.core.query.Namespace; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; + import java.util.concurrent.ExecutionException; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import org.junit.Assume; -import org.junit.Test; /** - * * @author Brian Roach + * @author Alex Moore * @since 2.0 */ -public class ITestBinIndexQuery extends ITestAutoCleanupBase +public class ITestBinIndexQuery extends ITestIndexBase { + private static final String OBJECT_KEY_BASE = "index_test_object_key"; + private static final String INDEX_NAME = "test_index"; + private static final String INDEX_KEY_BASE = "index_test_index_key"; + private static RiakClient client = new RiakClient(cluster); + + @BeforeClass + public static void setupData() throws InterruptedException + { + createIndexedPojo(0, 1); + createIndexedPojo(1, 1); + createIndexedPojo(2, 2); + } + @Test public void testMatchQuery() throws ExecutionException, InterruptedException { Assume.assumeTrue(test2i); - RiakClient client = new RiakClient(cluster); + BinIndexQuery indexQuery = new BinIndexQuery.Builder(namespace, INDEX_NAME, indexKey(1)).withKeyAndIndex(true) + .build(); + BinIndexQuery.Response indexResponse = client.execute(indexQuery); - IndexedPojo ip = new IndexedPojo(); - ip.key = "index_test_object_key"; - ip.bucketName = bucketName.toString(); - ip.indexKey = "index_test_index_key"; - ip.value = "My Object Value!"; + assertTrue(indexResponse.hasEntries()); + assertEquals(2, indexResponse.getEntries().size()); - StoreValue sv = new StoreValue.Builder(ip).build(); - RiakFuture svFuture = client.executeAsync(sv); + boolean found = assertFirstObjectFound(indexResponse.getEntries()); - svFuture.await(); - assertTrue(svFuture.isSuccess()); + assertTrue(found); + } - IndexedPojo ip2 = new IndexedPojo(); - ip2.key = "index_test_object_key2"; - ip2.bucketName = bucketName.toString(); - ip2.indexKey = "index_test_index_key"; - ip2.value = "My Object Value!"; + @Test + public void testRangeQuery() throws InterruptedException, ExecutionException + { + Assume.assumeTrue(test2i); - sv = new StoreValue.Builder(ip2).build(); - svFuture = client.executeAsync(sv); + BinIndexQuery indexQuery = new BinIndexQuery.Builder(namespace, + INDEX_NAME, + indexKey(0), + indexKey(9)).withKeyAndIndex(true).build(); + BinIndexQuery.Response indexResponse = client.execute(indexQuery); - svFuture.await(); - assertTrue(svFuture.isSuccess()); + assertTrue(indexResponse.hasEntries()); + assertEquals(3, indexResponse.getEntries().size()); + + boolean found = assertFirstObjectFound(indexResponse.getEntries()); + + assertTrue(found); + } - Namespace ns = new Namespace(Namespace.DEFAULT_BUCKET_TYPE, bucketName.toString()); + @Test + public void testStreamingRangeQuery() throws ExecutionException, InterruptedException + { + Assume.assumeTrue(test2i); - BinIndexQuery biq = - new BinIndexQuery.Builder(ns, "test_index", "index_test_index_key").withKeyAndIndex(true).build(); - BinIndexQuery.Response iResp = client.execute(biq); + BinIndexQuery indexQuery = new BinIndexQuery.Builder(namespace, + INDEX_NAME, + indexKey(0), + indexKey(9)).withKeyAndIndex(true).build(); - assertTrue(iResp.hasEntries()); - assertEquals(2, iResp.getEntries().size()); + final RiakFuture streamingFuture = + client.executeAsyncStreaming(indexQuery, 200); + final BinIndexQuery.StreamingResponse streamingResponse = streamingFuture.get(); + + assertTrue(streamingResponse.hasEntries()); + + final String expectedObjectKey = objectKey(1); + final String expectedIndexKey = indexKey(1); boolean found = false; - for (BinIndexQuery.Response.Entry e : iResp.getEntries()) + int size = 0; + + for (BinIndexQuery.Response.Entry e : streamingResponse) { - if (e.getRiakObjectLocation().getKey().toString().equals("index_test_object_key")) + size++; + if (e.getRiakObjectLocation().getKey().toString().equals(expectedObjectKey)) { found = true; - assertEquals(ip.indexKey, e.getIndexKey()); + assertEquals(expectedIndexKey, e.getIndexKey()); } } assertTrue(found); + assertEquals(3, size); } - @Test - public void testRangeQuery() throws InterruptedException, ExecutionException + private boolean assertFirstObjectFound(Iterable entries) { - Assume.assumeTrue(test2i); + final String expectedObjectKey = objectKey(1); + final String expectedIndexKey = indexKey(1); + boolean found = false; - RiakClient client = new RiakClient(cluster); + for (BinIndexQuery.Response.Entry e : entries) + { + if (e.getRiakObjectLocation().getKey().toString().equals(expectedObjectKey)) + { + found = true; + assertEquals(expectedIndexKey, e.getIndexKey()); + } + } + return found; + } + private static void createIndexedPojo(int keySuffix, int indexSuffix) throws InterruptedException + { IndexedPojo ip = new IndexedPojo(); - ip.key = "index_test_object_key1"; - ip.bucketName = bucketName.toString(); - ip.indexKey = "index_test_index_key1"; + ip.key = objectKey(keySuffix); + ip.bucketName = namespace.getBucketNameAsString(); + ip.indexKey = indexKey(indexSuffix); ip.value = "My Object Value!"; StoreValue sv = new StoreValue.Builder(ip).build(); @@ -112,41 +155,19 @@ public void testRangeQuery() throws InterruptedException, ExecutionException svFuture.await(); assertTrue(svFuture.isSuccess()); + } - IndexedPojo ip2 = new IndexedPojo(); - ip2.key = "index_test_object_key2"; - ip2.bucketName = bucketName.toString(); - ip2.indexKey = "index_test_index_key2"; - ip2.value = "My Object Value!"; - - sv = new StoreValue.Builder(ip2).build(); - svFuture = client.executeAsync(sv); - - svFuture.await(); - assertTrue(svFuture.isSuccess()); - - Namespace ns = new Namespace(Namespace.DEFAULT_BUCKET_TYPE, bucketName.toString()); - BinIndexQuery biq = - new BinIndexQuery.Builder(ns, "test_index", "index_test_index_key0", "index_test_index_key9").withKeyAndIndex(true).build(); - BinIndexQuery.Response iResp = client.execute(biq); - - assertTrue(iResp.hasEntries()); - assertEquals(2, iResp.getEntries().size()); - - boolean found = false; - for (BinIndexQuery.Response.Entry e : iResp.getEntries()) - { - if (e.getRiakObjectLocation().getKey().toString().equals("index_test_object_key1")) - { - found = true; - assertEquals(ip.indexKey, e.getIndexKey()); - } - } + private static String objectKey(int suffix) + { + return OBJECT_KEY_BASE + suffix; + } - assertTrue(found); + private static String indexKey(int suffix) + { + return INDEX_KEY_BASE + suffix; } - public static class IndexedPojo + private static class IndexedPojo { @RiakKey public String key; @@ -154,12 +175,9 @@ public static class IndexedPojo @RiakBucketName public String bucketName; - @RiakIndex(name="test_index") + @RiakIndex(name = INDEX_NAME) String indexKey; - @RiakVClock - VClock vclock; - public String value; } } diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIndexBase.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIndexBase.java new file mode 100644 index 000000000..81ea3d035 --- /dev/null +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIndexBase.java @@ -0,0 +1,18 @@ +package com.basho.riak.client.api.commands.indexes.itest; + +import com.basho.riak.client.core.operations.itest.ITestBase; +import com.basho.riak.client.core.query.Namespace; +import org.junit.AfterClass; + +import java.util.concurrent.ExecutionException; + +public abstract class ITestIndexBase extends ITestBase +{ + protected static Namespace namespace = new Namespace(bucketName.toString()); + + @AfterClass + public static void teardown() throws ExecutionException, InterruptedException + { + resetAndEmptyBucket(namespace); + } +} diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java index 0c27e3397..84f72f29a 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java @@ -20,91 +20,138 @@ import com.basho.riak.client.api.annotations.RiakBucketName; import com.basho.riak.client.api.annotations.RiakIndex; import com.basho.riak.client.api.annotations.RiakKey; -import com.basho.riak.client.api.annotations.RiakVClock; -import com.basho.riak.client.api.cap.VClock; -import com.basho.riak.client.core.RiakFuture; -import com.basho.riak.client.core.operations.itest.ITestAutoCleanupBase; import com.basho.riak.client.api.commands.indexes.IntIndexQuery; +import com.basho.riak.client.api.commands.kv.FetchValue; import com.basho.riak.client.api.commands.kv.StoreValue; +import com.basho.riak.client.core.RiakFuture; import com.basho.riak.client.core.query.Location; -import com.basho.riak.client.core.query.Namespace; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; + import java.util.concurrent.ExecutionException; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import org.junit.Assume; -import org.junit.Test; /** - * * @author Brian Roach + * @author Alex Moore * @since 2.0 */ -public class ITestIntIndexQuery extends ITestAutoCleanupBase +public class ITestIntIndexQuery extends ITestIndexBase { + private static final String OBJECT_KEY_BASE = "index_test_object_key"; + private static final String INDEX_NAME = "test_index"; + private static final Long DUP_INDEX_KEY = 123456L; + private static RiakClient client = new RiakClient(cluster); + + @BeforeClass + public static void setupData() throws InterruptedException, ExecutionException + { + createIndexedPojo(0, DUP_INDEX_KEY); + createIndexedPojo(1, DUP_INDEX_KEY); + createIndexedPojo(2, 25L); + } + + @Test + public void testFetchThings() throws ExecutionException, InterruptedException + { + FetchValue fv = new FetchValue.Builder(new Location(namespace, objectKey(0))).build(); + final FetchValue.Response execute = client.execute(fv); + + execute.getValues(); + } + @Test public void testMatchQuery() throws InterruptedException, ExecutionException { Assume.assumeTrue(test2i); - RiakClient client = new RiakClient(cluster); + IntIndexQuery indexQuery = new IntIndexQuery.Builder(namespace, + INDEX_NAME, DUP_INDEX_KEY).withKeyAndIndex(true).build(); - IndexedPojo ip = new IndexedPojo(); - ip.key = "index_test_object_key"; - ip.bucketName = bucketName.toString(); - ip.indexKey = 123456L; - ip.value = "My Object Value!"; + IntIndexQuery.Response indexResponse = client.execute(indexQuery); - StoreValue sv = new StoreValue.Builder(ip).build(); - RiakFuture svFuture = client.executeAsync(sv); + assertTrue(indexResponse.hasEntries()); + assertEquals(2, indexResponse.getEntries().size()); - svFuture.await(); - assertTrue(svFuture.isSuccess()); + assertFirstObjectFound(indexResponse); + } - IndexedPojo ip2 = new IndexedPojo(); - ip2.key = "index_test_object_key2"; - ip2.bucketName = bucketName.toString(); - ip2.indexKey = 123456L; - ip2.value = "My Object Value!"; + @Test + public void testRangeQuery() throws InterruptedException, ExecutionException + { + Assume.assumeTrue(test2i); - sv = new StoreValue.Builder(ip2).build(); - svFuture = client.executeAsync(sv); + IntIndexQuery indexQuery = new IntIndexQuery.Builder(namespace, + INDEX_NAME, + Long.MIN_VALUE, + Long.MAX_VALUE).withKeyAndIndex(true).build(); - svFuture.await(); - assertTrue(svFuture.isSuccess()); + IntIndexQuery.Response indexResponse = client.execute(indexQuery); + assertTrue(indexResponse.hasEntries()); + assertEquals(3, indexResponse.getEntries().size()); + + assertFirstObjectFound(indexResponse); + } + + @Test + public void testStreamingRangeQuery() throws ExecutionException, InterruptedException + { + Assume.assumeTrue(test2i); + + IntIndexQuery indexQuery = new IntIndexQuery.Builder(namespace, + INDEX_NAME, + Long.MIN_VALUE, + Long.MAX_VALUE).withKeyAndIndex(true).build(); - Namespace ns = new Namespace(Namespace.DEFAULT_BUCKET_TYPE, bucketName.toString()); + final RiakFuture streamingFuture = + client.executeAsyncStreaming(indexQuery, 200); - IntIndexQuery iiq = - new IntIndexQuery.Builder(ns, "test_index", 123456L).withKeyAndIndex(true).build(); - IntIndexQuery.Response iResp = client.execute(iiq); + final IntIndexQuery.StreamingResponse streamingResponse = streamingFuture.get(); - assertTrue(iResp.hasEntries()); - assertEquals(2, iResp.getEntries().size()); + assertTrue(streamingResponse.hasEntries()); + final String expectedObjectKey = objectKey(1); boolean found = false; - for (IntIndexQuery.Response.Entry e : iResp.getEntries()) + int size = 0; + + for (IntIndexQuery.Response.Entry e : streamingResponse) { - if (e.getRiakObjectLocation().getKey().toString().equals("index_test_object_key")) + size++; + if (e.getRiakObjectLocation().getKey().toString().equals(expectedObjectKey)) { found = true; - assertEquals(ip.indexKey, e.getIndexKey()); + assertEquals(DUP_INDEX_KEY, e.getIndexKey()); } } assertTrue(found); + assertEquals(3, size); } - @Test - public void testRangeQuery() throws InterruptedException, ExecutionException + private void assertFirstObjectFound(IntIndexQuery.Response indexResponse) { - Assume.assumeTrue(test2i); + boolean found = false; + for (IntIndexQuery.Response.Entry e : indexResponse.getEntries()) + { + if (e.getRiakObjectLocation().getKey().toString().equals(objectKey(1))) + { + found = true; + assertEquals(DUP_INDEX_KEY, e.getIndexKey()); + } + } - RiakClient client = new RiakClient(cluster); + assertTrue(found); + } + private static void createIndexedPojo(int keySuffix, long indexKey) throws InterruptedException + { IndexedPojo ip = new IndexedPojo(); - ip.key = "index_test_object_key1"; - ip.bucketName = bucketName.toString(); - ip.indexKey = 1L; + ip.key = objectKey(keySuffix); + ip.bucketName = namespace.getBucketNameAsString(); + ip.indexKey = indexKey; ip.value = "My Object Value!"; StoreValue sv = new StoreValue.Builder(ip).build(); @@ -112,44 +159,14 @@ public void testRangeQuery() throws InterruptedException, ExecutionException svFuture.await(); assertTrue(svFuture.isSuccess()); + } - IndexedPojo ip2 = new IndexedPojo(); - ip2.key = "index_test_object_key2"; - ip2.bucketName = bucketName.toString(); - ip2.indexKey = 25L; - ip2.value = "My Object Value!"; - - sv = new StoreValue.Builder(ip2).build(); - svFuture = client.executeAsync(sv); - - svFuture.await(); - assertTrue(svFuture.isSuccess()); - - Namespace ns = new Namespace(Namespace.DEFAULT_BUCKET_TYPE, bucketName.toString()); - - IntIndexQuery iiq = - new IntIndexQuery.Builder(ns, "test_index", Long.MIN_VALUE, Long.MAX_VALUE) - .withKeyAndIndex(true) - .build(); - - IntIndexQuery.Response iResp = client.execute(iiq); - assertTrue(iResp.hasEntries()); - assertEquals(2, iResp.getEntries().size()); - - boolean found = false; - for (IntIndexQuery.Response.Entry e : iResp.getEntries()) - { - if (e.getRiakObjectLocation().getKey().toString().equals("index_test_object_key1")) - { - found = true; - assertEquals(ip.indexKey, e.getIndexKey()); - } - } - - assertTrue(found); + private static String objectKey(int suffix) + { + return OBJECT_KEY_BASE + suffix; } - public static class IndexedPojo + private static class IndexedPojo { @RiakKey public String key; @@ -157,12 +174,9 @@ public static class IndexedPojo @RiakBucketName public String bucketName; - @RiakIndex(name="test_index") + @RiakIndex(name = "test_index") Long indexKey; - @RiakVClock - VClock vclock; - public String value; } } diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java index d0178aa2c..1ffebebd3 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java @@ -20,33 +20,40 @@ import com.basho.riak.client.api.annotations.RiakBucketName; import com.basho.riak.client.api.annotations.RiakIndex; import com.basho.riak.client.api.annotations.RiakKey; -import com.basho.riak.client.api.annotations.RiakVClock; -import com.basho.riak.client.api.cap.VClock; -import com.basho.riak.client.api.commands.indexes.*; +import com.basho.riak.client.api.commands.indexes.BinIndexQuery; +import com.basho.riak.client.api.commands.indexes.BucketIndexQuery; +import com.basho.riak.client.api.commands.indexes.KeyIndexQuery; +import com.basho.riak.client.api.commands.indexes.RawIndexQuery; import com.basho.riak.client.api.commands.indexes.SecondaryIndexQuery.Type; import com.basho.riak.client.api.commands.kv.StoreValue; import com.basho.riak.client.core.RiakFuture; import com.basho.riak.client.core.operations.StoreOperation; -import com.basho.riak.client.core.operations.itest.ITestAutoCleanupBase; +import com.basho.riak.client.core.operations.itest.ITestBase; import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.query.RiakObject; import com.basho.riak.client.core.query.indexes.IndexNames; import com.basho.riak.client.core.util.BinaryValue; -import org.junit.*; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; import java.util.concurrent.ExecutionException; +import java.util.stream.StreamSupport; import static org.junit.Assert.*; /** * * @author Brian Roach + * @author Alex Moore */ -public class ITestRawIndexQuery extends ITestAutoCleanupBase +public class ITestRawIndexQuery extends ITestBase { private static String sharedBucket = "ITestRawIndexQuery"; private static Namespace sharedNamespace = new Namespace(Namespace.DEFAULT_BUCKET_TYPE, sharedBucket); + private static RiakClient client = new RiakClient(cluster); @BeforeClass public static void Setup() throws ExecutionException, InterruptedException @@ -79,8 +86,6 @@ public void simpleTest() throws InterruptedException, ExecutionException setBucketNameToTestName(); - RiakClient client = new RiakClient(cluster); - BinaryValue indexKey = BinaryValue.create("index_test_index_key"); IndexedPojo ip = new IndexedPojo(); @@ -112,8 +117,6 @@ public void testKeyIndexHack() throws InterruptedException, ExecutionException { Assume.assumeTrue(test2i); - RiakClient client = new RiakClient(cluster); - RawIndexQuery biq = new RawIndexQuery.Builder(sharedNamespace, IndexNames.KEY, @@ -132,8 +135,6 @@ public void testKeyIndexQuery() throws InterruptedException, ExecutionException { Assume.assumeTrue(test2i); - RiakClient client = new RiakClient(cluster); - KeyIndexQuery kq = new KeyIndexQuery.Builder(sharedNamespace, "my_key10", "my_key19").build(); final RawIndexQuery.Response kqResp = client.execute(kq); @@ -146,8 +147,6 @@ public void testBucketIndexHack() throws InterruptedException, ExecutionExceptio { Assume.assumeTrue(test2i); - RiakClient client = new RiakClient(cluster); - RawIndexQuery biq = new RawIndexQuery.Builder(sharedNamespace, IndexNames.BUCKET, Type._BUCKET, BinaryValue.create(sharedBucket)) @@ -163,8 +162,6 @@ public void testBucketIndexQuery() throws InterruptedException, ExecutionExcepti { Assume.assumeTrue(test2i); - RiakClient client = new RiakClient(cluster); - BucketIndexQuery bq = new BucketIndexQuery.Builder(sharedNamespace).build(); final BinIndexQuery.Response bqResp = client.execute(bq); @@ -172,7 +169,26 @@ public void testBucketIndexQuery() throws InterruptedException, ExecutionExcepti assertEquals(100, bqResp.getEntries().size()); } - public static class IndexedPojo + @Test + public void testBucketIndexQueryStreaming() throws InterruptedException, ExecutionException + { + Assume.assumeTrue(test2i); + + BucketIndexQuery bq = new BucketIndexQuery.Builder(sharedNamespace).build(); + + final RiakFuture indexResult = + client.executeAsyncStreaming(bq, 100); + + final BinIndexQuery.StreamingResponse streamingResponse = indexResult.get(); + + assertTrue(streamingResponse.hasEntries()); + assertEquals(100, StreamSupport.stream(streamingResponse.spliterator(), false).count()); + + // Assert everything was consumed + assertFalse(streamingResponse.hasEntries()); + } + + private static class IndexedPojo { @RiakKey public String key; @@ -183,9 +199,6 @@ public static class IndexedPojo @RiakIndex(name="test_index_bin") byte[] indexKey; - @RiakVClock - VClock vclock; - public String value; } } From 9ac71b6bb8268b99907b5f7e9f89b7ae93758029 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Wed, 19 Oct 2016 11:28:13 -0400 Subject: [PATCH 14/55] Implement Streaming MapReduce Results + Test --- .../api/commands/mapreduce/MapReduce.java | 118 +++++++++++++++++- .../commands/itest/ITestBucketMapReduce.java | 79 ++++++++++++ 2 files changed, 191 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java index 4e53d6a1d..671022afe 100644 --- a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java +++ b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java @@ -13,13 +13,16 @@ */ package com.basho.riak.client.api.commands.mapreduce; -import com.basho.riak.client.api.RiakCommand; import com.basho.riak.client.api.RiakException; +import com.basho.riak.client.api.StreamableRiakCommand; import com.basho.riak.client.api.commands.CoreFutureAdapter; +import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter; import com.basho.riak.client.api.convert.ConversionException; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; +import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.MapReduceOperation; +import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.query.functions.Function; import com.basho.riak.client.core.util.BinaryValue; import com.fasterxml.jackson.core.JsonEncoding; @@ -33,10 +36,9 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TransferQueue; /** * Base abstract class for all MapReduce commands. @@ -46,7 +48,7 @@ * @author Dave Rusek * @since 2.0 */ -public abstract class MapReduce extends RiakCommand +public abstract class MapReduce extends StreamableRiakCommand { private final MapReduceSpec spec; @@ -95,6 +97,35 @@ protected BinaryValue convertQueryInfo(BinaryValue coreQueryInfo) return future; } + @Override + protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout) + { + BinaryValue jobSpec; + try + { + String spec = writeSpec(); + jobSpec = BinaryValue.create(spec); + } + catch (RiakException e) + { + throw new RuntimeException(e); + } + + MapReduceOperation operation = new MapReduceOperation.Builder(jobSpec).streamResults(true).build(); + + final StreamingRiakFuture coreFuture = cluster.execute(operation); + + final StreamingResponse streamingResponse = new StreamingResponse(coreFuture, timeout); + + ImmediateCoreFutureAdapter.SameQueryInfo future = + new ImmediateCoreFutureAdapter.SameQueryInfo( + coreFuture, streamingResponse) {}; + + coreFuture.addListener(future); + + return future; + } + /** * Creates the JSON string of the M/R job for submitting to the client *

@@ -395,4 +426,79 @@ private ArrayNode flattenResults() return flatArray; } } + + public static class StreamingResponse implements Iterable + { + final MapReduceResponseIterator responseIterator; + + StreamingResponse(StreamingRiakFuture coreFuture, + int pollTimeout) + { + responseIterator = new MapReduceResponseIterator(coreFuture, pollTimeout); + } + + @Override + public Iterator iterator() + { + return responseIterator; + } + + private class MapReduceResponseIterator implements Iterator + { + final StreamingRiakFuture coreFuture; + final TransferQueue resultsQueue; + private final int pollTimeout; + + MapReduceResponseIterator(StreamingRiakFuture coreFuture, + int pollTimeout) + { + this.coreFuture = coreFuture; + this.resultsQueue = coreFuture.getResultsQueue(); + this.pollTimeout = pollTimeout; + } + + @Override + public boolean hasNext() + { + if(resultsQueueHasAny()) + { + return true; + } + + try + { + return peekWaitForNextQueueEntry(); + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private boolean peekWaitForNextQueueEntry() throws InterruptedException + { + while (!resultsQueueHasAny() && !coreFuture.isDone()) + { + if(!resultsQueueHasAny()) + { + Thread.sleep(pollTimeout); + } + } + return resultsQueueHasAny(); + } + + private boolean resultsQueueHasAny() + { + return resultsQueue.peek() != null; + } + + @Override + public Response next() + { + final MapReduceOperation.Response responseChunk = resultsQueue.remove(); + return new Response(responseChunk.getResults()); + } + } + } } diff --git a/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java b/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java index f88e26dfb..3ede46452 100644 --- a/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java +++ b/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java @@ -34,12 +34,18 @@ import com.basho.riak.client.api.commands.mapreduce.filters.TokenizeFilter; import com.basho.riak.client.core.query.functions.Function; import com.basho.riak.client.core.util.BinaryValue; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; + +import java.util.List; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.junit.*; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -117,6 +123,19 @@ public void erlangBucketMRTestType() throws InterruptedException, ExecutionExcep erlangBucketMR(mapReduceBucketType.toString()); } + @Test + public void erlangBucketMRDefaultTypeStreaming() throws InterruptedException, ExecutionException + { + streamingErlangBucketMR(Namespace.DEFAULT_BUCKET_TYPE); + } + + @Test + public void erlangBucketMRTestTypeStreaming() throws InterruptedException, ExecutionException + { + Assume.assumeTrue(testBucketType); + streamingErlangBucketMR(mapReduceBucketType.toString()); + } + private void erlangBucketMR(String bucketType) throws InterruptedException, ExecutionException { Namespace ns = new Namespace(bucketType, mrBucketName); @@ -140,6 +159,66 @@ private void erlangBucketMR(String bucketType) throws InterruptedException, Exec assertEquals(199, result.get(199).asInt()); } + private void streamingErlangBucketMR(String bucketType) throws InterruptedException, ExecutionException + { + Namespace ns = new Namespace(bucketType, mrBucketName); + BucketMapReduce bmr = + new BucketMapReduce.Builder() + .withNamespace(ns) + .withMapPhase(Function.newErlangFunction("riak_kv_mapreduce", "map_object_value"), false) + .withReducePhase(Function.newErlangFunction("riak_kv_mapreduce", "reduce_string_to_integer"), false) + .withReducePhase(Function.newErlangFunction("riak_kv_mapreduce", "reduce_sort"), true) + .build(); + + final RiakFuture streamingFuture = + client.executeAsyncStreaming(bmr, 10); + + boolean found42 = false; + boolean found199 = false; + int count = 0; + + final MapReduce.StreamingResponse streamingResponse = streamingFuture.get(); + // The streaming query should return many results which are JSON arrays, each + // containing a piece of the array [0-199]. + // Streaming result would look like: [[0], [1,2,3], ... [..., 199]], with the outer + // array being the different response chunks streaming in. + for (MapReduce.Response response : streamingResponse) + { + int phaseSize = response.getResultsFromAllPhases().size(); + + if(phaseSize == 0) + { + continue; + } + + count += phaseSize; + + final ArrayNode result = response.getResultForPhase(2); + if(result == null) + { + continue; + } + + final String valuesString = result.toString(); + + if(!found42) + { + found42 = valuesString.contains("42"); + } + if(!found199) + { + found199 = valuesString.contains("199"); + } + } + + assertEquals(200, count); + assertTrue(found42); + assertTrue(found199); + + // Assert that we have consumed the responses, and none are left. + assertFalse(streamingFuture.get().iterator().hasNext()); + } + @Test public void JsBucketMRDefaultType() throws InterruptedException, ExecutionException { From d19cd3eff3debcda91d60c385faef463ab4a1b4d Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Thu, 20 Oct 2016 09:00:31 -0400 Subject: [PATCH 15/55] Fix spacing in control structures. --- .../client/api/commands/ChunkedResponseIterator.java | 4 ++-- .../client/api/commands/mapreduce/MapReduce.java | 4 ++-- .../riak/client/core/StreamingFutureOperation.java | 2 +- .../api/commands/buckets/itest/ITestListBuckets.java | 6 +++--- .../api/commands/buckets/itest/ITestListKeys.java | 6 +++--- .../api/commands/itest/ITestBucketMapReduce.java | 8 ++++---- .../operations/itest/ITestListBucketsOperation.java | 6 +++--- .../operations/itest/ITestListKeysOperation.java | 6 +++--- .../operations/itest/ITestMapReduceOperation.java | 12 ++++++------ .../operations/itest/ITestSecondaryIndexQueryOp.java | 6 +++--- 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java index 7ec733a3f..d36078c0a 100644 --- a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java +++ b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java @@ -46,7 +46,7 @@ public ChunkedResponseIterator(StreamingRiakFuture coreFuture, @Override public boolean hasNext() { - if(currentIteratorHasNext()) + if (currentIteratorHasNext()) { return true; } @@ -114,7 +114,7 @@ private void loadNextChunkIterator() private void loadContinuation(ChunkT nextChunk) { final BinaryValue fetchedContinuation = getContinuationFn.apply(nextChunk); - if(this.continuation == null && fetchedContinuation != null) + if (this.continuation == null && fetchedContinuation != null) { this.continuation = fetchedContinuation; } diff --git a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java index 671022afe..a4a59b1f9 100644 --- a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java +++ b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java @@ -460,7 +460,7 @@ private class MapReduceResponseIterator implements Iterator @Override public boolean hasNext() { - if(resultsQueueHasAny()) + if (resultsQueueHasAny()) { return true; } @@ -480,7 +480,7 @@ private boolean peekWaitForNextQueueEntry() throws InterruptedException { while (!resultsQueueHasAny() && !coreFuture.isDone()) { - if(!resultsQueueHasAny()) + if (!resultsQueueHasAny()) { Thread.sleep(pollTimeout); } diff --git a/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java b/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java index bf2429738..37ab59601 100644 --- a/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java @@ -21,7 +21,7 @@ protected StreamingFutureOperation(boolean streamResults) @Override protected void processMessage(ResponseType decodedMessage) { - if(!streamResults) + if (!streamResults) { super.processMessage(decodedMessage); return; diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java index 5af4e0eb8..7fbea0b54 100644 --- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java +++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java @@ -36,7 +36,7 @@ public class ITestListBuckets extends ITestBase public static void setup() throws ExecutionException, InterruptedException { storeTestObject(defaultNamespace); - if(testBucketType) + if (testBucketType) { storeTestObject(typedNamespace); } @@ -46,7 +46,7 @@ public static void setup() throws ExecutionException, InterruptedException public static void cleanup() throws ExecutionException, InterruptedException { resetAndEmptyBucket(defaultNamespace); - if(testBucketType) + if (testBucketType) { resetAndEmptyBucket(typedNamespace); } @@ -101,7 +101,7 @@ private void testListBucketsStreaming(Namespace namespace) throws InterruptedExc assumeTrue(iterator.hasNext()); boolean found = false; - while(!found && iterator.hasNext()) + while (!found && iterator.hasNext()) { final Namespace next = iterator.next(); found = next.getBucketName().toString().equals(bucketName); diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java index d74c23905..3461db04e 100644 --- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java +++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java @@ -33,7 +33,7 @@ public class ITestListKeys extends ITestBase @BeforeClass public static void setup() throws ExecutionException, InterruptedException { - if(testBucketType) + if (testBucketType) { storeTestObjects(typedNamespace); } @@ -42,7 +42,7 @@ public static void setup() throws ExecutionException, InterruptedException @AfterClass public static void cleanup() throws ExecutionException, InterruptedException { - if(testBucketType) + if (testBucketType) { resetAndEmptyBucket(typedNamespace); } @@ -67,7 +67,7 @@ public void testLargeStreamingListKeys() throws ExecutionException, InterruptedE { count++; - if(!foundLastKey) + if (!foundLastKey) { foundLastKey = location.getKeyAsString().equals("9999"); } diff --git a/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java b/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java index 3ede46452..6b385c1ff 100644 --- a/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java +++ b/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java @@ -186,7 +186,7 @@ private void streamingErlangBucketMR(String bucketType) throws InterruptedExcept { int phaseSize = response.getResultsFromAllPhases().size(); - if(phaseSize == 0) + if (phaseSize == 0) { continue; } @@ -194,18 +194,18 @@ private void streamingErlangBucketMR(String bucketType) throws InterruptedExcept count += phaseSize; final ArrayNode result = response.getResultForPhase(2); - if(result == null) + if (result == null) { continue; } final String valuesString = result.toString(); - if(!found42) + if (!found42) { found42 = valuesString.contains("42"); } - if(!found199) + if (!found199) { found199 = valuesString.contains("199"); } diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListBucketsOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListBucketsOperation.java index 4e646c318..6e7d7d934 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListBucketsOperation.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListBucketsOperation.java @@ -125,18 +125,18 @@ private void testBucketListStreaming(String bucketType, int bucketCount) throws List actualBuckets = new LinkedList<>(); int timeouts = 0; - while(!execute.isDone()) + while (!execute.isDone()) { final ListBucketsOperation.Response response = resultsQueue.poll(5, TimeUnit.MILLISECONDS); - if(response != null) + if (response != null) { actualBuckets.addAll(response.getBuckets()); continue; } timeouts++; - if(timeouts == 10) + if (timeouts == 10) { break; } diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java index 2a9f11c48..be7b7982a 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java @@ -169,18 +169,18 @@ private void testListKeysStreaming(String bucketType, int numExpected) throws In List actualKeys = new LinkedList<>(); int timeouts = 0; - while(!execute.isDone()) + while (!execute.isDone()) { final ListKeysOperation.Response response = resultsQueue.poll(5, TimeUnit.MILLISECONDS); - if(response != null) + if (response != null) { actualKeys.addAll(response.getKeys()); continue; } timeouts++; - if(timeouts == 10) + if (timeouts == 10) { break; } diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestMapReduceOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestMapReduceOperation.java index 9eacaf272..0abdfcbba 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestMapReduceOperation.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestMapReduceOperation.java @@ -51,7 +51,7 @@ public static void setup() throws ExecutionException, InterruptedException { insertData(new Namespace(Namespace.DEFAULT_BUCKET_TYPE, bucketName.toString())); - if(testBucketType) + if (testBucketType) { insertData(new Namespace(bucketType, bucketName)); } @@ -63,7 +63,7 @@ public static void cleanup() throws ExecutionException, InterruptedException { resetAndEmptyBucket(new Namespace(Namespace.DEFAULT_BUCKET_TYPE, bucketName.toString())); - if(testBucketType) + if (testBucketType) { resetAndEmptyBucket(new Namespace(bucketType, bucketName)); } @@ -138,10 +138,10 @@ private Map> testBasicStreamingMR(Namespace names Map> resultMap = new LinkedHashMap<>(); - while(!streamFuture.isDone()) + while (!streamFuture.isDone()) { final MapReduceOperation.Response response = resultsQueue.poll(10, TimeUnit.MILLISECONDS); - if(response == null) + if (response == null) { continue; } @@ -171,9 +171,9 @@ private void mergeWordCountMaps(Map> resultMap, @SuppressWarnings("unchecked") Map wordCountMap = objectMapper.readValue(wordCountMapJsonString, Map.class); - for(String wordCountKey: wordCountMap.keySet()) + for (String wordCountKey: wordCountMap.keySet()) { - if(currentPhaseMap.containsKey(wordCountKey)) + if (currentPhaseMap.containsKey(wordCountKey)) { final int newWordCountSum = currentPhaseMap.get(wordCountKey) + wordCountMap.get(wordCountKey); currentPhaseMap.put(wordCountKey, newWordCountSum); diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestSecondaryIndexQueryOp.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestSecondaryIndexQueryOp.java index e78e4aefe..bffb71596 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestSecondaryIndexQueryOp.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestSecondaryIndexQueryOp.java @@ -709,11 +709,11 @@ public StreamingResultsGatherer invoke() throws InterruptedException continuation = null; int timeouts = 0; - while(!future.isDone()) + while (!future.isDone()) { final SecondaryIndexQueryOperation.Response response = resultsQueue.poll(5, TimeUnit.MILLISECONDS); - if(response == null) + if (response == null) { if (++timeouts == 10) { @@ -741,7 +741,7 @@ private BinaryValue addResponseToResults(List Date: Thu, 20 Oct 2016 09:43:46 -0400 Subject: [PATCH 16/55] Fix headers for newest/recently changed files. --- .../riak/client/api/StreamableRiakCommand.java | 16 ++++++++++++++++ .../api/commands/ChunkedResponseIterator.java | 16 ++++++++++++++++ .../api/commands/ImmediateCoreFutureAdapter.java | 4 ++-- .../client/api/commands/datatypes/FetchHll.java | 5 +++-- .../client/api/commands/datatypes/HllUpdate.java | 5 +++-- .../client/api/commands/datatypes/UpdateHll.java | 5 +++-- .../commands/indexes/Streaming2iQueryFuture.java | 16 ++++++++++++++++ .../client/core/StreamingFutureOperation.java | 16 ++++++++++++++++ .../riak/client/core/StreamingRiakFuture.java | 4 ++-- .../client/core/query/crdt/types/RiakHll.java | 5 +++-- .../commands/buckets/itest/ITestListBuckets.java | 16 ++++++++++++++++ .../commands/buckets/itest/ITestListKeys.java | 16 ++++++++++++++++ .../commands/indexes/itest/ITestIndexBase.java | 16 ++++++++++++++++ .../itest/ITestListBucketsOperation.java | 1 + .../operations/itest/ITestListKeysOperation.java | 1 + 15 files changed, 130 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java index dfa2e9918..5522f0caf 100644 --- a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java +++ b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Basho Technologies, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.basho.riak.client.api; import com.basho.riak.client.core.RiakCluster; diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java index d36078c0a..f8cd28ea0 100644 --- a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java +++ b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Basho Technologies, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.basho.riak.client.api.commands; import com.basho.riak.client.core.StreamingRiakFuture; diff --git a/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java b/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java index 1922ef1b3..b98c5b769 100644 --- a/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java +++ b/src/main/java/com/basho/riak/client/api/commands/ImmediateCoreFutureAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright 2014 Brian Roach . + * Copyright 2016 Basho Technologies, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ * @param The converted query info type. * * @author Alex Moore - * @since 2.1 + * @since 2.1.0 */ public abstract class ImmediateCoreFutureAdapter extends CoreFutureAdapter { diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java index aa84aad08..169f54f8e 100644 --- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java +++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Basho Technologies Inc + * Copyright 2016 Basho Technologies, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,8 +38,9 @@ * } * * @author Alex Moore - * @since 2.1 + * @since 2.1.0 */ + public final class FetchHll extends FetchDatatype { private FetchHll(Builder builder) diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/HllUpdate.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/HllUpdate.java index 61cb5a642..ebb9c009a 100644 --- a/src/main/java/com/basho/riak/client/api/commands/datatypes/HllUpdate.java +++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/HllUpdate.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Basho Technologies Inc + * Copyright 2016 Basho Technologies, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.basho.riak.client.api.commands.datatypes; import com.basho.riak.client.core.query.crdt.ops.HllOp; @@ -30,7 +31,7 @@ * Riak HyperLogLog datatype. *

* @author Alex Moore - * @since 2.1 + * @since 2.1.0 */ public class HllUpdate implements DatatypeUpdate { diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java index de78cd304..fa8e81a64 100644 --- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java +++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Basho Technologies Inc + * Copyright 2016 Basho Technologies, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,8 +46,9 @@ *

* * @author Alex Moore - * @since 2.1 + * @since 2.1.0 */ + public class UpdateHll extends UpdateDatatype { private final HllUpdate update; diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/Streaming2iQueryFuture.java b/src/main/java/com/basho/riak/client/api/commands/indexes/Streaming2iQueryFuture.java index 909ddf217..126924b0e 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/Streaming2iQueryFuture.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/Streaming2iQueryFuture.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Basho Technologies, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.basho.riak.client.api.commands.indexes; import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter; diff --git a/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java b/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java index 37ab59601..cac8604f7 100644 --- a/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Basho Technologies, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.basho.riak.client.core; /** diff --git a/src/main/java/com/basho/riak/client/core/StreamingRiakFuture.java b/src/main/java/com/basho/riak/client/core/StreamingRiakFuture.java index 8e4043186..85bd2f604 100644 --- a/src/main/java/com/basho/riak/client/core/StreamingRiakFuture.java +++ b/src/main/java/com/basho/riak/client/core/StreamingRiakFuture.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 Basho Technologies Inc + * Copyright 2016 Basho Technologies, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.basho.riak.client.core; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.TransferQueue; /** diff --git a/src/main/java/com/basho/riak/client/core/query/crdt/types/RiakHll.java b/src/main/java/com/basho/riak/client/core/query/crdt/types/RiakHll.java index f47af3704..18638623e 100644 --- a/src/main/java/com/basho/riak/client/core/query/crdt/types/RiakHll.java +++ b/src/main/java/com/basho/riak/client/core/query/crdt/types/RiakHll.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Basho Technologies Inc + * Copyright 2016 Basho Technologies, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.basho.riak.client.core.query.crdt.types; /** @@ -22,7 +23,7 @@ *

* * @author Alex Moore - * @since 2.1 + * @since 2.1.0 */ public class RiakHll extends RiakDatatype { diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java index 7fbea0b54..85aea42fb 100644 --- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java +++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Basho Technologies, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.basho.riak.client.api.commands.buckets.itest; import com.basho.riak.client.api.RiakClient; diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java index 3461db04e..bd16bd1fa 100644 --- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java +++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Basho Technologies, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.basho.riak.client.api.commands.buckets.itest; import com.basho.riak.client.api.RiakClient; diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIndexBase.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIndexBase.java index 81ea3d035..24ffc77da 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIndexBase.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIndexBase.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Basho Technologies, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.basho.riak.client.api.commands.indexes.itest; import com.basho.riak.client.core.operations.itest.ITestBase; diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListBucketsOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListBucketsOperation.java index 6e7d7d934..1be1540ee 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListBucketsOperation.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListBucketsOperation.java @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.basho.riak.client.core.operations.itest; import com.basho.riak.client.core.RiakFuture; diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java index be7b7982a..b81bab2d4 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.basho.riak.client.core.operations.itest; import com.basho.riak.client.core.RiakFuture; From 3edac1f75551219cd340ae49418b1cb0c10003c9 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Thu, 20 Oct 2016 10:16:56 -0400 Subject: [PATCH 17/55] Don't need synchronization on next(), removing unused import --- .../basho/riak/client/api/commands/ChunkedResponseIterator.java | 2 +- .../client/api/commands/buckets/itest/ITestListBuckets.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java index f8cd28ea0..c6af48fd9 100644 --- a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java +++ b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java @@ -95,7 +95,7 @@ public BinaryValue getContinuation() } @Override - public synchronized FinalT next() + public FinalT next() { return createNext.apply(currentIterator.next()); } diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java index 85aea42fb..7f75a7400 100644 --- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java +++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java @@ -20,7 +20,6 @@ import com.basho.riak.client.api.commands.buckets.ListBuckets; import com.basho.riak.client.api.commands.kv.StoreValue; import com.basho.riak.client.core.RiakFuture; -import com.basho.riak.client.core.operations.itest.ITestAutoCleanupBase; import com.basho.riak.client.core.operations.itest.ITestBase; import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; From 4e0a90b79f48b58d84f17da633f3fdf69a529c8b Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 25 Oct 2016 00:17:59 -0400 Subject: [PATCH 18/55] Expand test coverage --- .../com/basho/riak/client/api/RiakClient.java | 5 +- .../commands/indexes/BigIntIndexQuery.java | 25 +------- .../buckets/itest/ITestListBuckets.java | 5 ++ .../indexes/itest/ITestRawIndexQuery.java | 24 ++++++++ .../core/ChunkedResponseIteratorTest.java | 61 +++++++++++++++++++ 5 files changed, 95 insertions(+), 25 deletions(-) create mode 100644 src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java diff --git a/src/main/java/com/basho/riak/client/api/RiakClient.java b/src/main/java/com/basho/riak/client/api/RiakClient.java index 346a097fd..4e922979c 100644 --- a/src/main/java/com/basho/riak/client/api/RiakClient.java +++ b/src/main/java/com/basho/riak/client/api/RiakClient.java @@ -413,9 +413,8 @@ public RiakFuture executeAsync(RiakCommand command) * @param StreamableRiakCommand's immediate return type, available before the command/operation is complete. * @param The RiakCommand's query info type. * @param command The RiakCommand to execute. - * @param timeoutMS The loading timeout in milliseconds for each result chunk. - * If the timeout is reached a {@see null} will be returned from the result's iterator, - * instead of blocking indefinitely. + * @param timeoutMS The polling timeout in milliseconds for each result chunk. + * If the timeout is reached it will try again, instead of blocking indefinitely. * @return a RiakFuture for the operation * @see RiakFuture */ diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java index d5d146946..cdaf25138 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java @@ -108,7 +108,9 @@ protected RiakFuture executeAsyncStreaming( StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout); - BigIntStreamingQueryFuture future = new BigIntStreamingQueryFuture(coreFuture, response); + Streaming2iQueryFuture future = + new Streaming2iQueryFuture<>(coreFuture, response, this); + coreFuture.addListener(future); return future; @@ -139,27 +141,6 @@ protected BigIntIndexQuery convertQueryInfo(SecondaryIndexQueryOperation.Query c } } - protected final class BigIntStreamingQueryFuture - extends ImmediateCoreFutureAdapter - { - - protected BigIntStreamingQueryFuture(StreamingRiakFuture coreFuture, - StreamingResponse immediateResponse) - { - super(coreFuture, immediateResponse); - } - - @Override - protected BigIntIndexQuery convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo) - { - return BigIntIndexQuery.this; - } - } - protected static abstract class Init> extends SecondaryIndexQuery.Init { public Init(Namespace namespace, String indexName, S start, S end) diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java index 7f75a7400..9978ac3f9 100644 --- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java +++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java @@ -32,6 +32,8 @@ import java.util.Iterator; import java.util.concurrent.ExecutionException; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; @@ -125,6 +127,9 @@ private void testListBucketsStreaming(Namespace namespace) throws InterruptedExc streamingFuture.await(); // Wait for command to finish, even if we've found our data assumeTrue(streamingFuture.isDone()); + assertFalse(streamingFuture.get().iterator().hasNext()); + assertEquals(namespace.getBucketType(), streamingFuture.getQueryInfo()); + assertTrue(found); } diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java index 1ffebebd3..da58194f6 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java @@ -188,6 +188,30 @@ public void testBucketIndexQueryStreaming() throws InterruptedException, Executi assertFalse(streamingResponse.hasEntries()); } + @Test + public void testIndexQueryStreamingContinuations() throws InterruptedException, ExecutionException + { + Assume.assumeTrue(test2i); + + BucketIndexQuery bq = new BucketIndexQuery.Builder(sharedNamespace).withMaxResults(50).withPaginationSort(true).build(); + + final RiakFuture indexResult = + client.executeAsyncStreaming(bq, 100); + + final BinIndexQuery.StreamingResponse streamingResponse = indexResult.get(); + + assertTrue(streamingResponse.hasEntries()); + assertEquals(50, StreamSupport.stream(streamingResponse.spliterator(), false).count()); + assertTrue(streamingResponse.hasContinuation()); + assertNotNull(streamingResponse.getContinuation()); + + final BinIndexQuery queryInfo = indexResult.getQueryInfo(); + assertEquals(bq, queryInfo); + + // Assert everything was consumed + assertFalse(streamingResponse.hasEntries()); + } + private static class IndexedPojo { @RiakKey diff --git a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java new file mode 100644 index 000000000..3f22ff3cc --- /dev/null +++ b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java @@ -0,0 +1,61 @@ +package com.basho.riak.client.core; + + +import com.basho.riak.client.api.commands.ChunkedResponseIterator; +import org.junit.Test; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TransferQueue; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.powermock.api.mockito.PowerMockito.mock; +import static org.powermock.api.mockito.PowerMockito.when; + +/** + * @author Alex Moore + */ +public class ChunkedResponseIteratorTest +{ + @Test + public void testInterruptedExceptionDealtWith() + { + boolean caught = false; + InterruptedException ie = null; + int timeout = 1000; + + try + { + @SuppressWarnings("unchecked") + TransferQueue fakeQueue = (TransferQueue)mock(TransferQueue.class); + when(fakeQueue.poll(timeout, TimeUnit.MILLISECONDS)).thenThrow(new InterruptedException("foo")); + + @SuppressWarnings("unchecked") + StreamingFutureOperation coreFuture = + (StreamingFutureOperation)mock(StreamingFutureOperation.class); + + when(coreFuture.getResultsQueue()).thenReturn(fakeQueue); + + // ChunkedResponseIterator polls the response queue when created, + // so we'll use that to simulate a Thread interrupt. + new ChunkedResponseIterator<>(coreFuture, timeout, Long::new, FakeResponse::iterator); + } + catch (RuntimeException ex) + { + caught = true; + ie = (InterruptedException)ex.getCause(); + } + catch (InterruptedException e) + { + // Mocking TransferQueue::poll(timeout) requires this CheckedException be dealt with + // If we actually catch one here we've failed at our jobs. + caught = false; + } + + assertTrue(caught); + assertEquals("foo", ie.getMessage()); + assertTrue(Thread.currentThread().isInterrupted()); + } + + static abstract class FakeResponse implements Iterable {} +} From 875999d87fa7ea47f76b1afd918dba58f5da0dab Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 25 Oct 2016 00:57:36 -0400 Subject: [PATCH 19/55] Add note on code coverage --- .../basho/riak/client/core/ChunkedResponseIteratorTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java index 3f22ff3cc..abadd6bf5 100644 --- a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java +++ b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java @@ -17,6 +17,12 @@ */ public class ChunkedResponseIteratorTest { + /* + NB: Disable this test if you want to use code coverage tools. + The Thread interrupt it generates triggers a shutdown hook race bug in Java, + which then doesn't allow EMMA to cleanly shutdown. + https://bugs.openjdk.java.net/browse/JDK-8154017 + */ @Test public void testInterruptedExceptionDealtWith() { From 3ac537cf690e974b1b94f9f34537ec0b5d4df4a1 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 25 Oct 2016 01:04:17 -0400 Subject: [PATCH 20/55] Move test to another thread, as to not upset other Interruptable tests on the main test thread --- .../core/ChunkedResponseIteratorTest.java | 70 +++++++++++-------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java index abadd6bf5..8c4f724a3 100644 --- a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java +++ b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java @@ -24,43 +24,55 @@ public class ChunkedResponseIteratorTest https://bugs.openjdk.java.net/browse/JDK-8154017 */ @Test - public void testInterruptedExceptionDealtWith() + public void testInterruptedExceptionDealtWith() throws InterruptedException { - boolean caught = false; - InterruptedException ie = null; + final boolean[] caught = {false}; + final InterruptedException[] ie = {null}; int timeout = 1000; - try + Thread t = new Thread(() -> { - @SuppressWarnings("unchecked") - TransferQueue fakeQueue = (TransferQueue)mock(TransferQueue.class); - when(fakeQueue.poll(timeout, TimeUnit.MILLISECONDS)).thenThrow(new InterruptedException("foo")); + try + { + @SuppressWarnings("unchecked") TransferQueue fakeQueue = + (TransferQueue) mock(TransferQueue.class); + when(fakeQueue.poll(timeout, + TimeUnit.MILLISECONDS)).thenThrow(new InterruptedException( + "foo")); - @SuppressWarnings("unchecked") - StreamingFutureOperation coreFuture = - (StreamingFutureOperation)mock(StreamingFutureOperation.class); + @SuppressWarnings("unchecked") StreamingFutureOperation + coreFuture = (StreamingFutureOperation) mock( + StreamingFutureOperation.class); - when(coreFuture.getResultsQueue()).thenReturn(fakeQueue); + when(coreFuture.getResultsQueue()).thenReturn(fakeQueue); - // ChunkedResponseIterator polls the response queue when created, - // so we'll use that to simulate a Thread interrupt. - new ChunkedResponseIterator<>(coreFuture, timeout, Long::new, FakeResponse::iterator); - } - catch (RuntimeException ex) - { - caught = true; - ie = (InterruptedException)ex.getCause(); - } - catch (InterruptedException e) - { - // Mocking TransferQueue::poll(timeout) requires this CheckedException be dealt with - // If we actually catch one here we've failed at our jobs. - caught = false; - } + // ChunkedResponseIterator polls the response queue when created, + // so we'll use that to simulate a Thread interrupt. + new ChunkedResponseIterator<>(coreFuture, + timeout, + Long::new, + FakeResponse::iterator); + } + catch (RuntimeException ex) + { + caught[0] = true; + ie[0] = (InterruptedException) ex.getCause(); + } + catch (InterruptedException e) + { + // Mocking TransferQueue::poll(timeout) requires this CheckedException be dealt with + // If we actually catch one here we've failed at our jobs. + caught[0] = false; + } + + assertTrue(Thread.currentThread().isInterrupted()); + }); + + t.start(); + t.join(); - assertTrue(caught); - assertEquals("foo", ie.getMessage()); - assertTrue(Thread.currentThread().isInterrupted()); + assertTrue(caught[0]); + assertEquals("foo", ie[0].getMessage()); } static abstract class FakeResponse implements Iterable {} From 051565d1bda11d071ad75ed99e0dbe13d7d03e66 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 25 Oct 2016 01:16:09 -0400 Subject: [PATCH 21/55] Fix flappy test --- .../client/api/commands/buckets/itest/ITestListBuckets.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java index 9978ac3f9..18e312893 100644 --- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java +++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java @@ -118,7 +118,7 @@ private void testListBucketsStreaming(Namespace namespace) throws InterruptedExc assumeTrue(iterator.hasNext()); boolean found = false; - while (!found && iterator.hasNext()) + while (iterator.hasNext()) { final Namespace next = iterator.next(); found = next.getBucketName().toString().equals(bucketName); From 7a1ccde85a4775b8f2b178604d0c182f8c996899 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 25 Oct 2016 15:00:10 -0400 Subject: [PATCH 22/55] Add javadocs on how to use Streaming interfaces. --- .../api/commands/buckets/ListBuckets.java | 19 +++++++++++ .../commands/indexes/BigIntIndexQuery.java | 32 +++++++++++++++++-- .../api/commands/indexes/BinIndexQuery.java | 26 +++++++++++++++ .../commands/indexes/BucketIndexQuery.java | 27 ++++++++++++++-- .../api/commands/indexes/IntIndexQuery.java | 27 ++++++++++++++++ .../api/commands/indexes/KeyIndexQuery.java | 25 +++++++++++++++ .../api/commands/indexes/RawIndexQuery.java | 28 ++++++++++++++++ .../api/commands/kv/FullBucketRead.java | 1 + .../riak/client/api/commands/kv/ListKeys.java | 21 ++++++++++++ 9 files changed, 201 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java index 56648c62f..865e749f3 100644 --- a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java +++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java @@ -42,7 +42,26 @@ * System.out.println(ns.getBucketName()); * }} *

+ *

+ * You can also stream the results back before the operation is fully complete. + * This reduces the time between executing the operation and seeing a result, + * and reduces overall memory usage if the iterator is consumed quickly enough. + * The result iterable can only be iterated once though. + * If the thread is interrupted while the iterator is polling for more results, + * a {@link RuntimeException} will be thrown. + * 

+ * {@code
+ * ListBuckets lb = new ListBuckets.Builder("my_type").build();
+ * final RiakFuture streamFuture =
+ *     client.executeAsyncStreaming(lb, 200);
+ * final ListBuckets.StreamingResponse streamingResponse = streamFuture.get();
+ * for (Namespace ns : streamingResponse)
+ * {
+ *     System.out.println(ns.getBucketName());
+ * }}
+ *

* @author Dave Rusek + * @author Alex Moore * @since 2.0 */ public final class ListBuckets extends StreamableRiakCommand diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java index cdaf25138..aa17b0061 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java @@ -17,18 +17,17 @@ package com.basho.riak.client.api.commands.indexes; import com.basho.riak.client.api.commands.ChunkedResponseIterator; -import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter; +import com.basho.riak.client.api.commands.CoreFutureAdapter; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; import com.basho.riak.client.core.StreamingRiakFuture; import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation; -import com.basho.riak.client.api.commands.CoreFutureAdapter; import com.basho.riak.client.core.query.Location; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.util.BinaryValue; + import java.math.BigInteger; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; /** @@ -46,6 +45,33 @@ * BigInteger key = new BigInteger(someReallyLongNumber); * BigIntIndexQuery q = new BigIntIndexQuery.Builder(ns, "my_index", key).build(); * BigIntIndexQuery.Response resp = client.execute(q);} + * + *

+ * You can also stream the results back before the operation is fully complete. + * This reduces the time between executing the operation and seeing a result, + * and reduces overall memory usage if the iterator is consumed quickly enough. + * The result iterable can only be iterated once though. + * If the thread is interrupted while the iterator is polling for more results, + * a {@link RuntimeException} will be thrown. + * 

+ * {@code
+ * Namespace ns = new Namespace("my_type", "my_bucket");
+ * BigInteger key = new BigInteger(someReallyLongNumber);
+ * BigIntIndexQuery q = new BigIntIndexQuery.Builder(ns, "my_index", key).build();
+ * RiakFuture streamingFuture =
+ *     client.executeAsyncStreaming(q, 200);
+ * BigIntIndexQuery.StreamingResponse streamingResponse = streamingFuture.get();
+ *
+ * for (BigIntIndexQuery.Response.Entry e : streamingResponse)
+ * {
+ *     System.out.println(e.getRiakObjectLocation().getKey().toString());
+ * }
+ * // Wait for the command to fully finish.
+ * streamingFuture.await();
+ * // The StreamingResponse will also contain the continuation, if the operation returned one.
+ * streamingResponse.getContinuation(); }
+ *

+ * * @author Brian Roach * @author Alex Moore * @since 2.0 diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java index e3e4d88cd..c0a119e00 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java @@ -46,6 +46,32 @@ * BinIndexQuery q = new BinIndexQuery.Builder(ns, "my_index", key).build(); * BinIndexQuery.Response resp = client.execute(q);} * + *

+ * You can also stream the results back before the operation is fully complete. + * This reduces the time between executing the operation and seeing a result, + * and reduces overall memory usage if the iterator is consumed quickly enough. + * The result iterable can only be iterated once though. + * If the thread is interrupted while the iterator is polling for more results, + * a {@link RuntimeException} will be thrown. + * 

+ * {@code
+ * Namespace ns = new Namespace("my_type", "my_bucket");
+ * String key = "some_key";
+ * BinIndexQuery q = new BinIndexQuery.Builder(ns, "my_index", key).build();
+ * RiakFuture streamingFuture =
+ *     client.executeAsyncStreaming(q, 200);
+ * BinIndexQuery.StreamingResponse streamingResponse = streamingFuture.get();
+ *
+ * for (BinIndexQuery.Response.Entry e : streamingResponse)
+ * {
+ *     System.out.println(e.getRiakObjectLocation().getKey().toString());
+ * }
+ * // Wait for the command to fully finish.
+ * streamingFuture.await();
+ * // The StreamingResponse will also contain the continuation, if the operation returned one.
+ * streamingResponse.getContinuation(); }
+ *

+ * * @author Brian Roach * @author Alex Moore * @since 2.0 diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BucketIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BucketIndexQuery.java index 730209403..3ddce50ea 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/BucketIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BucketIndexQuery.java @@ -1,9 +1,7 @@ package com.basho.riak.client.api.commands.indexes; -import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation; import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.query.indexes.IndexNames; -import com.basho.riak.client.core.util.BinaryValue; /** * Performs a 2i query across the special $bucket index, for a known bucket, and returns the keys in that bucket. @@ -17,6 +15,31 @@ * BucketIndexQuery q = new BucketIndexQuery.Builder(ns).build(); * RawIndexquery.Response resp = client.execute(q);} * + *

+ * You can also stream the results back before the operation is fully complete. + * This reduces the time between executing the operation and seeing a result, + * and reduces overall memory usage if the iterator is consumed quickly enough. + * The result iterable can only be iterated once though. + * If the thread is interrupted while the iterator is polling for more results, + * a {@link RuntimeException} will be thrown. + * 

+ * {@code
+ * Namespace ns = new Namespace("my_type", "my_bucket");
+ * BucketIndexQuery q = new BucketIndexQuery.Builder(ns).build();
+ * RiakFuture streamingFuture =
+ *     client.executeAsyncStreaming(q, 200);
+ * BinIndexQuery.StreamingResponse streamingResponse = streamingFuture.get();
+ *
+ * for (BinIndexQuery.Response.Entry e : streamingResponse)
+ * {
+ *     System.out.println(e.getRiakObjectLocation().getKey().toString());
+ * }
+ * // Wait for the command to fully finish.
+ * streamingFuture.await();
+ * // The StreamingResponse will also contain the continuation, if the operation returned one.
+ * streamingResponse.getContinuation(); }
+ *

+ * * @author Alex Moore * @since 2.0.7 */ diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java index 28de516ff..e6af40452 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java @@ -41,6 +41,33 @@ * long key = 1234L; * IntIndexQuery q = new IntIndexQuery.Builder(ns, "my_index", key).build(); * IntIndexQuery.Response resp = client.execute(q);} + * + *

+ * You can also stream the results back before the operation is fully complete. + * This reduces the time between executing the operation and seeing a result, + * and reduces overall memory usage if the iterator is consumed quickly enough. + * The result iterable can only be iterated once though. + * If the thread is interrupted while the iterator is polling for more results, + * a {@link RuntimeException} will be thrown. + * 

+ * {@code
+ * Namespace ns = new Namespace("my_type", "my_bucket");
+ * long key = 1234L;
+ * IntIndexQuery q = new IntIndexQuery.Builder(ns, "my_index", key).build();
+ * RiakFuture streamingFuture =
+ *     client.executeAsyncStreaming(q, 200);
+ * IntIndexQuery.StreamingResponse streamingResponse = streamingFuture.get();
+ *
+ * for (IntIndexQuery.Response.Entry e : streamingResponse)
+ * {
+ *     System.out.println(e.getRiakObjectLocation().getKey().toString());
+ * }
+ * // Wait for the command to fully finish.
+ * streamingFuture.await();
+ * // The StreamingResponse will also contain the continuation, if the operation returned one.
+ * streamingResponse.getContinuation(); }
+ *

+ * * @author Brian Roach * @author Alex Moore * @since 2.0 diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/KeyIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/KeyIndexQuery.java index bfebfd24a..255c8bc67 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/KeyIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/KeyIndexQuery.java @@ -19,6 +19,31 @@ * KeyIndexQuery q = new KeyIndexQuery.Builder(ns, "foo10", "foo19").build(); * RawIndexquery.Response resp = client.execute(q);} * + *

+ * You can also stream the results back before the operation is fully complete. + * This reduces the time between executing the operation and seeing a result, + * and reduces overall memory usage if the iterator is consumed quickly enough. + * The result iterable can only be iterated once though. + * If the thread is interrupted while the iterator is polling for more results, + * a {@link RuntimeException} will be thrown. + * 

+ * {@code
+ * Namespace ns = new Namespace("my_type", "my_bucket");
+ * KeyIndexQuery q = new KeyIndexQuery.Builder(ns, "foo10", "foo19").build();
+ * RiakFuture streamingFuture =
+ *     client.executeAsyncStreaming(q, 200);
+ * RawIndexQuery.StreamingResponse streamingResponse = streamingFuture.get();
+ *
+ * for (BinIndexQuery.Response.Entry e : streamingResponse)
+ * {
+ *     System.out.println(e.getRiakObjectLocation().getKeyAsString());
+ * }
+ * // Wait for the command to fully finish.
+ * streamingFuture.await();
+ * // The StreamingResponse will also contain the continuation, if the operation returned one.
+ * streamingResponse.getContinuation(); }
+ *

+ * * @author Alex Moore * @since 2.0.7 */ diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java index 4e88de13f..1f92b6049 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java @@ -42,6 +42,34 @@ * Namespace ns = new Namespace("my_type", "my_bucket"); * RawIndexQuery q = new RawIndexQuery.Builder(ns, "my_index", Type._BIN, key).build(); * RawIndexquery.Response resp = client.execute(q);} + * + *

+ * You can also stream the results back before the operation is fully complete. + * This reduces the time between executing the operation and seeing a result, + * and reduces overall memory usage if the iterator is consumed quickly enough. + * The result iterable can only be iterated once though. + * If the thread is interrupted while the iterator is polling for more results, + * a {@link RuntimeException} will be thrown. + * 

+ * {@code
+ * byte[] bytes = new byte[] { 1,2,3,4};
+ * BinaryValue key = BinaryValue.create(bytes);
+ * Namespace ns = new Namespace("my_type", "my_bucket");
+ * RawIndexQuery q = new RawIndexQuery.Builder(ns, "my_index", Type._BIN, key).build();
+ * RiakFuture streamingFuture =
+ *     client.executeAsyncStreaming(q, 200);
+ * RawIndexQuery.StreamingResponse streamingResponse = streamingFuture.get();
+ *
+ * for (RawIndexQuery.Response.Entry e : streamingResponse)
+ * {
+ *     System.out.println(e.getRiakObjectLocation().getKey().toString());
+ * }
+ * // Wait for the command to fully finish.
+ * streamingFuture.await();
+ * // The StreamingResponse will also contain the continuation, if the operation returned one.
+ * streamingResponse.getContinuation(); }
+ *

+ * * @author Brian Roach * @author Alex Moore * @since 2.0 diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java b/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java index f9bbf53c0..7a16d4165 100644 --- a/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java +++ b/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java @@ -50,6 +50,7 @@ * for querying buckets that contain a big amount of data. * * @author Sergey Galkin + * @author Alex Moore * @see CoveragePlan */ public class FullBucketRead extends SecondaryIndexQuery diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java index b0d69e4d1..467c452ff 100644 --- a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java +++ b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java @@ -47,10 +47,31 @@ * }} *

*

+ * You can also stream the results back before the operation is fully complete. + * This reduces the time between executing the operation and seeing a result, + * and reduces overall memory usage if the iterator is consumed quickly enough. + * The result iterable can only be iterated once though. + * If the thread is interrupted while the iterator is polling for more results, + * a {@link RuntimeException} will be thrown. + * 

+ * {@code
+ * Namespace ns = new Namespace("my_type", "my_bucket");
+ * ListKeys lk = new ListKeys.Builder(ns).build();
+ * RiakFuture streamFuture =
+ *     client.executeAsyncStreaming(lk, 200);
+ * final ListKeys.StreamingResponse streamingResponse = streamFuture.get();
+ * ListKeys.Response response = client.execute(lk);
+ * for (Location l : streamingResponse)
+ * {
+ *     System.out.println(l.getKeyAsString());
+ * }}
+ *

+ *

* This is a very expensive operation and is not recommended for use on a production system *

* * @author Dave Rusek + * @author Alex Moore * @since 2.0 */ public final class ListKeys extends StreamableRiakCommand From 5b9f6b4cb1c62130dfa7be6d4e7cd8f3efe21d17 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 25 Oct 2016 15:00:35 -0400 Subject: [PATCH 23/55] Fix misc javadocs warnings and errors --- src/main/java/com/basho/riak/client/api/RiakClient.java | 5 +++-- .../client/api/commands/indexes/SecondaryIndexQuery.java | 2 +- .../com/basho/riak/client/api/commands/kv/KvBuilderBase.java | 1 - .../com/basho/riak/client/api/commands/kv/MultiDelete.java | 2 +- src/main/java/com/basho/riak/client/core/RiakNode.java | 4 ++-- .../client/core/query/timeseries/FullColumnDescription.java | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/RiakClient.java b/src/main/java/com/basho/riak/client/api/RiakClient.java index 4e922979c..6bec3fae2 100644 --- a/src/main/java/com/basho/riak/client/api/RiakClient.java +++ b/src/main/java/com/basho/riak/client/api/RiakClient.java @@ -277,8 +277,9 @@ public static RiakClient newClient(InetSocketAddress... addresses) throws Unknow * @return a new RiakClient instance. * @throws java.net.UnknownHostException if a supplied hostname cannot be resolved. * @since 2.0.3 - * @see com.basho.riak.client.core.RiakCluster.Builder#Builder(RiakNode.Builder, List) + * @see com.basho.riak.client.core.RiakCluster.Builder#RiakCluster.Builder(RiakNode.Builder, List) */ + // NB: IntelliJ will see the above @see statement as invalid, but it's correct: https://bugs.openjdk.java.net/browse/JDK-8031625 public static RiakClient newClient(RiakNode.Builder nodeBuilder, List addresses) throws UnknownHostException { final RiakCluster cluster = new RiakCluster.Builder(nodeBuilder, addresses).build(); @@ -408,7 +409,7 @@ public RiakFuture executeAsync(RiakCommand command) * Calling this method causes the client to execute the provided StreamableRiakCommand * asynchronously. It will immediately return a RiakFuture that contains an immediately available result that data * will be streamed to. The RiakFuture will also keep track of the overall operation's progress with the - * {@see RiakFuture#isDone}, etc methods. + * {@link RiakFuture#isDone}, etc methods. * * @param StreamableRiakCommand's immediate return type, available before the command/operation is complete. * @param The RiakCommand's query info type. diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java index abb1b8ffa..8c82e3969 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java @@ -545,7 +545,7 @@ public T withCoverageContext(byte[] coverageContext) * * It has protected access since, due to performance reasons, it might be used only for the Full Bucket Read * @param returnBody - * @return + * @return a reference to this object. */ protected T withReturnBody(boolean returnBody) { diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/KvBuilderBase.java b/src/main/java/com/basho/riak/client/api/commands/kv/KvBuilderBase.java index 1cbd4d099..0bb7a84d9 100644 --- a/src/main/java/com/basho/riak/client/api/commands/kv/KvBuilderBase.java +++ b/src/main/java/com/basho/riak/client/api/commands/kv/KvBuilderBase.java @@ -27,7 +27,6 @@ protected KvBuilderBase(Location location) * * @param option the option * @param value the value for the option - * @return a reference to this object. */ protected void addOption(RiakOption option, Object value) { diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/MultiDelete.java b/src/main/java/com/basho/riak/client/api/commands/kv/MultiDelete.java index d9690b86e..83f525c97 100644 --- a/src/main/java/com/basho/riak/client/api/commands/kv/MultiDelete.java +++ b/src/main/java/com/basho/riak/client/api/commands/kv/MultiDelete.java @@ -21,7 +21,7 @@ * 
  * {@code
  * MultiDelete multiDelete = ...;
- * MultiDelete.Response response = client.execute(multiDelete);
+ * MultiDelete.Response response = client.execute(multiDelete);}
  * 

  * 

  * The maximum number of concurrent requests defaults to 10. This can be changed
diff --git a/src/main/java/com/basho/riak/client/core/RiakNode.java b/src/main/java/com/basho/riak/client/core/RiakNode.java
index 7a2e009c2..e40d3c7e2 100644
--- a/src/main/java/com/basho/riak/client/core/RiakNode.java
+++ b/src/main/java/com/basho/riak/client/core/RiakNode.java
@@ -1603,7 +1603,7 @@ public static List buildNodes(Builder builder, List remoteAddr
          * @param builder         a configured builder, used for common properties among the nodes
          *
          * @return a list of constructed RiakNodes
-         * @@since 2.0.6
+         * @since 2.0.6
          */
         public static List buildNodes(Collection remoteHosts, Builder builder)
         {
@@ -1630,8 +1630,8 @@ public static List buildNodes(Collection remoteHosts, Bui
          * Build a set of RiakNodes.
          * The provided builder will be used to construct a set of RiakNodes using the supplied addresses.
          *
-         * @see #buildNodes(Builder, List)
          * @since 2.0.3
+         * @see #buildNodes(RiakNode.Builder, List)
          */
         public static List buildNodes(Builder builder, String... remoteAddresses)
                 throws UnknownHostException
diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/FullColumnDescription.java b/src/main/java/com/basho/riak/client/core/query/timeseries/FullColumnDescription.java
index 00ce5edf5..b56b1f794 100644
--- a/src/main/java/com/basho/riak/client/core/query/timeseries/FullColumnDescription.java
+++ b/src/main/java/com/basho/riak/client/core/query/timeseries/FullColumnDescription.java
@@ -55,7 +55,7 @@ public FullColumnDescription(String name,
      * @param quantum The {@link Quantum} setting if this column
      *                is used in partition key time quantization.
      *                Use
-     *                {@link FullColumnDescription#FullColumnDescription(String, ColumnType, boolean, Integer)}
+     *                {@link #FullColumnDescription(String, ColumnDescription.ColumnType, boolean, Integer)}
      *                if the quantum is not needed. 
      * @throws IllegalArgumentException if the Column Name or Column Type are null or empty,
      *                                  or if the quantum is set on a non-Timestamp column,
@@ -108,7 +108,7 @@ public FullColumnDescription(String name,
      * @param quantum The {@link Quantum} setting if this column
      *                is used in partition key time quantization.
      *                Use
-     *                {@link FullColumnDescription#FullColumnDescription(String, ColumnType, boolean, Integer, Integer)}
+     *                {@link #FullColumnDescription(String, ColumnDescription.ColumnType, boolean, Integer, Integer)}
      *                if the quantum is not needed. 
      * @throws IllegalArgumentException if the Column Name or Column Type are null or empty,
      *                                  or if the quantum is set on a non-Timestamp column,

From 511f95ea29e2aeedb7a9e841d12123a7dd8ea1b7 Mon Sep 17 00:00:00 2001
From: Alex Moore 
Date: Tue, 25 Oct 2016 15:33:50 -0400
Subject: [PATCH 24/55] Fix another minor test bug

---
 .../commands/buckets/itest/ITestListBuckets.java   | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java
index 18e312893..0d3621f84 100644
--- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java
+++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java
@@ -114,20 +114,24 @@ private void testListBucketsStreaming(Namespace namespace) throws InterruptedExc
         final RiakFuture streamingFuture =
                 client.executeAsyncStreaming(listBucketsCommand, 500);
 
-        Iterator iterator = streamingFuture.get().iterator();
+        final ListBuckets.StreamingResponse streamResponse = streamingFuture.get();
+        final Iterator iterator = streamResponse.iterator();
+
         assumeTrue(iterator.hasNext());
         boolean found = false;
 
-        while (iterator.hasNext())
+        for (Namespace ns : streamResponse)
         {
-            final Namespace next = iterator.next();
-            found = next.getBucketName().toString().equals(bucketName);
+            if(!found)
+            {
+                found = ns.getBucketName().toString().equals(bucketName);
+            }
         }
 
         streamingFuture.await(); // Wait for command to finish, even if we've found our data
         assumeTrue(streamingFuture.isDone());
 
-        assertFalse(streamingFuture.get().iterator().hasNext());
+        assertFalse(iterator.hasNext());
         assertEquals(namespace.getBucketType(), streamingFuture.getQueryInfo());
 
         assertTrue(found);

From a54aa45d29887eae7afc32c907be395f652013e9 Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Mon, 24 Oct 2016 16:50:52 +0300
Subject: [PATCH 25/55] Generalize StreamingFutureOperation.

---
 .../client/core/StreamingFutureOperation.java    | 15 +++++++++++++--
 .../core/operations/ListBucketsOperation.java    | 14 ++------------
 .../core/operations/ListKeysOperation.java       | 16 ++--------------
 .../core/operations/MapReduceOperation.java      | 16 ++--------------
 .../operations/SecondaryIndexQueryOperation.java | 14 ++------------
 5 files changed, 21 insertions(+), 54 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java b/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java
index cac8604f7..4b3b98672 100644
--- a/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java
+++ b/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java
@@ -16,6 +16,9 @@
 
 package com.basho.riak.client.core;
 
+import java.util.concurrent.LinkedTransferQueue;
+import java.util.concurrent.TransferQueue;
+
 /**
  * @author Alex Moore 
  * @param  The type returned by the streaming and non-streaming operation versions
@@ -27,11 +30,13 @@ public abstract class StreamingFutureOperation
         implements StreamingRiakFuture
 {
+    private final TransferQueue responseQueue;
     private boolean streamResults;
 
     protected StreamingFutureOperation(boolean streamResults)
     {
         this.streamResults = streamResults;
+        this.responseQueue = new LinkedTransferQueue<>();
     }
 
     @Override
@@ -43,8 +48,14 @@ protected void processMessage(ResponseType decodedMessage)
             return;
         }
 
-        processStreamingChunk(decodedMessage);
+        final ReturnType r = processStreamingChunk(decodedMessage);
+        responseQueue.offer(r);
     }
 
-    abstract protected void processStreamingChunk(ResponseType rawResponseChunk);
+    abstract protected ReturnType processStreamingChunk(ResponseType rawResponseChunk);
+
+    public final TransferQueue getResultsQueue()
+    {
+        return this.responseQueue;
+    }
 }
diff --git a/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java
index 5297af5eb..54536e4a1 100644
--- a/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java
+++ b/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java
@@ -27,8 +27,6 @@
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.LinkedTransferQueue;
-import java.util.concurrent.TransferQueue;
 
 public class ListBucketsOperation extends StreamingFutureOperation responseQueue;
 
     private ListBucketsOperation(Builder builder)
     {
         super(builder.streamResults);
         this.reqBuilder = builder.reqBuilder;
         this.bucketType = builder.bucketType;
-        this.responseQueue = new LinkedTransferQueue<>();
     }
 
     @Override
@@ -76,10 +72,10 @@ private List convertSingleResponse(RiakKvPB.RpbListBucketsResp resp
     }
 
     @Override
-    protected void processStreamingChunk(RiakKvPB.RpbListBucketsResp rawResponseChunk)
+    protected Response processStreamingChunk(RiakKvPB.RpbListBucketsResp rawResponseChunk)
     {
         final List buckets = convertSingleResponse(rawResponseChunk);
-        this.responseQueue.add(new Response(bucketType, buckets));
+        return new Response(bucketType, buckets);
     }
 
     @Override
@@ -108,12 +104,6 @@ public BinaryValue getQueryInfo()
         return bucketType;
     }
 
-    @Override
-    public TransferQueue getResultsQueue()
-    {
-        return this.responseQueue;
-    }
-
     public static class Builder
     {
         private final RiakKvPB.RpbListBucketsReq.Builder reqBuilder =
diff --git a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java
index d1bd57f11..dbfa7b528 100644
--- a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java
+++ b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java
@@ -23,20 +23,15 @@
 import com.basho.riak.protobuf.RiakMessageCodes;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.LinkedTransferQueue;
-import java.util.concurrent.TransferQueue;
 
 public class ListKeysOperation extends StreamingFutureOperation
 {
     private final Namespace namespace;
     private final RiakKvPB.RpbListKeysReq.Builder reqBuilder;
-    private final LinkedTransferQueue responseQueue;
 
 
     private ListKeysOperation(Builder builder)
@@ -44,7 +39,6 @@ private ListKeysOperation(Builder builder)
         super(builder.streamResults);
         this.reqBuilder = builder.reqBuilder;
         this.namespace = builder.namespace;
-        this.responseQueue = new LinkedTransferQueue<>();
     }
 
     @Override
@@ -103,15 +97,9 @@ public Namespace getQueryInfo()
     }
 
     @Override
-    protected void processStreamingChunk(RiakKvPB.RpbListKeysResp rawResponseChunk)
+    protected Response processStreamingChunk(RiakKvPB.RpbListKeysResp rawResponseChunk)
     {
-        this.responseQueue.add(new Response.Builder().addKeys(convertSingleResponse(rawResponseChunk)).build());
-    }
-
-    @Override
-    public TransferQueue getResultsQueue()
-    {
-        return this.responseQueue;
+        return new Response.Builder().addKeys(convertSingleResponse(rawResponseChunk)).build();
     }
 
     public static class Builder
diff --git a/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java b/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java
index 7b7d99090..046410764 100644
--- a/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java
+++ b/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java
@@ -31,8 +31,6 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.LinkedTransferQueue;
-import java.util.concurrent.TransferQueue;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -46,7 +44,6 @@ public class MapReduceOperation extends StreamingFutureOperation responseQueue;
     private final ObjectMapper objectMapper = new ObjectMapper();
     private final JsonNodeFactory jsonNodeFactory = JsonNodeFactory.instance;
 
@@ -56,7 +53,6 @@ private MapReduceOperation(Builder builder)
         super(builder.streamResults);
         this.reqBuilder = builder.reqBuilder;
         this.mapReduce = builder.mapReduce;
-        this.responseQueue = new LinkedTransferQueue<>();
     }
 
     @Override
@@ -152,20 +148,12 @@ public BinaryValue getQueryInfo()
     }
 
     @Override
-    protected void processStreamingChunk(RiakKvPB.RpbMapRedResp rawResponseChunk)
+    protected Response processStreamingChunk(RiakKvPB.RpbMapRedResp rawResponseChunk)
     {
         final Map resultMap = new LinkedHashMap<>();
 
         convertSingleResponse(resultMap, rawResponseChunk);
-
-        final Response chunkResponse = new Response(resultMap);
-        this.responseQueue.offer(chunkResponse);
-    }
-
-    @Override
-    public TransferQueue getResultsQueue()
-    {
-        return this.responseQueue;
+        return new Response(resultMap);
     }
 
     public static class Builder
diff --git a/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java b/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java
index 82aca503b..0395cb588 100644
--- a/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java
+++ b/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java
@@ -30,8 +30,6 @@
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.LinkedTransferQueue;
-import java.util.concurrent.TransferQueue;
 
 /**
  *
@@ -45,7 +43,6 @@ public class SecondaryIndexQueryOperation
 {
     private final RiakKvPB.RpbIndexReq pbReq;
     private final Query query;
-    private final TransferQueue responseQueue;
 
     private SecondaryIndexQueryOperation(Builder builder)
     {
@@ -57,7 +54,6 @@ private SecondaryIndexQueryOperation(Builder builder)
         builder.pbReqBuilder.setStream(true);
         this.query = builder.query;
         this.pbReq = builder.pbReqBuilder.build();
-        this.responseQueue = new LinkedTransferQueue<>();
     }
 
     @Override
@@ -232,7 +228,7 @@ public Query getQueryInfo()
     }
 
     @Override
-    protected void processStreamingChunk(Object rawResponseChunk)
+    protected Response processStreamingChunk(Object rawResponseChunk)
     {
         SecondaryIndexQueryOperation.Response.Builder responseBuilder =
                 new SecondaryIndexQueryOperation.Response.Builder();
@@ -254,13 +250,7 @@ protected void processStreamingChunk(Object rawResponseChunk)
             processBatchMessage(continuationOnlyResponse);
         }
 
-        this.responseQueue.add(response);
-    }
-
-    @Override
-    public TransferQueue getResultsQueue()
-    {
-        return this.responseQueue;
+        return response;
     }
 
     /**

From 3dd8230a3b0e3b487d0e092e8fd94484c04ddf91 Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Wed, 26 Oct 2016 12:33:09 +0300
Subject: [PATCH 26/55] Introduce PBStreamingFutureOperation.

---
 .../commands/indexes/SecondaryIndexQuery.java |  1 +
 ...n.java => PBStreamingFutureOperation.java} | 15 ++++++--
 .../basho/riak/client/core/RiakCluster.java   |  2 +-
 .../core/operations/ListBucketsOperation.java | 35 +++++--------------
 .../core/operations/ListKeysOperation.java    | 35 +++++--------------
 .../core/operations/MapReduceOperation.java   | 35 ++++---------------
 .../SecondaryIndexQueryOperation.java         | 20 +++++------
 .../api/commands/buckets/ListBucketsTest.java |  2 +-
 .../core/ChunkedResponseIteratorTest.java     |  6 ++--
 9 files changed, 51 insertions(+), 100 deletions(-)
 rename src/main/java/com/basho/riak/client/core/{StreamingFutureOperation.java => PBStreamingFutureOperation.java} (69%)

diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
index 8c82e3969..51fd339b9 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
@@ -19,6 +19,7 @@
 import com.basho.riak.client.api.RiakCommand;
 import com.basho.riak.client.api.StreamableRiakCommand;
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
+import com.basho.riak.client.core.PBStreamingFutureOperation;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.StreamingRiakFuture;
diff --git a/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java b/src/main/java/com/basho/riak/client/core/PBStreamingFutureOperation.java
similarity index 69%
rename from src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java
rename to src/main/java/com/basho/riak/client/core/PBStreamingFutureOperation.java
index 4b3b98672..f00d77295 100644
--- a/src/main/java/com/basho/riak/client/core/StreamingFutureOperation.java
+++ b/src/main/java/com/basho/riak/client/core/PBStreamingFutureOperation.java
@@ -16,6 +16,10 @@
 
 package com.basho.riak.client.core;
 
+import com.basho.riak.client.core.operations.PBFutureOperation;
+import com.basho.riak.protobuf.RiakKvPB;
+import com.google.protobuf.GeneratedMessage;
+
 import java.util.concurrent.LinkedTransferQueue;
 import java.util.concurrent.TransferQueue;
 
@@ -26,15 +30,20 @@
  * @param  Query info type
  * @since 2.1.0
  */
-public abstract class StreamingFutureOperation
-        extends FutureOperation
+public abstract class PBStreamingFutureOperation
+        extends PBFutureOperation
         implements StreamingRiakFuture
 {
     private final TransferQueue responseQueue;
     private boolean streamResults;
 
-    protected StreamingFutureOperation(boolean streamResults)
+    protected PBStreamingFutureOperation(final byte reqMessageCode,
+                                         final byte respMessageCode,
+                                         final GeneratedMessage.Builder reqBuilder,
+                                         com.google.protobuf.Parser respParser,
+                                         boolean streamResults)
     {
+        super(reqMessageCode, respMessageCode, reqBuilder, respParser);
         this.streamResults = streamResults;
         this.responseQueue = new LinkedTransferQueue<>();
     }
diff --git a/src/main/java/com/basho/riak/client/core/RiakCluster.java b/src/main/java/com/basho/riak/client/core/RiakCluster.java
index 93e1640a0..772a7684b 100644
--- a/src/main/java/com/basho/riak/client/core/RiakCluster.java
+++ b/src/main/java/com/basho/riak/client/core/RiakCluster.java
@@ -224,7 +224,7 @@ public  RiakFuture execute(FutureOperation operation)
         return executeFutureOperation(operation);
     }
 
-    public  StreamingRiakFuture execute(StreamingFutureOperation operation)
+    public  StreamingRiakFuture execute(PBStreamingFutureOperation operation)
     {
         executeFutureOperation(operation);
         return operation;
diff --git a/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java
index 54536e4a1..e3b922fec 100644
--- a/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java
+++ b/src/main/java/com/basho/riak/client/core/operations/ListBucketsOperation.java
@@ -15,30 +15,31 @@
  */
 package com.basho.riak.client.core.operations;
 
-import com.basho.riak.client.core.RiakMessage;
-import com.basho.riak.client.core.StreamingFutureOperation;
+import com.basho.riak.client.core.PBStreamingFutureOperation;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
 import com.basho.riak.protobuf.RiakKvPB;
 import com.basho.riak.protobuf.RiakMessageCodes;
 import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
 
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-public class ListBucketsOperation extends StreamingFutureOperation
 {
-    private final RiakKvPB.RpbListBucketsReq.Builder reqBuilder;
     private final BinaryValue bucketType;
 
     private ListBucketsOperation(Builder builder)
     {
-        super(builder.streamResults);
-        this.reqBuilder = builder.reqBuilder;
+        super(RiakMessageCodes.MSG_ListBucketsReq,
+                RiakMessageCodes.MSG_ListBucketsResp,
+                builder.reqBuilder,
+                RiakKvPB.RpbListBucketsResp.PARSER,
+                builder.streamResults);
+
         this.bucketType = builder.bucketType;
     }
 
@@ -78,26 +79,6 @@ protected Response processStreamingChunk(RiakKvPB.RpbListBucketsResp rawResponse
         return new Response(bucketType, buckets);
     }
 
-    @Override
-    protected RiakMessage createChannelMessage()
-    {
-        return new RiakMessage(RiakMessageCodes.MSG_ListBucketsReq, reqBuilder.build().toByteArray());
-    }
-
-    @Override
-    protected RiakKvPB.RpbListBucketsResp decode(RiakMessage rawMessage)
-    {
-        try
-        {
-            Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_ListBucketsResp);
-            return RiakKvPB.RpbListBucketsResp.parseFrom(rawMessage.getData());
-        }
-        catch (InvalidProtocolBufferException e)
-        {
-            throw new IllegalArgumentException("Invalid message received", e);
-        }
-    }
-
     @Override
     public BinaryValue getQueryInfo()
     {
diff --git a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java
index dbfa7b528..edbd12cbc 100644
--- a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java
+++ b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java
@@ -15,29 +15,30 @@
  */
 package com.basho.riak.client.core.operations;
 
-import com.basho.riak.client.core.RiakMessage;
-import com.basho.riak.client.core.StreamingFutureOperation;
+import com.basho.riak.client.core.PBStreamingFutureOperation;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
 import com.basho.riak.protobuf.RiakKvPB;
 import com.basho.riak.protobuf.RiakMessageCodes;
 import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
 
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-public class ListKeysOperation extends StreamingFutureOperation
+public class ListKeysOperation extends PBStreamingFutureOperation
 {
     private final Namespace namespace;
-    private final RiakKvPB.RpbListKeysReq.Builder reqBuilder;
 
 
     private ListKeysOperation(Builder builder)
     {
-        super(builder.streamResults);
-        this.reqBuilder = builder.reqBuilder;
+        super(RiakMessageCodes.MSG_ListKeysReq,
+                RiakMessageCodes.MSG_ListKeysResp,
+                builder.reqBuilder,
+                RiakKvPB.RpbListKeysResp.PARSER,
+                builder.streamResults);
+
         this.namespace = builder.namespace;
     }
 
@@ -64,26 +65,6 @@ private List convertSingleResponse(RiakKvPB.RpbListKeysResp resp)
         return keys;
     }
 
-    @Override
-    protected RiakMessage createChannelMessage()
-    {
-        return new RiakMessage(RiakMessageCodes.MSG_ListKeysReq, reqBuilder.build().toByteArray());
-    }
-
-    @Override
-    protected RiakKvPB.RpbListKeysResp decode(RiakMessage rawMessage)
-    {
-        try
-        {
-            Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_ListKeysResp);
-            return RiakKvPB.RpbListKeysResp.parseFrom(rawMessage.getData());
-        }
-        catch (InvalidProtocolBufferException e)
-        {
-            throw new IllegalArgumentException("Invalid message received", e);
-        }
-    }
-
     @Override
     protected boolean done(RiakKvPB.RpbListKeysResp message)
     {
diff --git a/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java b/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java
index 046410764..7565ed8a7 100644
--- a/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java
+++ b/src/main/java/com/basho/riak/client/core/operations/MapReduceOperation.java
@@ -15,8 +15,7 @@
  */
 package com.basho.riak.client.core.operations;
 
-import com.basho.riak.client.core.RiakMessage;
-import com.basho.riak.client.core.StreamingFutureOperation;
+import com.basho.riak.client.core.PBStreamingFutureOperation;
 import com.basho.riak.client.core.util.BinaryValue;
 import com.basho.riak.protobuf.RiakMessageCodes;
 import com.basho.riak.protobuf.RiakKvPB;
@@ -25,7 +24,6 @@
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
 import java.io.IOException;
 
 import java.util.LinkedHashMap;
@@ -39,9 +37,8 @@
  * A Map/Reduce Operation on Riak. No error checking is done on the content type of the content itself
  * with the exception to making sure they are provided.
  */
-public class MapReduceOperation extends StreamingFutureOperation
+public class MapReduceOperation extends PBStreamingFutureOperation
 {
-    private final RiakKvPB.RpbMapRedReq.Builder reqBuilder;
     private final BinaryValue mapReduce;
     private final Logger logger = LoggerFactory.getLogger(MapReduceOperation.class);
     private final ObjectMapper objectMapper = new ObjectMapper();
@@ -50,8 +47,11 @@ public class MapReduceOperation extends StreamingFutureOperation resultMap,
         }
     }
 
-    @Override
-    protected RiakMessage createChannelMessage()
-    {
-        RiakKvPB.RpbMapRedReq request = reqBuilder.build();
-        return new RiakMessage(RiakMessageCodes.MSG_MapRedReq, request.toByteArray());
-    }
-
-    @Override
-    protected RiakKvPB.RpbMapRedResp decode(RiakMessage rawMessage)
-    {
-        Operations.checkPBMessageType(rawMessage, RiakMessageCodes.MSG_MapRedResp);
-        try
-        {
-            return RiakKvPB.RpbMapRedResp.parseFrom(rawMessage.getData());
-        }
-        catch (InvalidProtocolBufferException e)
-        {
-            throw new IllegalArgumentException(e);
-        }
-    }
-
     @Override
     protected boolean done(RiakKvPB.RpbMapRedResp message)
     {
diff --git a/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java b/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java
index 0395cb588..af8e05c54 100644
--- a/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java
+++ b/src/main/java/com/basho/riak/client/core/operations/SecondaryIndexQueryOperation.java
@@ -15,8 +15,8 @@
  */
 package com.basho.riak.client.core.operations;
 
+import com.basho.riak.client.core.PBStreamingFutureOperation;
 import com.basho.riak.client.core.RiakMessage;
-import com.basho.riak.client.core.StreamingFutureOperation;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.query.indexes.IndexNames;
 import com.basho.riak.client.core.util.BinaryValue;
@@ -38,8 +38,8 @@
  * @since 2.0
  */
 public class SecondaryIndexQueryOperation
-        extends StreamingFutureOperation
+        extends PBStreamingFutureOperation
 {
     private final RiakKvPB.RpbIndexReq pbReq;
     private final Query query;
@@ -48,11 +48,17 @@ private SecondaryIndexQueryOperation(Builder builder)
     {
         // Decide if we should release results as they come in (stream), or gather them all until the operation is
         // done (not stream).
-        super(builder.streamResults);
+        super(RiakMessageCodes.MSG_IndexReq,
+                RiakMessageCodes.MSG_IndexResp,
+                builder.pbReqBuilder,
+                null,
+                builder.streamResults);
 
         // Yo dawg, we don't ever not want to use streaming.
         builder.pbReqBuilder.setStream(true);
         this.query = builder.query;
+
+        // TODO: get rid of pbReq usage by switching to use query insted
         this.pbReq = builder.pbReqBuilder.build();
     }
 
@@ -177,12 +183,6 @@ private void convertTerms(SecondaryIndexQueryOperation.Response.Builder builder,
         }
     }
 
-    @Override
-    protected RiakMessage createChannelMessage()
-    {
-        return new RiakMessage(RiakMessageCodes.MSG_IndexReq, pbReq.toByteArray());
-    }
-
     @Override
     protected Object decode(RiakMessage rawMessage)
     {
diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/ListBucketsTest.java b/src/test/java/com/basho/riak/client/api/commands/buckets/ListBucketsTest.java
index 532ed7e30..2cb7bbc7e 100644
--- a/src/test/java/com/basho/riak/client/api/commands/buckets/ListBucketsTest.java
+++ b/src/test/java/com/basho/riak/client/api/commands/buckets/ListBucketsTest.java
@@ -47,7 +47,7 @@ public void init() throws Exception
         when(mockFuture.isCancelled()).thenReturn(false);
         when(mockFuture.isDone()).thenReturn(true);
         when(mockFuture.isSuccess()).thenReturn(true);
-        when(mockCluster.execute(any(StreamingFutureOperation.class))).thenReturn(mockFuture);
+        when(mockCluster.execute(any(PBStreamingFutureOperation.class))).thenReturn(mockFuture);
         client = new RiakClient(mockCluster);
     }
 
diff --git a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java
index 8c4f724a3..6ebd58802 100644
--- a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java
+++ b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java
@@ -40,9 +40,9 @@ public void testInterruptedExceptionDealtWith() throws InterruptedException
                                     TimeUnit.MILLISECONDS)).thenThrow(new InterruptedException(
                         "foo"));
 
-                @SuppressWarnings("unchecked") StreamingFutureOperation
-                        coreFuture = (StreamingFutureOperation) mock(
-                        StreamingFutureOperation.class);
+                @SuppressWarnings("unchecked") PBStreamingFutureOperation
+                        coreFuture = (PBStreamingFutureOperation) mock(
+                        PBStreamingFutureOperation.class);
 
                 when(coreFuture.getResultsQueue()).thenReturn(fakeQueue);
 

From 1c5883c92420561e38e3ef2a2b216e181db48f2c Mon Sep 17 00:00:00 2001
From: Alex Moore 
Date: Wed, 26 Oct 2016 10:33:58 -0400
Subject: [PATCH 27/55] Fix test that stubs a final method

---
 .../core/ChunkedResponseIteratorTest.java     | 48 +++++++++++--------
 1 file changed, 28 insertions(+), 20 deletions(-)

diff --git a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java
index 6ebd58802..1ccf01d6d 100644
--- a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java
+++ b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java
@@ -3,18 +3,23 @@
 
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TransferQueue;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
+import static org.powermock.api.mockito.PowerMockito.doThrow;
 import static org.powermock.api.mockito.PowerMockito.mock;
 import static org.powermock.api.mockito.PowerMockito.when;
 
 /**
  * @author Alex Moore 
  */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(PBStreamingFutureOperation.class)
 public class ChunkedResponseIteratorTest
 {
     /*
@@ -26,23 +31,22 @@ public class ChunkedResponseIteratorTest
     @Test
     public void testInterruptedExceptionDealtWith() throws InterruptedException
     {
-        final boolean[] caught = {false};
-        final InterruptedException[] ie = {null};
+        final Throwable[] ex = {null};
         int timeout = 1000;
 
-        Thread t = new Thread(() ->
+        Thread testThread = new Thread(() ->
         {
             try
             {
-                @SuppressWarnings("unchecked") TransferQueue fakeQueue =
+                @SuppressWarnings("unchecked")
+                TransferQueue fakeQueue =
                         (TransferQueue) mock(TransferQueue.class);
-                when(fakeQueue.poll(timeout,
-                                    TimeUnit.MILLISECONDS)).thenThrow(new InterruptedException(
-                        "foo"));
 
-                @SuppressWarnings("unchecked") PBStreamingFutureOperation
-                        coreFuture = (PBStreamingFutureOperation) mock(
-                        PBStreamingFutureOperation.class);
+                when(fakeQueue.poll(timeout, TimeUnit.MILLISECONDS)).thenThrow(new InterruptedException("foo"));
+
+                @SuppressWarnings("unchecked")
+                PBStreamingFutureOperation coreFuture =
+                        (PBStreamingFutureOperation) mock(PBStreamingFutureOperation.class);
 
                 when(coreFuture.getResultsQueue()).thenReturn(fakeQueue);
 
@@ -53,26 +57,30 @@ public void testInterruptedExceptionDealtWith() throws InterruptedException
                                               Long::new,
                                               FakeResponse::iterator);
             }
-            catch (RuntimeException ex)
+            catch (RuntimeException rex)
             {
-                caught[0] = true;
-                ie[0] = (InterruptedException) ex.getCause();
+                ex[0] = rex;
             }
             catch (InterruptedException e)
             {
                 // Mocking TransferQueue::poll(timeout) requires this CheckedException be dealt with
                 // If we actually catch one here we've failed at our jobs.
-                caught[0] = false;
+                fail(e.getMessage());
             }
 
             assertTrue(Thread.currentThread().isInterrupted());
         });
 
-        t.start();
-        t.join();
+        testThread.start();
+        testThread.join();
+
+        Throwable caughtException = ex[0];
+        assertNotNull(caughtException);
 
-        assertTrue(caught[0]);
-        assertEquals("foo", ie[0].getMessage());
+        Throwable wrappedException = caughtException.getCause();
+        assertNotNull(caughtException.getMessage(), wrappedException);
+        assertEquals("foo", wrappedException.getMessage());
+        assertTrue(wrappedException instanceof InterruptedException);
     }
 
     static abstract class FakeResponse implements Iterable {}

From 386910719b7db8f08e2fa66bf403a08a3168cc78 Mon Sep 17 00:00:00 2001
From: Alex Moore 
Date: Wed, 26 Oct 2016 23:24:51 -0400
Subject: [PATCH 28/55] Remove RuntimeException + reset interrupted status
 after we're done loading

---
 .../api/commands/ChunkedResponseIterator.java |  83 +++++++---
 .../core/ChunkedResponseIteratorTest.java     | 148 ++++++++++++++++--
 2 files changed, 196 insertions(+), 35 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java
index c6af48fd9..9e72967ee 100644
--- a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java
+++ b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java
@@ -56,20 +56,63 @@ public ChunkedResponseIterator(StreamingRiakFuture coreFuture,
         this.createNext = createNextFn;
         this.getNextIterator = getNextIteratorFn;
         this.getContinuationFn = getContinuationFn;
-        loadNextChunkIterator();
+
+
+        boolean interrupted = false;
+        try
+        {
+            boolean lastLoadInterrupted;
+            do
+            {
+                lastLoadInterrupted = false;
+                try
+                {
+                    tryLoadNextChunkIterator();
+                }
+                catch (InterruptedException ex)
+                {
+                    interrupted = true;
+                    lastLoadInterrupted = true;
+                }
+            } while (lastLoadInterrupted);
+        }
+        finally
+        {
+            if (interrupted)
+            {
+                Thread.currentThread().interrupt();
+            }
+        }
     }
 
     @Override
     public boolean hasNext()
     {
-        if (currentIteratorHasNext())
+        boolean interrupted = false;
+        Boolean dataLoaded = null;
+
+        try
+        {
+            while (!currentIteratorHasNext() && dataLoaded == null)
+            {
+                try
+                {
+                    dataLoaded = tryLoadNextChunkIterator();
+                }
+                catch (InterruptedException ex)
+                {
+                    interrupted = true;
+                }
+            }
+            return currentIteratorHasNext();
+        }
+        finally
         {
-            return true;
+            if (interrupted)
+            {
+                Thread.currentThread().interrupt();
+            }
         }
-
-        loadNextChunkIterator();
-
-        return currentIteratorHasNext();
     }
 
     private boolean currentIteratorHasNext()
@@ -100,31 +143,25 @@ public FinalT next()
         return createNext.apply(currentIterator.next());
     }
 
-    private void loadNextChunkIterator()
+    private boolean tryLoadNextChunkIterator() throws InterruptedException
     {
         this.currentIterator = null;
         boolean populatedChunkLoaded = false;
 
-        try
+        while (!populatedChunkLoaded && possibleChunksRemaining())
         {
-            while (!populatedChunkLoaded && possibleChunksRemaining())
-            {
-                final ChunkT nextChunk = chunkQueue.poll(timeout, TimeUnit.MILLISECONDS);
+            final ChunkT nextChunk = chunkQueue.poll(timeout, TimeUnit.MILLISECONDS);
 
-                if (nextChunk != null)
-                {
-                    this.currentIterator = getNextIterator.apply(nextChunk);
-                    populatedChunkLoaded = currentIteratorHasNext();
+            if (nextChunk != null)
+            {
+                this.currentIterator = getNextIterator.apply(nextChunk);
+                populatedChunkLoaded = currentIteratorHasNext();
 
-                    loadContinuation(nextChunk);
-                }
+                loadContinuation(nextChunk);
             }
         }
-        catch (InterruptedException e)
-        {
-            Thread.currentThread().interrupt();
-            throw new RuntimeException(e);
-        }
+
+        return populatedChunkLoaded;
     }
 
     private void loadContinuation(ChunkT nextChunk)
diff --git a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java
index 1ccf01d6d..5b73bd865 100644
--- a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java
+++ b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java
@@ -4,14 +4,18 @@
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TransferQueue;
 
 import static org.junit.Assert.*;
-import static org.powermock.api.mockito.PowerMockito.doThrow;
 import static org.powermock.api.mockito.PowerMockito.mock;
 import static org.powermock.api.mockito.PowerMockito.when;
 
@@ -29,9 +33,8 @@ public class ChunkedResponseIteratorTest
       https://bugs.openjdk.java.net/browse/JDK-8154017
      */
     @Test
-    public void testInterruptedExceptionDealtWith() throws InterruptedException
+    public void testInterruptedExceptionUponInitialisation() throws InterruptedException
     {
-        final Throwable[] ex = {null};
         int timeout = 1000;
 
         Thread testThread = new Thread(() ->
@@ -42,7 +45,16 @@ public void testInterruptedExceptionDealtWith() throws InterruptedException
                 TransferQueue fakeQueue =
                         (TransferQueue) mock(TransferQueue.class);
 
-                when(fakeQueue.poll(timeout, TimeUnit.MILLISECONDS)).thenThrow(new InterruptedException("foo"));
+                when(fakeQueue.poll(timeout, TimeUnit.MILLISECONDS))
+                        .thenThrow(new InterruptedException())
+                        .thenReturn(
+                            new FakeResponse() {
+                                @Override
+                                public Iterator iterator()
+                                {
+                                    return Arrays.asList(1,2,3,4,5).iterator();
+                                }
+                            });
 
                 @SuppressWarnings("unchecked")
                 PBStreamingFutureOperation coreFuture =
@@ -57,9 +69,67 @@ public void testInterruptedExceptionDealtWith() throws InterruptedException
                                               Long::new,
                                               FakeResponse::iterator);
             }
-            catch (RuntimeException rex)
+            catch (InterruptedException e)
+            {
+                // Mocking TransferQueue::poll(timeout) requires this CheckedException be dealt with
+                // If we actually catch one here we've failed at our jobs.
+                fail(e.getMessage());
+            }
+
+            assertTrue(Thread.currentThread().isInterrupted());
+        });
+
+        testThread.start();
+        testThread.join();
+        assertFalse(Thread.currentThread().isInterrupted());
+    }
+
+    @Test
+    public void testInterruptedExceptionUponNextChunkLoad() throws InterruptedException
+    {
+        int timeout = 1000;
+
+        Thread testThread = new Thread(() ->
+        {
+            try
             {
-                ex[0] = rex;
+                @SuppressWarnings("unchecked")
+                TransferQueue fakeQueue =
+                        (TransferQueue) mock(TransferQueue.class);
+
+                when(fakeQueue.poll(timeout, TimeUnit.MILLISECONDS))
+                        .thenReturn(
+                                new FakeResponse() {
+                                    @Override
+                                    public Iterator iterator()
+                                    {
+                                        return Collections.singletonList(1).iterator();
+                                    }
+                                })
+                        .thenThrow(new InterruptedException())
+                        .thenReturn(
+                                new FakeResponse() {
+                                    @Override
+                                    public Iterator iterator()
+                                    {
+                                        return Collections.singletonList(2).iterator();
+                                    }
+                                });
+
+                @SuppressWarnings("unchecked")
+                PBStreamingFutureOperation coreFuture =
+                        (PBStreamingFutureOperation) mock(PBStreamingFutureOperation.class);
+
+                when(coreFuture.getResultsQueue()).thenReturn(fakeQueue);
+
+                ChunkedResponseIterator chunkedResponseIterator =
+                        new ChunkedResponseIterator<>(coreFuture, timeout, Long::new, FakeResponse::iterator);
+
+                assertTrue(chunkedResponseIterator.hasNext());
+                assertEquals(new Long(1), chunkedResponseIterator.next());
+                // Should hit InterruptedException here, and then take care of it.
+                assertTrue(chunkedResponseIterator.hasNext());
+                assertEquals(new Long(2), chunkedResponseIterator.next());
             }
             catch (InterruptedException e)
             {
@@ -73,14 +143,68 @@ public void testInterruptedExceptionDealtWith() throws InterruptedException
 
         testThread.start();
         testThread.join();
+        assertFalse(Thread.currentThread().isInterrupted());
+    }
 
-        Throwable caughtException = ex[0];
-        assertNotNull(caughtException);
+    @Test
+    public void testConcurrentDoneAndInterruptedException() throws InterruptedException
+    {
+        int timeout = 1000;
 
-        Throwable wrappedException = caughtException.getCause();
-        assertNotNull(caughtException.getMessage(), wrappedException);
-        assertEquals("foo", wrappedException.getMessage());
-        assertTrue(wrappedException instanceof InterruptedException);
+        Thread testThread = new Thread(() ->
+        {
+            try
+            {
+                @SuppressWarnings("unchecked")
+                PBStreamingFutureOperation coreFuture =
+                        (PBStreamingFutureOperation) mock(PBStreamingFutureOperation.class);
+                when(coreFuture.isDone()).thenReturn(false);
+
+                @SuppressWarnings("unchecked")
+                TransferQueue fakeQueue =
+                        (TransferQueue) mock(TransferQueue.class);
+
+                when(fakeQueue.poll(timeout, TimeUnit.MILLISECONDS))
+                        .thenReturn(
+                                new FakeResponse() {
+                                    @Override
+                                    public Iterator iterator()
+                                    {
+                                        return Collections.singletonList(1).iterator();
+                                    }
+                                })
+                        .thenAnswer(invocationOnMock ->
+                                    {
+                                        when(coreFuture.isDone()).thenReturn(true);
+                                        when(fakeQueue.isEmpty()).thenReturn(true);
+                                        throw new InterruptedException();
+                                    });
+
+                when(coreFuture.getResultsQueue()).thenReturn(fakeQueue);
+
+                ChunkedResponseIterator chunkedResponseIterator =
+                        new ChunkedResponseIterator<>(coreFuture, timeout, Long::new, FakeResponse::iterator);
+
+                assertTrue(chunkedResponseIterator.hasNext());
+                assertEquals(new Long(1), chunkedResponseIterator.next());
+                // InterruptedException should happen when we try to load the next chunk,
+                // But the catch + next attempt to load a chunk should check to see if we're done.
+                assertFalse(chunkedResponseIterator.hasNext());
+
+            }
+            catch (InterruptedException e)
+            {
+                // Mocking TransferQueue::poll(timeout) requires this CheckedException be dealt with
+                // If we actually catch one here we've failed at our jobs.
+                fail(e.getMessage());
+            }
+
+            assertTrue(Thread.currentThread().isInterrupted());
+        });
+
+        testThread.start();
+        testThread.join();
+        assertFalse(Thread.currentThread().isInterrupted());
     }
 
     static abstract class FakeResponse implements Iterable {}

From def141f379f4c7ca93343832504224453d6cf632 Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Thu, 27 Oct 2016 20:37:52 +0300
Subject: [PATCH 29/55] Avoid useless queue creation in
 PBStreamingFutureOperation

---
 .../basho/riak/client/core/PBStreamingFutureOperation.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/basho/riak/client/core/PBStreamingFutureOperation.java b/src/main/java/com/basho/riak/client/core/PBStreamingFutureOperation.java
index f00d77295..d0484790c 100644
--- a/src/main/java/com/basho/riak/client/core/PBStreamingFutureOperation.java
+++ b/src/main/java/com/basho/riak/client/core/PBStreamingFutureOperation.java
@@ -45,7 +45,7 @@ protected PBStreamingFutureOperation(final byte reqMessageCode,
     {
         super(reqMessageCode, respMessageCode, reqBuilder, respParser);
         this.streamResults = streamResults;
-        this.responseQueue = new LinkedTransferQueue<>();
+        this.responseQueue = streamResults ? new LinkedTransferQueue<>() : null;
     }
 
     @Override
@@ -58,6 +58,7 @@ protected void processMessage(ResponseType decodedMessage)
         }
 
         final ReturnType r = processStreamingChunk(decodedMessage);
+        assert this.responseQueue != null;
         responseQueue.offer(r);
     }
 
@@ -65,6 +66,7 @@ protected void processMessage(ResponseType decodedMessage)
 
     public final TransferQueue getResultsQueue()
     {
+        assert this.responseQueue != null;
         return this.responseQueue;
     }
 }

From 96f1bb6ae8d3db3daa99eeea28d7720459ea5227 Mon Sep 17 00:00:00 2001
From: Alex Moore 
Date: Fri, 28 Oct 2016 12:12:23 -0400
Subject: [PATCH 30/55] Fix interrupts in Streaming MR, check interrupted
 status before waiting, add docs about interrupted threads

---
 .../com/basho/riak/client/api/RiakClient.java | 29 ++++++++---
 .../api/commands/ChunkedResponseIterator.java | 14 +++++-
 .../api/commands/mapreduce/MapReduce.java     | 50 ++++++++++++-------
 3 files changed, 68 insertions(+), 25 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/RiakClient.java b/src/main/java/com/basho/riak/client/api/RiakClient.java
index 6bec3fae2..1e91392cb 100644
--- a/src/main/java/com/basho/riak/client/api/RiakClient.java
+++ b/src/main/java/com/basho/riak/client/api/RiakClient.java
@@ -18,6 +18,7 @@
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.RiakNode;
+import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.util.HostAndPort;
 
 import java.net.InetSocketAddress;
@@ -404,18 +405,34 @@ public  RiakFuture executeAsync(RiakCommand command)
     }
 
     /**
-     * Execute a StreamableRiakCommand asynchronously, and stream the results back before the command is complete.
+     * Execute a StreamableRiakCommand asynchronously, and stream the results back before
+     * the command {@link RiakFuture#isDone() is done}.
      * 

-     * Calling this method  causes the client to execute the provided StreamableRiakCommand
-     * asynchronously. It will immediately return a RiakFuture that contains an immediately available result that data
-     * will be streamed to. The RiakFuture will also keep track of the overall operation's progress with the
-     * {@link RiakFuture#isDone}, etc methods.
-     *
+     *     Calling this method  causes the client to execute the provided
+     *     StreamableRiakCommand asynchronously.
+     *     It will immediately return a RiakFuture that contains an
+     *     immediately available result (via {@link RiakFuture#get()}) that
+     *     data will be streamed to.
+     *     The RiakFuture will also keep track of the overall operation's progress
+     *     with the {@link RiakFuture#isDone}, etc methods.
+     * 

+     * 

+     *     Because the consumer thread will poll for new results, it is advisable to check the
+     *     consumer thread's interrupted status via
+     *     {@link Thread#isInterrupted() Thread.currentThread().isInterrupted() }, as the result
+     *     iterator will not propagate an InterruptedException, but it will set the Thread's
+     *     interrupted flag.
+     * 

      * @param  StreamableRiakCommand's immediate return type, available before the command/operation is complete.
      * @param  The RiakCommand's query info type.
      * @param command The RiakCommand to execute.
      * @param timeoutMS The polling timeout in milliseconds for each result chunk.
      *                  If the timeout is reached it will try again, instead of blocking indefinitely.
+     *                  If the value is too small (less than the average chunk arrival time), the
+     *                  result iterator will essentially busy wait.
+     *                  If the timeout is too large (much greater than the average chunk arrival time),
+     *                  the result iterator can block the consuming thread from seeing the done()
+     *                  status until the timeout is reached.
      * @return a RiakFuture for the operation
      * @see RiakFuture
      */
diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java
index 9e72967ee..29c953fec 100644
--- a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java
+++ b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java
@@ -58,7 +58,10 @@ public ChunkedResponseIterator(StreamingRiakFuture coreFuture,
         this.getContinuationFn = getContinuationFn;
 
 
-        boolean interrupted = false;
+        // Check & clear interrupted flag so we don't get an
+        // InterruptedException every time if the user
+        // doesn't clear it / deal with it.
+        boolean interrupted = Thread.interrupted();
         try
         {
             boolean lastLoadInterrupted;
@@ -80,6 +83,8 @@ public ChunkedResponseIterator(StreamingRiakFuture coreFuture,
         {
             if (interrupted)
             {
+                // Reset interrupted flag if we came in with it
+                // or we were interrupted while waiting.
                 Thread.currentThread().interrupt();
             }
         }
@@ -88,7 +93,10 @@ public ChunkedResponseIterator(StreamingRiakFuture coreFuture,
     @Override
     public boolean hasNext()
     {
-        boolean interrupted = false;
+        // Check & clear interrupted flag so we don't get an
+        // InterruptedException every time if the user
+        // doesn't clear it / deal with it.
+        boolean interrupted = Thread.interrupted();
         Boolean dataLoaded = null;
 
         try
@@ -110,6 +118,8 @@ public boolean hasNext()
         {
             if (interrupted)
             {
+                // Reset interrupted flag if we came in with it
+                // or we were interrupted while waiting.
                 Thread.currentThread().interrupt();
             }
         }
diff --git a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
index a4a59b1f9..8040332fb 100644
--- a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
+++ b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
@@ -460,37 +460,53 @@ private class MapReduceResponseIterator implements Iterator
             @Override
             public boolean hasNext()
             {
-                if (resultsQueueHasAny())
-                {
-                    return true;
-                }
-
+                // Check & clear interrupted flag so we don't get an
+                // InterruptedException every time if the user
+                // doesn't clear it / deal with it.
+                boolean interrupted = Thread.interrupted();
                 try
                 {
-                    return peekWaitForNextQueueEntry();
+                    boolean foundEntry = false;
+                    boolean interruptedLastLoop;
+
+                    do
+                    {
+                        interruptedLastLoop = false;
+
+                        try
+                        {
+                            foundEntry = peekWaitForNextQueueEntry();
+                        }
+                        catch (InterruptedException e)
+                        {
+                            interrupted = true;
+                            interruptedLastLoop = true;
+                        }
+                    } while (interruptedLastLoop);
+
+                    return foundEntry;
                 }
-                catch (InterruptedException e)
+                finally
                 {
-                    Thread.currentThread().interrupt();
-                    throw new RuntimeException(e);
+                    if(interrupted)
+                    {
+                        // Reset interrupted flag if we came in with it
+                        // or we were interrupted while waiting.
+                        Thread.currentThread().interrupt();
+                    }
                 }
             }
 
             private boolean peekWaitForNextQueueEntry() throws InterruptedException
             {
-                while (!resultsQueueHasAny() && !coreFuture.isDone())
+                while (resultsQueue.isEmpty() && !coreFuture.isDone())
                 {
-                    if (!resultsQueueHasAny())
+                    if (resultsQueue.isEmpty())
                     {
                         Thread.sleep(pollTimeout);
                     }
                 }
-                return resultsQueueHasAny();
-            }
-
-            private boolean resultsQueueHasAny()
-            {
-                return resultsQueue.peek() != null;
+                return !resultsQueue.isEmpty();
             }
 
             @Override

From 443cb40c073213e6704a9603584c5d80a64de27e Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Thu, 3 Nov 2016 05:22:25 +0200
Subject: [PATCH 31/55] Initial simplification of SecondaryIndexQuery command
 hierarchy

---
 .../api/commands/ChunkedResponseIterator.java |   2 +-
 .../commands/indexes/BigIntIndexQuery.java    | 132 +---------
 .../api/commands/indexes/BinIndexQuery.java   | 125 +--------
 .../api/commands/indexes/IntIndexQuery.java   | 127 +--------
 .../api/commands/indexes/RawIndexQuery.java   | 128 +---------
 .../commands/indexes/SecondaryIndexQuery.java | 241 ++++++++++++++----
 .../indexes/Streaming2iQueryFuture.java       |  50 ----
 .../api/commands/kv/FullBucketRead.java       | 147 ++---------
 .../indexes/itest/ITestBigIntIndexQuery.java  |   4 +-
 .../indexes/itest/ITestBinIndexQuery.java     |   6 +-
 .../indexes/itest/ITestFullBucketRead.java    |   6 +-
 .../indexes/itest/ITestIntIndexQuery.java     |   7 +-
 .../indexes/itest/ITestRawIndexQuery.java     |  10 +-
 .../operations/itest/ITestCoveragePlan.java   |   4 +-
 14 files changed, 260 insertions(+), 729 deletions(-)
 delete mode 100644 src/main/java/com/basho/riak/client/api/commands/indexes/Streaming2iQueryFuture.java

diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java
index 29c953fec..dc4a96623 100644
--- a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java
+++ b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java
@@ -34,7 +34,7 @@ public class ChunkedResponseIterator, Cor
     private final Function> getNextIterator;
     private final Function getContinuationFn;
 
-    private Iterator currentIterator = null;
+    protected Iterator currentIterator = null;
 
     public ChunkedResponseIterator(StreamingRiakFuture coreFuture,
                                    int pollTimeout,
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java
index aa17b0061..52933a788 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java
@@ -17,18 +17,11 @@
 package com.basho.riak.client.api.commands.indexes;
 
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
-import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
-import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
 
 import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.List;
 
 /**
  * Performs a 2i query where the 2i index keys are numeric.
@@ -74,13 +67,11 @@
  *
  * @author Brian Roach 
  * @author Alex Moore 
+ * @author Sergey Galkin 
  * @since 2.0
  */
 public class BigIntIndexQuery
-        extends SecondaryIndexQuery
+        extends SecondaryIndexQuery
 {
     private final IndexConverter converter;
 
@@ -92,7 +83,7 @@ protected IndexConverter getConverter()
 
     protected BigIntIndexQuery(Init builder)
     {
-        super(builder);
+        super(builder, Response::new, Response::new);
         this.converter = new IndexConverter()
         {
             @Override
@@ -114,59 +105,6 @@ public BinaryValue convert(BigInteger input)
         };
     }
 
-    @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            executeCoreAsync(cluster);
-
-        BigIntQueryFuture future = new BigIntQueryFuture(coreFuture);
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    @Override
-    protected RiakFuture executeAsyncStreaming(RiakCluster cluster,
-                                                                                                     int timeout)
-    {
-        StreamingRiakFuture coreFuture =
-                executeCoreAsyncStreaming(cluster);
-
-        StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout);
-
-        Streaming2iQueryFuture future =
-                new Streaming2iQueryFuture<>(coreFuture, response, this);
-
-        coreFuture.addListener(future);
-
-        return future;
-    }
-
-    protected final class BigIntQueryFuture
-            extends CoreFutureAdapter
-    {
-        public BigIntQueryFuture(RiakFuture coreFuture)
-        {
-            super(coreFuture);
-        }
-
-        @Override
-        protected Response convertResponse(SecondaryIndexQueryOperation.Response coreResponse)
-        {
-            return new Response(namespace, coreResponse, converter);
-        }
-
-        @Override
-        protected BigIntIndexQuery convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo)
-        {
-            return BigIntIndexQuery.this;
-        }
-    }
-
     protected static abstract class Init> extends SecondaryIndexQuery.Init
     {
         public Init(Namespace namespace, String indexName, S start, S end)
@@ -258,68 +196,14 @@ public BigIntIndexQuery build()
         }
     }
 
-    public static class Response extends SecondaryIndexQuery.Response
-    {
-        protected Response(Namespace queryLocation,
-                           SecondaryIndexQueryOperation.Response coreResponse,
-                           IndexConverter converter)
-        {
-            super(queryLocation, coreResponse, converter);
-        }
-
-        @Override
-        public List getEntries()
-        {
-            List convertedList = new ArrayList<>();
-            for (SecondaryIndexQueryOperation.Response.Entry e : coreResponse.getEntryList())
-            {
-                Location loc = getLocationFromCoreEntry(e);
-                Entry ce = new Entry(loc, e.getIndexKey(), converter);
-                convertedList.add(ce);
-            }
-            return convertedList;
-        }
-
-        public static class Entry extends SecondaryIndexQuery.Response.Entry
-        {
-            protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverter converter)
-            {
-                super(riakObjectLocation, indexKey, converter);
-            }
-        }
-    }
-
-    public static class StreamingResponse extends SecondaryIndexQuery.StreamingResponse
+    public static class Response extends SecondaryIndexQuery.Response>
     {
-        StreamingResponse(Namespace namespace, IndexConverter converter,
-                          StreamingRiakFuture coreFuture,
-                          int pollTimeout)
-        {
-            super(namespace, converter, createResponseIterator(namespace, converter, coreFuture, pollTimeout));
-        }
-
-        private static ChunkedResponseIterator createResponseIterator(
-                Namespace namespace,
-                IndexConverter converter,
-                StreamingRiakFuture coreFuture,
-                int pollTimeout)
-        {
-            return new ChunkedResponseIterator<>(
-                    coreFuture,
-                    pollTimeout,
-                    (e) -> createExternalResponseEntry(namespace, e, converter),
-                    SecondaryIndexQueryOperation.Response::iterator,
-                    SecondaryIndexQueryOperation.Response::getContinuation);
+        protected Response(Namespace queryLocation, IndexConverter converter, ChunkedResponseIterator chunkedResponseIterator) {
+            super(queryLocation, converter, chunkedResponseIterator);
         }
 
-        private static Response.Entry createExternalResponseEntry(Namespace namespace,
-                                                                  SecondaryIndexQueryOperation.Response.Entry baseEntry,
-                                                                  IndexConverter converter)
-        {
-            return new Response.Entry(SecondaryIndexQuery.Response.getLocationFromCoreEntry(namespace, baseEntry),
-                                      baseEntry.getIndexKey(),
-                                      converter);
+        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter) {
+            super(queryLocation, coreResponse, converter);
         }
     }
 }
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java
index c0a119e00..1a65a2374 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java
@@ -17,20 +17,13 @@
 package com.basho.riak.client.api.commands.indexes;
 
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
-import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
-import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.query.indexes.IndexNames;
 import com.basho.riak.client.core.util.BinaryValue;
 import com.basho.riak.client.core.util.DefaultCharset;
 
 import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
 
 /**
  * Performs a 2i query where the 2i index keys are strings.
@@ -74,16 +67,17 @@
  *
  * @author Brian Roach 
  * @author Alex Moore 
+ * @author Sergey Galkin 
  * @since 2.0
  */
-public class BinIndexQuery extends SecondaryIndexQuery
+public class BinIndexQuery extends SecondaryIndexQuery
 {
     private final Charset charset;
     private final IndexConverter converter;
 
     protected BinIndexQuery(Init builder)
     {
-        super(builder);
+        super(builder, Response::new, Response::new);
         this.charset = builder.charset;
         this.converter = new StringIndexConverter();
     }
@@ -94,33 +88,6 @@ protected IndexConverter getConverter()
         return converter;
     }
 
-    @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-                executeCoreAsync(cluster);
-
-        BinQueryFuture future = new BinQueryFuture(coreFuture);
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    @Override
-    protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
-    {
-        StreamingRiakFuture coreFuture =
-                executeCoreAsyncStreaming(cluster);
-
-        StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout);
-
-        Streaming2iQueryFuture future =
-                new Streaming2iQueryFuture<>(coreFuture, response, this);
-
-        coreFuture.addListener(future);
-
-        return future;
-    }
-
     @Override
     public boolean equals(Object o)
     {
@@ -267,91 +234,15 @@ public BinIndexQuery build()
         }
     }
 
-    public static class Response extends SecondaryIndexQuery.Response
-    {
-        protected Response(Namespace queryLocation,
-                           SecondaryIndexQueryOperation.Response coreResponse,
-                           IndexConverter converter)
-        {
-            super(queryLocation, coreResponse, converter);
-        }
-
-        @Override
-        public List getEntries()
-        {
-            List convertedList = new ArrayList<>();
-            for (SecondaryIndexQueryOperation.Response.Entry e : coreResponse.getEntryList())
-            {
-                Location loc = getLocationFromCoreEntry(e);
-                Entry ce = new Entry(loc, e.getIndexKey(), converter);
-                convertedList.add(ce);
-            }
-            return convertedList;
-        }
-
-        public static class Entry extends SecondaryIndexQuery.Response.Entry
-        {
-            protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverter converter)
-            {
-                super(riakObjectLocation, indexKey, converter);
-            }
-        }
-    }
-
-    public static class StreamingResponse extends SecondaryIndexQuery.StreamingResponse
-    {
-        StreamingResponse(Namespace namespace, IndexConverter converter,
-                          StreamingRiakFuture coreFuture,
-                          int pollTimeout)
-        {
-            super(namespace, converter, createResponseIterator(namespace, converter, coreFuture, pollTimeout));
-        }
-
-        private static ChunkedResponseIterator createResponseIterator(
-                Namespace namespace,
-                IndexConverter converter,
-                StreamingRiakFuture coreFuture,
-                int pollTimeout)
-        {
-            return new ChunkedResponseIterator<>(
-                    coreFuture,
-                    pollTimeout,
-                    (e) -> createExternalResponseEntry(namespace, e, converter),
-                    SecondaryIndexQueryOperation.Response::iterator,
-                    SecondaryIndexQueryOperation.Response::getContinuation);
-        }
-
-        private static Response.Entry createExternalResponseEntry(Namespace namespace,
-                                                                  SecondaryIndexQueryOperation.Response.Entry baseEntry,
-                                                                  IndexConverter converter)
-        {
-            return new Response.Entry(SecondaryIndexQuery.Response.getLocationFromCoreEntry(namespace, baseEntry),
-                                      baseEntry.getIndexKey(),
-                                      converter);
-        }
-    }
-
-    protected final class BinQueryFuture
-            extends CoreFutureAdapter
+    public static class Response extends SecondaryIndexQuery.Response>
     {
-        public BinQueryFuture(RiakFuture
-                                      coreFuture)
-        {
-            super(coreFuture);
-        }
 
-        @Override
-        protected Response convertResponse(SecondaryIndexQueryOperation.Response coreResponse)
-        {
-            return new Response(namespace, coreResponse, converter);
+        protected Response(Namespace queryLocation, IndexConverter converter, ChunkedResponseIterator chunkedResponseIterator) {
+            super(queryLocation, converter, chunkedResponseIterator);
         }
 
-        @Override
-        protected BinIndexQuery convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo)
-        {
-            return BinIndexQuery.this;
+        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter) {
+            super(queryLocation, coreResponse, converter);
         }
     }
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java
index e6af40452..50ac2dca3 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java
@@ -17,16 +17,10 @@
 package com.basho.riak.client.api.commands.indexes;
 
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
-import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
-import java.util.ArrayList;
-import java.util.List;
+
 
 /**
  * Performs a 2i query where the 2i index keys are numeric.
@@ -70,10 +64,10 @@
  *
  * @author Brian Roach 
  * @author Alex Moore 
+ * @author Sergey Galkin 
  * @since 2.0
  */
-public class IntIndexQuery extends SecondaryIndexQuery
+public class IntIndexQuery extends SecondaryIndexQuery
 {
     private final IndexConverter converter;
 
@@ -85,7 +79,7 @@ protected IndexConverter getConverter()
 
     protected IntIndexQuery(Init builder)
     {
-        super(builder);
+        super(builder, Response::new, Response::new);
         this.converter = new IndexConverter()
         {
             @Override
@@ -108,57 +102,6 @@ public BinaryValue convert(Long input)
         };
     }
 
-    @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            executeCoreAsync(cluster);
-
-        IntQueryFuture future = new IntQueryFuture(coreFuture);
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    @Override
-    protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
-    {
-        StreamingRiakFuture coreFuture =
-                executeCoreAsyncStreaming(cluster);
-
-        StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout);
-
-        Streaming2iQueryFuture future =
-                new Streaming2iQueryFuture<>(coreFuture, response, this);
-
-        coreFuture.addListener(future);
-
-        return future;
-    }
-
-    protected final class IntQueryFuture
-            extends CoreFutureAdapter
-    {
-        public IntQueryFuture(RiakFuture coreFuture)
-        {
-            super(coreFuture);
-        }
-
-        @Override
-        protected Response convertResponse(SecondaryIndexQueryOperation.Response coreResponse)
-        {
-            return new Response(namespace, coreResponse, converter);
-        }
-
-        @Override
-        protected IntIndexQuery convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo)
-        {
-            return IntIndexQuery.this;
-        }
-    }
-
     protected static abstract class Init> extends SecondaryIndexQuery.Init
     {
         public Init(Namespace namespace, String indexName, S start, S end)
@@ -250,66 +193,14 @@ public IntIndexQuery build()
         }
     }
 
-    public static class Response extends SecondaryIndexQuery.Response
+    public static class Response extends SecondaryIndexQuery.Response>
     {
-        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter)
-        {
-            super(queryLocation, coreResponse, converter);
+        protected Response(Namespace queryLocation, IndexConverter converter, ChunkedResponseIterator chunkedResponseIterator) {
+            super(queryLocation, converter, chunkedResponseIterator);
         }
 
-        @Override
-        public List getEntries()
-        {
-            List convertedList = new ArrayList<>();
-            for (SecondaryIndexQueryOperation.Response.Entry e : coreResponse.getEntryList())
-            {
-                Location loc = getLocationFromCoreEntry(e);
-                Entry ce = new Entry(loc, e.getIndexKey(), converter);
-                convertedList.add(ce);
-            }
-            return convertedList;
-        }
-
-        public static class Entry extends SecondaryIndexQuery.Response.Entry
-        {
-            protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverter converter)
-            {
-                super(riakObjectLocation, indexKey, converter);
-            }
-        }
-    }
-
-    public static class StreamingResponse extends SecondaryIndexQuery.StreamingResponse
-    {
-        StreamingResponse(Namespace namespace, IndexConverter converter,
-                          StreamingRiakFuture coreFuture,
-                          int pollTimeout)
-        {
-            super(namespace, converter, createResponseIterator(namespace, converter, coreFuture, pollTimeout));
-        }
-
-        private static ChunkedResponseIterator createResponseIterator(
-                Namespace namespace,
-                IndexConverter converter,
-                StreamingRiakFuture coreFuture,
-                int pollTimeout)
-        {
-            return new ChunkedResponseIterator<>(
-                    coreFuture,
-                    pollTimeout,
-                    (e) -> createExternalResponseEntry(namespace, e, converter),
-                    SecondaryIndexQueryOperation.Response::iterator,
-                    SecondaryIndexQueryOperation.Response::getContinuation);
-        }
-
-        private static Response.Entry createExternalResponseEntry(Namespace namespace,
-                                                                  SecondaryIndexQueryOperation.Response.Entry baseEntry,
-                                                                  IndexConverter converter)
-        {
-            return new Response.Entry(SecondaryIndexQuery.Response.getLocationFromCoreEntry(namespace, baseEntry),
-                                      baseEntry.getIndexKey(),
-                                      converter);
+        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter) {
+            super(queryLocation, coreResponse, converter);
         }
     }
 }
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java
index 1f92b6049..fa6714eef 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java
@@ -17,16 +17,9 @@
 package com.basho.riak.client.api.commands.indexes;
 
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
-import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
-import java.util.ArrayList;
-import java.util.List;
 
 /**
  * Performs a 2i query where the 2i index keys are raw bytes.
@@ -72,15 +65,16 @@
  *
  * @author Brian Roach 
  * @author Alex Moore 
+ * @author Sergey Galkin 
  * @since 2.0
  */
-public class RawIndexQuery extends SecondaryIndexQuery
+public class RawIndexQuery extends SecondaryIndexQuery
 {
     private final IndexConverter converter;
 
     protected RawIndexQuery(Init builder)
     {
-        super(builder);
+        super(builder, Response::new, Response::new);
         this.converter = new IndexConverter()
         {
             @Override
@@ -97,58 +91,6 @@ protected IndexConverter getConverter()
         return converter;
     }
 
-    @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            executeCoreAsync(cluster);
-
-        RawQueryFuture future = new RawQueryFuture(coreFuture);
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    @Override
-    protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
-    {
-        StreamingRiakFuture coreFuture =
-                executeCoreAsyncStreaming(cluster);
-
-        StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout);
-
-        Streaming2iQueryFuture future =
-                new Streaming2iQueryFuture<>(coreFuture, response, this);
-
-        coreFuture.addListener(future);
-
-        return future;
-    }
-
-    protected final class RawQueryFuture
-            extends CoreFutureAdapter
-    {
-        public RawQueryFuture(RiakFuture coreFuture)
-        {
-            super(coreFuture);
-        }
-
-        @Override
-        protected Response convertResponse(SecondaryIndexQueryOperation.Response coreResponse)
-        {
-            return new Response(namespace, coreResponse, converter);
-        }
-
-        @Override
-        protected RawIndexQuery convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo)
-        {
-            return RawIndexQuery.this;
-        }
-    }
-
     /**
      * Builder used to construct a RawIndexQuery command.
      */
@@ -203,68 +145,14 @@ public RawIndexQuery build()
         }
     }
 
-    public static class Response extends SecondaryIndexQuery.Response
-    {
-        protected Response(Namespace queryLocation,
-                           SecondaryIndexQueryOperation.Response coreResponse,
-                           IndexConverter converter)
-        {
-            super(queryLocation, coreResponse, converter);
-        }
-
-        @Override
-        public List getEntries()
-        {
-            List convertedList = new ArrayList<>();
-            for (SecondaryIndexQueryOperation.Response.Entry e : coreResponse.getEntryList())
-            {
-                Location loc = getLocationFromCoreEntry(e);
-                Entry ce = new Entry(loc, e.getIndexKey(), converter);
-                convertedList.add(ce);
-            }
-            return convertedList;
-        }
-
-        public static class Entry extends SecondaryIndexQuery.Response.Entry
-        {
-            protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverter converter)
-            {
-                super(riakObjectLocation, indexKey, converter);
-            }
-        }
-    }
-
-    public static class StreamingResponse extends SecondaryIndexQuery.StreamingResponse
+    public static class Response extends SecondaryIndexQuery.Response>
     {
-        StreamingResponse(Namespace namespace, IndexConverter converter,
-                          StreamingRiakFuture coreFuture,
-                          int pollTimeout)
-        {
-            super(namespace, converter, createResponseIterator(namespace, converter, coreFuture, pollTimeout));
-        }
-
-        private static ChunkedResponseIterator createResponseIterator(
-                Namespace namespace,
-                IndexConverter converter,
-                StreamingRiakFuture coreFuture,
-                int pollTimeout)
-        {
-            return new ChunkedResponseIterator<>(
-                    coreFuture,
-                    pollTimeout,
-                    (e) -> createExternalResponseEntry(namespace, e, converter),
-                    SecondaryIndexQueryOperation.Response::iterator,
-                    SecondaryIndexQueryOperation.Response::getContinuation);
+        protected Response(Namespace queryLocation, IndexConverter converter, ChunkedResponseIterator chunkedResponseIterator) {
+            super(queryLocation, converter, chunkedResponseIterator);
         }
 
-        private static Response.Entry createExternalResponseEntry(Namespace namespace,
-                                                                  SecondaryIndexQueryOperation.Response.Entry baseEntry,
-                                                                  IndexConverter converter)
-        {
-            return new Response.Entry(SecondaryIndexQuery.Response.getLocationFromCoreEntry(namespace, baseEntry),
-                                      baseEntry.getIndexKey(),
-                                      converter);
+        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter) {
+            super(queryLocation, coreResponse, converter);
         }
     }
 }
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
index 51fd339b9..47d8cf242 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
@@ -16,10 +16,10 @@
 
 package com.basho.riak.client.api.commands.indexes;
 
-import com.basho.riak.client.api.RiakCommand;
 import com.basho.riak.client.api.StreamableRiakCommand;
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
-import com.basho.riak.client.core.PBStreamingFutureOperation;
+import com.basho.riak.client.api.commands.CoreFutureAdapter;
+import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.StreamingRiakFuture;
@@ -28,6 +28,7 @@
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
 
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
@@ -40,10 +41,27 @@
  * @param  the type being used for the query.
  * @author Brian Roach 
  * @author Alex Moore 
+ * @author Sergey Galkin 
  * @since 2.0
  */
-public abstract class SecondaryIndexQuery extends StreamableRiakCommand
+public abstract class SecondaryIndexQuery, U> extends StreamableRiakCommand
 {
+    @FunctionalInterface
+    public interface StreamableResponseCreator>
+    {
+
+        R createResponse(Namespace queryLocation,
+                         IndexConverter converter,
+                         ChunkedResponseIterator chunkedResponseIterator);
+    }
+
+    public interface GatherableResponseCreator>
+    {
+        R createResponse(Namespace queryLocation,
+                         SecondaryIndexQueryOperation.Response coreResponse,
+                         IndexConverter converter);
+    }
+
     protected final Namespace namespace;
     protected final String indexName;
     protected final BinaryValue continuation;
@@ -57,7 +75,11 @@ public abstract class SecondaryIndexQuery extends StreamableRiakComm
     protected Integer timeout;
     protected final byte[] coverageContext;
     protected final boolean returnBody;
-    protected SecondaryIndexQuery(Init builder)
+    private final StreamableResponseCreator streamableResponseCreator;
+    private final GatherableResponseCreator gatherableResponseCreator;
+
+    protected SecondaryIndexQuery(Init builder, StreamableResponseCreator streamableCreator,
+                                  GatherableResponseCreator gatherableResponseCreator)
     {
         this.namespace = builder.namespace;
         this.indexName = builder.indexName;
@@ -72,6 +94,8 @@ protected SecondaryIndexQuery(Init builder)
         this.timeout = builder.timeout;
         this.coverageContext = builder.coverageContext;
         this.returnBody = builder.returnBody;
+        this.streamableResponseCreator = streamableCreator;
+        this.gatherableResponseCreator = gatherableResponseCreator;
     }
 
     protected abstract IndexConverter getConverter();
@@ -209,7 +233,7 @@ protected final SecondaryIndexQueryOperation.Query createCoreQuery()
         else
         {
             coreQueryBuilder.withRangeStart(converter.convert(start))
-                            .withRangeEnd(converter.convert(end));
+                    .withRangeEnd(converter.convert(end));
         }
 
         if (maxResults != null)
@@ -230,7 +254,7 @@ protected final SecondaryIndexQueryOperation.Query createCoreQuery()
     }
 
     protected RiakFuture executeCoreAsync(RiakCluster cluster)
+            SecondaryIndexQueryOperation.Query> executeCoreAsync(RiakCluster cluster)
     {
         SecondaryIndexQueryOperation.Builder builder =
                 new SecondaryIndexQueryOperation.Builder(this.createCoreQuery());
@@ -239,7 +263,7 @@ SecondaryIndexQueryOperation.Query> executeCoreAsync(RiakCluster cluster)
     }
 
     protected StreamingRiakFuture executeCoreAsyncStreaming(RiakCluster cluster)
+            SecondaryIndexQueryOperation.Query> executeCoreAsyncStreaming(RiakCluster cluster)
     {
         SecondaryIndexQueryOperation.Builder builder =
                 new SecondaryIndexQueryOperation.Builder(this.createCoreQuery()).streamResults(true);
@@ -247,6 +271,82 @@ SecondaryIndexQueryOperation.Query> executeCoreAsyncStreaming(RiakCluster cluste
         return cluster.execute(builder.build());
     }
 
+    private S convertResponse(final SecondaryIndexQueryOperation.Response coreResponse,
+                                final ChunkedResponseIterator iterator)
+    {
+        final S response;
+
+        if (iterator != null)
+        {
+            response = streamableResponseCreator.createResponse(namespace, getConverter(), iterator);
+        }
+        else
+        {
+            response = gatherableResponseCreator.createResponse(namespace, coreResponse, getConverter());
+        }
+
+        return response;
+    }
+
+    @Override
+    protected final RiakFuture executeAsync(RiakCluster cluster)
+    {
+        final RiakFuture coreFuture = executeCoreAsync(cluster);
+        final CoreFutureAdapter future =
+                new CoreFutureAdapter(coreFuture) {
+                    @Override
+                    protected S convertResponse(SecondaryIndexQueryOperation.Response coreResponse) {
+                        return SecondaryIndexQuery.this.convertResponse(coreResponse, null);
+                    }
+
+                    @SuppressWarnings("unchecked")
+                    @Override
+                    protected U convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo) {
+                        return (U)SecondaryIndexQuery.this;
+                    }
+                };
+
+        coreFuture.addListener(future);
+        return future;
+    }
+
+    @Override
+    protected final RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
+    {
+        final StreamingRiakFuture coreFuture =
+                executeCoreAsyncStreaming(cluster);
+
+        final Response[] responses = {null};
+
+        final ChunkedResponseIterator iterator = new ChunkedResponseIterator(
+            coreFuture, timeout, null,
+            SecondaryIndexQueryOperation.Response::iterator,
+            SecondaryIndexQueryOperation.Response::getContinuation)
+        {
+            @SuppressWarnings("unchecked")
+            @Override
+            public Response.Entry next() {
+                final SecondaryIndexQueryOperation.Response.Entry coreEntity = currentIterator.next();
+                return responses[0].createEntry(namespace, coreEntity, getConverter());
+            }
+        };
+
+
+        final S response = convertResponse(null, iterator);
+        responses[0] = response;
+
+        final ImmediateCoreFutureAdapter future = new ImmediateCoreFutureAdapter(coreFuture, response) {
+            @SuppressWarnings("unchecked")
+            @Override
+            protected U convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo) {
+                return (U)SecondaryIndexQuery.this;
+            }
+        };
+
+        coreFuture.addListener(future);
+        return future;
+    }
+
     @Override
     public boolean equals(Object o)
     {
@@ -259,7 +359,7 @@ public boolean equals(Object o)
             return false;
         }
 
-        SecondaryIndexQuery that = (SecondaryIndexQuery) o;
+        SecondaryIndexQuery that = (SecondaryIndexQuery) o;
 
         if (returnTerms != that.returnTerms)
         {
@@ -560,19 +660,51 @@ protected T withReturnBody(boolean returnBody)
      *
      * @param  The type contained in the resposne.
      */
-    public abstract static class Response
+    public static class Response> implements Iterable
     {
         final protected IndexConverter converter;
         final protected SecondaryIndexQueryOperation.Response coreResponse;
         final protected Namespace queryLocation;
+        private final ChunkedResponseIterator chunkedResponseIterator;
+
+        private Response(Namespace queryLocation,
+                         IndexConverter converter,
+                         ChunkedResponseIterator chunkedResponseIterator,
+                         SecondaryIndexQueryOperation.Response coreResponse)
+        {
+            this.converter = converter;
+            this.queryLocation = queryLocation;
+            this.chunkedResponseIterator = chunkedResponseIterator;
+            this.coreResponse = coreResponse;
+        }
+
+        protected Response(Namespace queryLocation,
+                           IndexConverter converter,
+                           ChunkedResponseIterator chunkedResponseIterator)
+        {
+            this(queryLocation, converter, chunkedResponseIterator, null);
+        }
 
         protected Response(Namespace queryLocation,
                            SecondaryIndexQueryOperation.Response coreResponse,
                            IndexConverter converter)
         {
-            this.coreResponse = coreResponse;
-            this.converter = converter;
-            this.queryLocation = queryLocation;
+            this(queryLocation, converter, null, coreResponse);
+        }
+
+        public boolean isStreamable()
+        {
+            return chunkedResponseIterator != null;
+        }
+
+        public Iterator iterator()
+        {
+            if (isStreamable()) {
+                return (Iterator)chunkedResponseIterator;
+            }
+
+            // TODO: add support for not streamable responses
+            throw new UnsupportedOperationException("Iterating is only supported for streamable response.");
         }
 
         /**
@@ -582,6 +714,11 @@ protected Response(Namespace queryLocation,
          */
         public boolean hasContinuation()
         {
+            if (isStreamable())
+            {
+                return chunkedResponseIterator.hasContinuation();
+            }
+
             return coreResponse.hasContinuation();
         }
 
@@ -592,6 +729,11 @@ public boolean hasContinuation()
          */
         public BinaryValue getContinuation()
         {
+            if (isStreamable())
+            {
+                return chunkedResponseIterator.getContinuation();
+            }
+
             return coreResponse.getContinuation();
         }
 
@@ -602,6 +744,11 @@ public BinaryValue getContinuation()
          */
         public boolean hasEntries()
         {
+            if (isStreamable())
+            {
+                return chunkedResponseIterator.hasNext();
+            }
+
             return !coreResponse.getEntryList().isEmpty();
         }
 
@@ -617,9 +764,38 @@ public static Location getLocationFromCoreEntry(Namespace queryLocation,
             return loc;
         }
 
-        public abstract List getEntries();
+        public final List getEntries()
+        {
+            final List coreEntries = coreResponse.getEntryList();
+            final List convertedList = new ArrayList<>(coreEntries.size());
+
+            for (SecondaryIndexQueryOperation.Response.Entry e : coreEntries)
+            {
+                final Location loc = getLocationFromCoreEntry(e);
+                final E ce = createEntry(loc, e, converter);
+                convertedList.add(ce);
+            }
+            return convertedList;
+        }
+
+        /**
+         * Factory method.
+         * @param location
+         * @param coreEntry
+         * @param converter
+         * @return
+         */
+        protected E createEntry(Location location, SecondaryIndexQueryOperation.Response.Entry coreEntry, IndexConverter converter)
+        {
+            return (E)new Entry(location, coreEntry.getIndexKey(), converter);
+        }
+
+        protected final E createEntry(Namespace namespace, SecondaryIndexQueryOperation.Response.Entry coreEntry, IndexConverter converter)
+        {
+            return createEntry(Response.getLocationFromCoreEntry(namespace, coreEntry), coreEntry, converter);
+        }
 
-        public abstract static class Entry
+        public static class Entry
         {
             private final Location RiakObjectLocation;
             private final BinaryValue indexKey;
@@ -655,41 +831,4 @@ public T getIndexKey()
             }
         }
     }
-
-    public abstract static class StreamingResponse implements Iterable
-    {
-        final protected IndexConverter converter;
-        final protected Namespace queryLocation;
-
-        private final ChunkedResponseIterator chunkedResponseIterator;
-
-        protected StreamingResponse(Namespace queryLocation,
-                                    IndexConverter converter,
-                                    ChunkedResponseIterator chunkedResponseIterator)
-        {
-            this.converter = converter;
-            this.queryLocation = queryLocation;
-            this.chunkedResponseIterator = chunkedResponseIterator;
-        }
-
-        public Iterator iterator()
-        {
-            return chunkedResponseIterator;
-        }
-
-        public boolean hasContinuation()
-        {
-            return chunkedResponseIterator.hasContinuation();
-        }
-
-        public BinaryValue getContinuation()
-        {
-            return chunkedResponseIterator.getContinuation();
-        }
-
-        public boolean hasEntries()
-        {
-            return chunkedResponseIterator.hasNext();
-        }
-    }
 }
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/Streaming2iQueryFuture.java b/src/main/java/com/basho/riak/client/api/commands/indexes/Streaming2iQueryFuture.java
deleted file mode 100644
index 126924b0e..000000000
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/Streaming2iQueryFuture.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2016 Basho Technologies, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.basho.riak.client.api.commands.indexes;
-
-import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
-import com.basho.riak.client.core.StreamingRiakFuture;
-import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
-
-/**
- * Streamlined ImmediateCoreFutureAdapter for converting streaming 2i operation results to command results.
- * @param  The converted response type.
- * @param  The converted query info type.
- *
- * @author Alex Moore 
- * @since 2.1.0
- */
-public class Streaming2iQueryFuture
-        extends ImmediateCoreFutureAdapter
-{
-    private S indexQuery;
-
-    public Streaming2iQueryFuture(StreamingRiakFuture coreFuture,
-                                  T immediateResponse,
-                                  S indexQuery)
-    {
-        super(coreFuture, immediateResponse);
-        this.indexQuery = indexQuery;
-    }
-
-    @Override
-    protected S convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo)
-    {
-        return indexQuery;
-    }
-}
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java b/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java
index 7a16d4165..faed8bd48 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java
@@ -16,12 +16,7 @@
 package com.basho.riak.client.api.commands.kv;
 
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
 import com.basho.riak.client.api.commands.indexes.SecondaryIndexQuery;
-import com.basho.riak.client.api.commands.indexes.Streaming2iQueryFuture;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
-import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.FetchOperation;
 import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
 import com.basho.riak.client.core.query.Location;
@@ -29,7 +24,6 @@
 import com.basho.riak.client.core.query.indexes.IndexNames;
 import com.basho.riak.client.core.util.BinaryValue;
 
-import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -49,17 +43,17 @@
  * Note that this command must not be used without coverage context
  * for querying buckets that contain a big amount of data.
  *
- * @author Sergey Galkin 
+ * @author Sergey Galkin 
  * @author Alex Moore 
  * @see CoveragePlan
  */
-public class FullBucketRead extends SecondaryIndexQuery
+public class FullBucketRead extends SecondaryIndexQuery
 {
     private final IndexConverter converter;
 
     protected FullBucketRead(Builder builder)
     {
-        super(builder.get2iBuilder());
+        super(builder.get2iBuilder(), Response::new, Response::new);
         this.converter = new IndexConverter()
         {
             @Override
@@ -76,58 +70,6 @@ protected IndexConverter getConverter()
         return converter;
     }
 
-    @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-                executeCoreAsync(cluster);
-
-        RawQueryFuture future = new RawQueryFuture(coreFuture);
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    @Override
-    protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
-    {
-        StreamingRiakFuture coreFuture =
-                executeCoreAsyncStreaming(cluster);
-
-        StreamingResponse response = new StreamingResponse(namespace, converter, coreFuture, timeout);
-
-        Streaming2iQueryFuture future =
-                new Streaming2iQueryFuture<>(coreFuture, response, this);
-
-        coreFuture.addListener(future);
-
-        return future;
-    }
-
-    protected final class RawQueryFuture
-            extends CoreFutureAdapter
-    {
-        public RawQueryFuture(RiakFuture coreFuture)
-        {
-            super(coreFuture);
-        }
-
-        @Override
-        protected Response convertResponse(SecondaryIndexQueryOperation.Response coreResponse)
-        {
-            return new Response(namespace, coreResponse, converter);
-        }
-
-        @Override
-        protected FullBucketRead convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo)
-        {
-            return FullBucketRead.this;
-        }
-    }
-
     /**
      * Builder used to construct a FullBucketRead command.
      */
@@ -255,49 +197,32 @@ public FullBucketRead build()
         }
     }
 
-    public static class Response extends SecondaryIndexQuery.Response
+    public static class Response extends SecondaryIndexQuery.Response
     {
         private transient List convertedList = null;
 
-        protected Response(Namespace queryLocation,
-                           SecondaryIndexQueryOperation.Response coreResponse,
-                           SecondaryIndexQuery.IndexConverter converter)
-        {
-            super(queryLocation, coreResponse, converter);
+        protected Response(Namespace queryLocation, IndexConverter converter, ChunkedResponseIterator chunkedResponseIterator) {
+            super(queryLocation, converter, chunkedResponseIterator);
         }
 
-        @Override
-        public List getEntries()
-        {
-            if (convertedList != null)
-            {
-                return convertedList;
-            }
-
-            convertedList = new ArrayList<>(coreResponse.getEntryList().size());
-            for (SecondaryIndexQueryOperation.Response.Entry e : coreResponse.getEntryList())
-            {
-                Entry ce = createKeyValueEntry(queryLocation, e);
-                convertedList.add(ce);
-            }
-            return convertedList;
+        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter) {
+            super(queryLocation, coreResponse, converter);
         }
 
-        static Entry createKeyValueEntry(Namespace namespace, SecondaryIndexQueryOperation.Response.Entry e)
+        @Override
+        protected Entry createEntry(Location location, SecondaryIndexQueryOperation.Response.Entry coreEntry, IndexConverter converter)
         {
-            final Location loc = getLocationFromCoreEntry(namespace, e);
-
             final FetchValue.Response fr;
-            if (e.hasBody())
+            if (coreEntry.hasBody())
             {
-                FetchOperation.Response resp = e.getBody();
+                FetchOperation.Response resp = coreEntry.getBody();
 
                 // The following code has been copied from the FetchValue.executeAsync - CoreFutureAdapter
                 fr = new FetchValue.Response.Builder()
                     .withNotFound(resp.isNotFound())
                     .withUnchanged(resp.isUnchanged())
                     .withValues(resp.getObjectList())
-                    .withLocation(loc) // for ORM
+                    .withLocation(location) // for ORM
                     .build();
             }
             else
@@ -305,13 +230,12 @@ static Entry createKeyValueEntry(Namespace namespace, SecondaryIndexQueryOperati
                 fr = null;
             }
 
-            return new Entry(loc, fr);
+            return new Entry(location, fr);
         }
 
-        public static class Entry
+        public static class Entry extends SecondaryIndexQuery.Response.Entry
         {
             private final FetchValue.Response fetchedValue;
-            private final Location location;
 
             public Entry(Location location)
             {
@@ -320,8 +244,8 @@ public Entry(Location location)
 
             public Entry(Location location, FetchValue.Response fetchedResponse)
             {
+                super(location, null, null);
                 this.fetchedValue = fetchedResponse;
-                this.location = location;
             }
 
             public boolean hasFetchedValue()
@@ -334,51 +258,14 @@ public FetchValue.Response getFetchedValue()
                 return fetchedValue;
             }
 
-            public Location getLocation()
-            {
-                return location;
-            }
-
             @Override
             public String toString()
             {
                 return "FullBucketRead.Response.Entry{" +
-                        "location=" + location +
+                        "location=" + getRiakObjectLocation() +
                         ", hasFetchedValue=" + hasFetchedValue() +
                         '}';
             }
         }
     }
-
-    public static class StreamingResponse extends SecondaryIndexQuery.StreamingResponse
-    {
-        StreamingResponse(Namespace namespace, IndexConverter converter,
-                          StreamingRiakFuture coreFuture,
-                          int pollTimeout)
-        {
-            super(namespace, converter, createResponseIterator(namespace, converter, coreFuture, pollTimeout));
-        }
-
-        private static ChunkedResponseIterator createResponseIterator(
-                Namespace namespace,
-                IndexConverter converter,
-                StreamingRiakFuture coreFuture,
-                int pollTimeout)
-        {
-            return new ChunkedResponseIterator<>(
-                    coreFuture,
-                    pollTimeout,
-                    (e) -> createExternalResponseEntry(namespace, e, converter),
-                    SecondaryIndexQueryOperation.Response::iterator,
-                    SecondaryIndexQueryOperation.Response::getContinuation);
-        }
-
-        private static Response.Entry createExternalResponseEntry(Namespace namespace,
-                                                                  SecondaryIndexQueryOperation.Response.Entry baseEntry,
-                                                                  IndexConverter unused)
-        {
-            return Response.createKeyValueEntry(namespace, baseEntry);
-        }
-    }
 }
diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java
index b0f4db43f..3f311253d 100644
--- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java
+++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java
@@ -101,10 +101,10 @@ public void testStreamingRangeQuery() throws ExecutionException, InterruptedExce
                                                                    INDEX_ENTRY,
                                                                    INDEX_ENTRY2).withKeyAndIndex(true).build();
 
-        final RiakFuture streamingFuture =
+        final RiakFuture streamingFuture =
                 client.executeAsyncStreaming(indexQuery, 200);
 
-        final BigIntIndexQuery.StreamingResponse streamingResponse = streamingFuture.get();
+        final BigIntIndexQuery.Response streamingResponse = streamingFuture.get();
 
         assertTrue(streamingResponse.hasEntries());
 
diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java
index 19874fd49..7ad99e301 100644
--- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java
+++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java
@@ -99,10 +99,10 @@ public void testStreamingRangeQuery() throws ExecutionException, InterruptedExce
                                                              indexKey(0),
                                                              indexKey(9)).withKeyAndIndex(true).build();
 
-        final RiakFuture streamingFuture =
+        final RiakFuture streamingFuture =
                 client.executeAsyncStreaming(indexQuery, 200);
 
-        final BinIndexQuery.StreamingResponse streamingResponse = streamingFuture.get();
+        final BinIndexQuery.Response streamingResponse = streamingFuture.get();
 
         assertTrue(streamingResponse.hasEntries());
 
@@ -125,7 +125,7 @@ public void testStreamingRangeQuery() throws ExecutionException, InterruptedExce
         assertEquals(3, size);
     }
 
-    private boolean assertFirstObjectFound(Iterable entries)
+    private boolean assertFirstObjectFound(Iterable> entries)
     {
         final String expectedObjectKey = objectKey(1);
         final String expectedIndexKey = indexKey(1);
diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestFullBucketRead.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestFullBucketRead.java
index 5c5e73ae9..792ae119a 100644
--- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestFullBucketRead.java
+++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestFullBucketRead.java
@@ -110,7 +110,7 @@ public void readPlainTextValues() throws ExecutionException, InterruptedExceptio
         {
             assertFalse(e.hasFetchedValue());
             assertNull(e.getFetchedValue());
-            returnedKeys.add(e.getLocation().getKeyAsString());
+            returnedKeys.add(e.getRiakObjectLocation().getKeyAsString());
         }
 
         assertEquals(NUMBER_OF_TEST_VALUES, returnedKeys.size());
@@ -134,7 +134,7 @@ public void readPlainTextValuesWithReturnBody()
             assertTrue(e.hasFetchedValue());
             final RiakObject ro = e.getFetchedValue().getValue(RiakObject.class);
 
-            final int expectedValue = Integer.parseInt(e.getLocation().getKeyAsString().substring(1));
+            final int expectedValue = Integer.parseInt(e.getRiakObjectLocation().getKeyAsString().substring(1));
             assertEquals("v" + expectedValue, ro.getValue().toString());
 
             assertEquals("plain/text", ro.getContentType());
@@ -354,7 +354,7 @@ private Map transformChunkedKeysToRiakObjects(Map streamingFuture =
-                client.executeAsyncStreaming(indexQuery, 200);
+        final RiakFuture streamingFuture =
+                client.executeAsyncStreaming((StreamableRiakCommand) indexQuery, 200);
 
-        final IntIndexQuery.StreamingResponse streamingResponse = streamingFuture.get();
+        final IntIndexQuery.Response streamingResponse = streamingFuture.get();
 
         assertTrue(streamingResponse.hasEntries());
 
diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java
index da58194f6..723972f8a 100644
--- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java
+++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java
@@ -107,7 +107,7 @@ public void simpleTest() throws InterruptedException, ExecutionException
         RawIndexQuery.Response iResp = client.execute(biq);
 
         assertTrue(iResp.hasEntries());
-        RawIndexQuery.Response.Entry first = iResp.getEntries().iterator().next();
+        RawIndexQuery.Response.Entry first = iResp.getEntries().iterator().next();
         assertEquals(ip.key, first.getRiakObjectLocation().getKey().toString());
         assertArrayEquals(ip.indexKey, first.getIndexKey().getValue());
     }
@@ -176,10 +176,10 @@ public void testBucketIndexQueryStreaming() throws InterruptedException, Executi
 
         BucketIndexQuery bq = new BucketIndexQuery.Builder(sharedNamespace).build();
 
-        final RiakFuture indexResult =
+        final RiakFuture indexResult =
                 client.executeAsyncStreaming(bq, 100);
 
-        final BinIndexQuery.StreamingResponse streamingResponse = indexResult.get();
+        final BinIndexQuery.Response streamingResponse = indexResult.get();
 
         assertTrue(streamingResponse.hasEntries());
         assertEquals(100, StreamSupport.stream(streamingResponse.spliterator(), false).count());
@@ -195,10 +195,10 @@ public void testIndexQueryStreamingContinuations() throws InterruptedException,
 
         BucketIndexQuery bq = new BucketIndexQuery.Builder(sharedNamespace).withMaxResults(50).withPaginationSort(true).build();
 
-        final RiakFuture indexResult =
+        final RiakFuture indexResult =
                 client.executeAsyncStreaming(bq, 100);
 
-        final BinIndexQuery.StreamingResponse streamingResponse = indexResult.get();
+        final BinIndexQuery.Response streamingResponse = indexResult.get();
 
         assertTrue(streamingResponse.hasEntries());
         assertEquals(50, StreamSupport.stream(streamingResponse.spliterator(), false).count());
diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestCoveragePlan.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestCoveragePlan.java
index 02c26ce9e..ed5b5090a 100644
--- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestCoveragePlan.java
+++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestCoveragePlan.java
@@ -132,7 +132,7 @@ public void fetchAllDataByUsingCoverageContext() throws ExecutionException, Inte
             logger.info(sbld.toString());
         }
 
-        final Map> chunkedKeys
+        final Map>> chunkedKeys
                 = new HashMap<>();
 
         for (HostAndPort host: response.hosts())
@@ -170,7 +170,7 @@ public void fetchAllDataByUsingCoverageContext() throws ExecutionException, Inte
         }
 
         final Set keys = new HashSet<>(NUMBER_OF_TEST_VALUES);
-        for (Map.Entry> e: chunkedKeys.entrySet())
+        for (Map.Entry>> e: chunkedKeys.entrySet())
         {
             final CoverageEntry ce = e.getKey();
             if (e.getValue().isEmpty())

From 37c6d55fcb026a2a32801284660fd5db37bd8de7 Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Thu, 3 Nov 2016 06:29:10 +0200
Subject: [PATCH 32/55] Fix some unchecked cast warnings

---
 .../basho/riak/client/api/commands/indexes/BinIndexQuery.java | 4 ++--
 .../client/api/commands/indexes/itest/ITestIntIndexQuery.java | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java
index 1a65a2374..4bb1c1ed7 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java
@@ -237,11 +237,11 @@ public BinIndexQuery build()
     public static class Response extends SecondaryIndexQuery.Response>
     {
 
-        protected Response(Namespace queryLocation, IndexConverter converter, ChunkedResponseIterator chunkedResponseIterator) {
+        protected Response(Namespace queryLocation, IndexConverter converter, ChunkedResponseIterator chunkedResponseIterator) {
             super(queryLocation, converter, chunkedResponseIterator);
         }
 
-        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter) {
+        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter) {
             super(queryLocation, coreResponse, converter);
         }
     }
diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java
index 194e7fc39..a33c6e9b8 100644
--- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java
+++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java
@@ -108,7 +108,7 @@ public void testStreamingRangeQuery() throws ExecutionException, InterruptedExce
                                                              Long.MAX_VALUE).withKeyAndIndex(true).build();
 
         final RiakFuture streamingFuture =
-                client.executeAsyncStreaming((StreamableRiakCommand) indexQuery, 200);
+                client.executeAsyncStreaming(indexQuery, 200);
 
         final IntIndexQuery.Response streamingResponse = streamingFuture.get();
 

From a112bf203e1fcb9f123ca73b65fc2200b4fc0a64 Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Thu, 3 Nov 2016 06:30:34 +0200
Subject: [PATCH 33/55] Coarse fix of unchecked cast warnings in
 SecondaryIndexQuery

---
 .../riak/client/api/commands/indexes/SecondaryIndexQuery.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
index 47d8cf242..4e6865adf 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
@@ -697,6 +697,7 @@ public boolean isStreamable()
             return chunkedResponseIterator != null;
         }
 
+        @SuppressWarnings("unchecked")
         public Iterator iterator()
         {
             if (isStreamable()) {
@@ -785,6 +786,7 @@ public final List getEntries()
          * @param converter
          * @return
          */
+        @SuppressWarnings("unchecked")
         protected E createEntry(Location location, SecondaryIndexQueryOperation.Response.Entry coreEntry, IndexConverter converter)
         {
             return (E)new Entry(location, coreEntry.getIndexKey(), converter);

From 0f62ed97a4eeb9077a5e0cd1d68449b25172ccd0 Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Thu, 3 Nov 2016 08:07:40 +0200
Subject: [PATCH 34/55] Get rid of Response.getLocationFromCoreEntity().

---
 .../commands/indexes/SecondaryIndexQuery.java  | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
index 4e6865adf..91ad1b1be 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
@@ -753,18 +753,6 @@ public boolean hasEntries()
             return !coreResponse.getEntryList().isEmpty();
         }
 
-        protected final Location getLocationFromCoreEntry(SecondaryIndexQueryOperation.Response.Entry e)
-        {
-            return Response.getLocationFromCoreEntry(this.queryLocation, e);
-        }
-
-        public static Location getLocationFromCoreEntry(Namespace queryLocation,
-                                                        SecondaryIndexQueryOperation.Response.Entry e)
-        {
-            final Location loc = new Location(queryLocation, e.getObjectKey());
-            return loc;
-        }
-
         public final List getEntries()
         {
             final List coreEntries = coreResponse.getEntryList();
@@ -772,8 +760,7 @@ public final List getEntries()
 
             for (SecondaryIndexQueryOperation.Response.Entry e : coreEntries)
             {
-                final Location loc = getLocationFromCoreEntry(e);
-                final E ce = createEntry(loc, e, converter);
+                final E ce = createEntry(queryLocation, e, converter);
                 convertedList.add(ce);
             }
             return convertedList;
@@ -794,7 +781,8 @@ protected E createEntry(Location location, SecondaryIndexQueryOperation.Response
 
         protected final E createEntry(Namespace namespace, SecondaryIndexQueryOperation.Response.Entry coreEntry, IndexConverter converter)
         {
-            return createEntry(Response.getLocationFromCoreEntry(namespace, coreEntry), coreEntry, converter);
+            final Location loc = new Location(queryLocation, coreEntry.getObjectKey());
+            return createEntry(loc, coreEntry, converter);
         }
 
         public static class Entry

From 576eccf508725b913485bd27d6d64683bc1e64db Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Sun, 6 Nov 2016 07:46:41 +0200
Subject: [PATCH 35/55] Fix upper case in variable name

---
 .../client/api/commands/indexes/SecondaryIndexQuery.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
index 91ad1b1be..5c274428d 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
@@ -787,13 +787,13 @@ protected final E createEntry(Namespace namespace, SecondaryIndexQueryOperation.
 
         public static class Entry
         {
-            private final Location RiakObjectLocation;
+            private final Location riakObjectLocation;
             private final BinaryValue indexKey;
             private final IndexConverter converter;
 
             protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverter converter)
             {
-                this.RiakObjectLocation = riakObjectLocation;
+                this.riakObjectLocation = riakObjectLocation;
                 this.indexKey = indexKey;
                 this.converter = converter;
             }
@@ -805,7 +805,7 @@ protected Entry(Location riakObjectLocation, BinaryValue indexKey, IndexConverte
              */
             public Location getRiakObjectLocation()
             {
-                return RiakObjectLocation;
+                return riakObjectLocation;
             }
 
             /**

From 9630a114c27c25f7fb1984feaee645ba331b9ed5 Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Mon, 7 Nov 2016 16:37:39 +0200
Subject: [PATCH 36/55] Simplification of a bunch of RiakCommand descendants

---
 .../riak/client/api/GenericRiakCommand.java   | 87 +++++++++++++++++
 .../com/basho/riak/client/api/RiakClient.java |  3 +-
 .../client/api/StreamableRiakCommand.java     | 21 +++-
 .../buckets/FetchBucketProperties.java        | 34 +------
 .../api/commands/buckets/ListBuckets.java     | 32 ++-----
 .../buckets/ResetBucketProperties.java        | 34 +------
 .../buckets/StoreBucketProperties.java        | 33 +------
 .../api/commands/datatypes/FetchCounter.java  | 41 ++------
 .../api/commands/datatypes/FetchDatatype.java |  5 +-
 .../api/commands/datatypes/FetchHll.java      | 31 +-----
 .../api/commands/datatypes/FetchMap.java      | 41 ++------
 .../api/commands/datatypes/FetchSet.java      | 41 ++------
 .../api/commands/datatypes/UpdateCounter.java | 66 ++++---------
 .../commands/datatypes/UpdateDatatype.java    | 27 +++++-
 .../api/commands/datatypes/UpdateHll.java     | 66 ++++---------
 .../api/commands/datatypes/UpdateMap.java     | 66 ++++---------
 .../api/commands/datatypes/UpdateSet.java     | 66 ++++---------
 .../commands/indexes/SecondaryIndexQuery.java | 57 ++++-------
 .../client/api/commands/kv/CoveragePlan.java  | 33 ++-----
 .../client/api/commands/kv/DeleteValue.java   | 33 ++-----
 .../client/api/commands/kv/FetchValue.java    | 42 +++-----
 .../riak/client/api/commands/kv/ListKeys.java | 31 ++----
 .../api/commands/mapreduce/MapReduce.java     | 96 ++++++++-----------
 .../api/commands/search/DeleteIndex.java      | 35 +------
 .../api/commands/search/FetchIndex.java       | 30 +-----
 .../api/commands/search/FetchSchema.java      | 58 ++++++-----
 .../client/api/commands/search/Search.java    | 34 +------
 .../api/commands/search/StoreIndex.java       | 33 +------
 .../api/commands/search/StoreSchema.java      | 34 +------
 .../api/commands/timeseries/ListKeys.java     | 34 +------
 .../indexes/SecondaryIndexQueryTest.java      | 12 +--
 .../commands/itest/ITestBucketMapReduce.java  |  5 +-
 32 files changed, 405 insertions(+), 856 deletions(-)
 create mode 100644 src/main/java/com/basho/riak/client/api/GenericRiakCommand.java

diff --git a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
new file mode 100644
index 000000000..4f52c9f73
--- /dev/null
+++ b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
@@ -0,0 +1,87 @@
+package com.basho.riak.client.api;
+
+/**
+ * Created by srg on 11/6/16.
+ */
+
+import com.basho.riak.client.api.commands.CoreFutureAdapter;
+import com.basho.riak.client.core.FutureOperation;
+import com.basho.riak.client.core.PBStreamingFutureOperation;
+import com.basho.riak.client.core.RiakCluster;
+import com.basho.riak.client.core.RiakFuture;
+
+
+public abstract class GenericRiakCommand extends RiakCommand
+{
+    @FunctionalInterface
+    protected interface Converter
+    {
+        T convert(O source);
+    }
+
+    private Converter responseConverter;
+    private Converter infoConverter;
+
+    public GenericRiakCommand()
+    {
+        this.responseConverter = this::convertResponse;
+        this.infoConverter = this::convertInfo;
+    }
+
+    public GenericRiakCommand(final Converter responseConverter, final Converter infoConverter)
+    {
+        this.responseConverter = responseConverter;
+        this.infoConverter = infoConverter;
+    }
+
+    protected abstract FutureOperation buildCoreOperation();
+
+    protected RiakFuture executeAsync(RiakCluster cluster)
+    {
+        final FutureOperation coreOperation = buildCoreOperation();
+        assert coreOperation != null;
+
+        // TODO: WE NEED TO GET RID SUCH A WEIRD IF-FORK
+        final RiakFuture coreFuture;
+        if (coreOperation instanceof PBStreamingFutureOperation)
+        {
+            coreFuture = cluster.execute((PBStreamingFutureOperation) coreOperation);
+        }
+        else
+        {
+            coreFuture = cluster.execute(coreOperation);
+        }
+
+        assert coreFuture != null;
+
+        final CoreFutureAdapter future =
+                new CoreFutureAdapter(coreFuture)
+                {
+                    @Override
+                    protected R convertResponse(CoreR coreResponse)
+                    {
+                        return responseConverter.convert(coreResponse);
+                    }
+
+                    @Override
+                    protected I convertQueryInfo(CoreI coreQueryInfo)
+                    {
+                        return infoConverter.convert(coreQueryInfo);
+                    }
+                };
+        coreFuture.addListener(future);
+        return future;
+    }
+
+    @SuppressWarnings("unchecked")
+    protected R convertResponse(CoreR coreResponse)
+    {
+        return (R)coreResponse;
+    }
+
+    @SuppressWarnings("unchecked")
+    protected I convertInfo(CoreI coreInfo)
+    {
+        return (I)coreInfo;
+    }
+}
\ No newline at end of file
diff --git a/src/main/java/com/basho/riak/client/api/RiakClient.java b/src/main/java/com/basho/riak/client/api/RiakClient.java
index 1e91392cb..71bfc3584 100644
--- a/src/main/java/com/basho/riak/client/api/RiakClient.java
+++ b/src/main/java/com/basho/riak/client/api/RiakClient.java
@@ -18,7 +18,6 @@
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.RiakNode;
-import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.util.HostAndPort;
 
 import java.net.InetSocketAddress;
@@ -436,7 +435,7 @@ public  RiakFuture executeAsync(RiakCommand command)
      * @return a RiakFuture for the operation
      * @see RiakFuture
      */
-    public  RiakFuture executeAsyncStreaming(StreamableRiakCommand command, int timeoutMS)
+    public  RiakFuture executeAsyncStreaming(StreamableRiakCommand command, int timeoutMS)
     {
         return command.executeAsyncStreaming(cluster, timeoutMS);
     }
diff --git a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
index 5522f0caf..6cc36e91f 100644
--- a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
@@ -16,6 +16,7 @@
 
 package com.basho.riak.client.api;
 
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.RiakFuture;
 
@@ -27,12 +28,26 @@
  * that data will flow into.
  * @param  The response type returned by "streaming mode" {@link executeAsyncStreaming}
  * @param  The response type returned by the "batch mode" @{link executeAsync}
- * @param  The query info type
+ * @param  The query info type
  * @author Dave Rusek
  * @author Brian Roach 
  * @since 2.0
  */
-public abstract class StreamableRiakCommand extends RiakCommand
+public abstract class StreamableRiakCommand extends GenericRiakCommand
 {
-    protected abstract RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout);
+    public StreamableRiakCommand() {
+    }
+
+    public StreamableRiakCommand(Converter responseConverter, Converter infoConverter) {
+        super(responseConverter, infoConverter);
+    }
+
+    protected abstract RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout);
+
+    @Override
+    protected final FutureOperation buildCoreOperation() {
+        return buildCoreOperation(false);
+    }
+
+    protected abstract  FutureOperation buildCoreOperation(boolean streamResults);
 }
diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/FetchBucketProperties.java b/src/main/java/com/basho/riak/client/api/commands/buckets/FetchBucketProperties.java
index a4c393975..4b521825c 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/FetchBucketProperties.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/FetchBucketProperties.java
@@ -16,10 +16,7 @@
 
 package com.basho.riak.client.api.commands.buckets;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.core.operations.FetchBucketPropsOperation;
 import com.basho.riak.client.core.query.Namespace;
 
@@ -39,7 +36,8 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class FetchBucketProperties extends RiakCommand
+public final class FetchBucketProperties extends GenericRiakCommand
 {
     private final Namespace namespace;
 
@@ -49,31 +47,7 @@ public FetchBucketProperties(Builder builder)
     }
 
     @Override
-    protected final RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected FetchBucketPropsOperation.Response convertResponse(FetchBucketPropsOperation.Response coreResponse)
-                {
-                    return coreResponse;
-                }
-
-                @Override
-                protected Namespace convertQueryInfo(Namespace coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    private FetchBucketPropsOperation buildCoreOperation()
+    protected FetchBucketPropsOperation buildCoreOperation()
     {
         return new FetchBucketPropsOperation.Builder(namespace).build();
     }
diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
index 865e749f3..c010d6ce8 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
@@ -16,7 +16,6 @@
 package com.basho.riak.client.api.commands.buckets;
 
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
 import com.basho.riak.client.api.StreamableRiakCommand;
 import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
 import com.basho.riak.client.core.RiakCluster;
@@ -64,7 +63,8 @@
  * @author Alex Moore 
  * @since 2.0
  */
-public final class ListBuckets extends StreamableRiakCommand
+public final class ListBuckets extends StreamableRiakCommand
 {
     private final int timeout;
     private final BinaryValue type;
@@ -76,30 +76,11 @@ public final class ListBuckets extends StreamableRiakCommand executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation(false));
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected Response convertResponse(ListBucketsOperation.Response coreResponse)
-                {
-                    return new Response(type, coreResponse.getBuckets());
-                }
-
-                @Override
-                protected BinaryValue convertQueryInfo(BinaryValue coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+    protected Response convertResponse(ListBucketsOperation.Response coreResponse) {
+        return new Response(type, coreResponse.getBuckets());
     }
 
+    @Override
     protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
     {
         StreamingRiakFuture coreFuture =
@@ -115,7 +96,8 @@ protected RiakFuture executeAsyncStreaming(RiakC
         return future;
     }
 
-    private ListBucketsOperation buildCoreOperation(boolean streamResults)
+    @Override
+    protected ListBucketsOperation buildCoreOperation(boolean streamResults)
     {
         ListBucketsOperation.Builder builder = new ListBucketsOperation.Builder();
         if (timeout > 0)
diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ResetBucketProperties.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ResetBucketProperties.java
index f952bab09..cce63bbef 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/ResetBucketProperties.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ResetBucketProperties.java
@@ -1,9 +1,6 @@
 package com.basho.riak.client.api.commands.buckets;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.core.operations.ResetBucketPropsOperation;
 import com.basho.riak.client.core.query.Namespace;
 
@@ -21,7 +18,7 @@
  * @author Chris Mancini 
  * @since 2.0
  */
-public class ResetBucketProperties extends RiakCommand
+public class ResetBucketProperties extends GenericRiakCommand
 {
     private final Namespace namespace;
 
@@ -31,32 +28,7 @@ public ResetBucketProperties(Builder builder)
     }
 
     @Override
-    protected final RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-                cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-                new CoreFutureAdapter(coreFuture)
-                {
-                    @Override
-                    protected Void convertResponse(Void coreResponse)
-                    {
-                        return coreResponse;
-                    }
-
-                    @Override
-                    protected Namespace convertQueryInfo(Namespace coreQueryInfo)
-                    {
-                        return coreQueryInfo;
-                    }
-                };
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    private ResetBucketPropsOperation buildCoreOperation()
-    {
+    protected ResetBucketPropsOperation buildCoreOperation() {
         ResetBucketPropsOperation.Builder builder =
                 new ResetBucketPropsOperation.Builder(namespace);
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/StoreBucketProperties.java b/src/main/java/com/basho/riak/client/api/commands/buckets/StoreBucketProperties.java
index 14cd54d72..52f71895e 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/StoreBucketProperties.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/StoreBucketProperties.java
@@ -16,10 +16,7 @@
 
 package com.basho.riak.client.api.commands.buckets;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.core.operations.StoreBucketPropsOperation;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.query.functions.Function;
@@ -39,7 +36,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class StoreBucketProperties extends RiakCommand
+public final class StoreBucketProperties extends GenericRiakCommand
 {
     private final Namespace namespace;
     private final Boolean allowMulti;
@@ -95,31 +92,7 @@ public final class StoreBucketProperties extends RiakCommand
     }
 
     @Override
-    protected final RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected Void convertResponse(Void coreResponse)
-                {
-                    return coreResponse;
-                }
-
-                @Override
-                protected Namespace convertQueryInfo(Namespace coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    private StoreBucketPropsOperation buildCoreOperation()
+    protected StoreBucketPropsOperation buildCoreOperation()
     {
         StoreBucketPropsOperation.Builder builder =
             new StoreBucketPropsOperation.Builder(namespace);
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchCounter.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchCounter.java
index 815d80fac..b47df947b 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchCounter.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchCounter.java
@@ -16,9 +16,6 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.operations.DtFetchOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.crdt.types.RiakCounter;
@@ -48,38 +45,18 @@ private FetchCounter(Builder builder)
     }
 
     @Override
-    protected final RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected FetchCounter.Response convertResponse(DtFetchOperation.Response coreResponse)
-                {
-                    RiakDatatype element = coreResponse.getCrdtElement();
-
-                    Context context = null;
-                    if (coreResponse.hasContext())
-                    {
-                        context = new Context(coreResponse.getContext());
-                    }
+    protected Response convertResponse(DtFetchOperation.Response coreResponse) {
+        RiakDatatype element = coreResponse.getCrdtElement();
 
-                    RiakCounter datatype = extractDatatype(element);
+        Context context = null;
+        if (coreResponse.hasContext())
+        {
+            context = new Context(coreResponse.getContext());
+        }
 
-                    return new Response(datatype, context);
-                }
+        RiakCounter datatype = extractDatatype(element);
 
-                @Override
-                protected Location convertQueryInfo(Location coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+        return new Response(datatype, context);
     }
 
     @Override
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchDatatype.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchDatatype.java
index f6320346f..a77b50716 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchDatatype.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchDatatype.java
@@ -15,7 +15,7 @@
  */
 package com.basho.riak.client.api.commands.datatypes;
 
-import com.basho.riak.client.api.RiakCommand;
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.api.cap.Quorum;
 import com.basho.riak.client.api.commands.RiakOption;
 import com.basho.riak.client.core.operations.DtFetchOperation;
@@ -31,7 +31,7 @@
   * @author Dave Rusek 
   * @since 2.0
   */
-public abstract class FetchDatatype extends RiakCommand
+public abstract class FetchDatatype extends GenericRiakCommand
 {
     private final Location location;
     private final Map, Object> options = new HashMap<>();
@@ -56,6 +56,7 @@ public  FetchDatatype withOption(Option option, V value)
 
     public abstract T extractDatatype(RiakDatatype element);
 
+    @Override
     protected final DtFetchOperation buildCoreOperation()
     {
         DtFetchOperation.Builder builder =
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java
index 169f54f8e..b5e0b5fa3 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java
@@ -16,9 +16,6 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.operations.DtFetchOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.crdt.types.RiakDatatype;
@@ -49,32 +46,12 @@ private FetchHll(Builder builder)
     }
 
     @Override
-    protected final RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected RiakHll convertResponse(DtFetchOperation.Response coreResponse)
-                {
-                    RiakDatatype element = coreResponse.getCrdtElement();
-
-                    RiakHll datatype = extractDatatype(element);
+    protected RiakHll convertResponse(DtFetchOperation.Response coreResponse) {
+        RiakDatatype element = coreResponse.getCrdtElement();
 
-                    return datatype;
-                }
+        RiakHll datatype = extractDatatype(element);
 
-                @Override
-                protected Location convertQueryInfo(Location coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+        return datatype;
     }
 
     @Override
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchMap.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchMap.java
index b5ca89daf..627947117 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchMap.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchMap.java
@@ -16,9 +16,6 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.operations.DtFetchOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.crdt.types.RiakDatatype;
@@ -56,38 +53,18 @@ public RiakMap extractDatatype(RiakDatatype element)
     }
 
     @Override
-    protected final RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected FetchMap.Response convertResponse(DtFetchOperation.Response coreResponse)
-                {
-                    RiakDatatype element = coreResponse.getCrdtElement();
-
-                    Context context = null;
-                    if (coreResponse.hasContext())
-                    {
-                        context = new Context(coreResponse.getContext());
-                    }
+    protected Response convertResponse(DtFetchOperation.Response coreResponse) {
+        RiakDatatype element = coreResponse.getCrdtElement();
 
-                    RiakMap datatype = extractDatatype(element);
+        Context context = null;
+        if (coreResponse.hasContext())
+        {
+            context = new Context(coreResponse.getContext());
+        }
 
-                    return new Response(datatype, context);
-                }
+        RiakMap datatype = extractDatatype(element);
 
-                @Override
-                protected Location convertQueryInfo(Location coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+        return new Response(datatype, context);
     }
 
     /**
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchSet.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchSet.java
index 3e2cf7e47..34d9b1763 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchSet.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchSet.java
@@ -16,9 +16,6 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.operations.DtFetchOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.crdt.types.RiakDatatype;
@@ -49,38 +46,18 @@ private FetchSet(Builder builder)
     }
 
     @Override
-    protected final RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected FetchSet.Response convertResponse(DtFetchOperation.Response coreResponse)
-                {
-                    RiakDatatype element = coreResponse.getCrdtElement();
-
-                    Context context = null;
-                    if (coreResponse.hasContext())
-                    {
-                        context = new Context(coreResponse.getContext());
-                    }
+    protected Response convertResponse(DtFetchOperation.Response coreResponse) {
+        RiakDatatype element = coreResponse.getCrdtElement();
 
-                    RiakSet datatype = extractDatatype(element);
+        Context context = null;
+        if (coreResponse.hasContext())
+        {
+            context = new Context(coreResponse.getContext());
+        }
 
-                    return new Response(datatype, context);
-                }
+        RiakSet datatype = extractDatatype(element);
 
-                @Override
-                protected Location convertQueryInfo(Location coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+        return new Response(datatype, context);
     }
 
     @Override
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateCounter.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateCounter.java
index 38f794c71..6c4c269d3 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateCounter.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateCounter.java
@@ -16,9 +16,6 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.operations.DtUpdateOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
@@ -51,53 +48,30 @@
  */
 public class UpdateCounter extends UpdateDatatype
 {
-    private final CounterUpdate update;
-
     private UpdateCounter(Builder builder)
     {
         super(builder);
-        this.update = builder.update;
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation(update));
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected Response convertResponse(DtUpdateOperation.Response coreResponse)
-                {
-                    RiakCounter counter = null;
-                    if (coreResponse.hasCrdtElement())
-                    {
-                        RiakDatatype element = coreResponse.getCrdtElement();
-                        counter = element.getAsCounter();
-                    }
-                    BinaryValue returnedKey = coreResponse.hasGeneratedKey()
-                        ? coreResponse.getGeneratedKey()
-                        : null;
-
-                    Context returnedCtx = null;
-                    if (coreResponse.hasContext())
-                    {
-                        returnedCtx = new Context(coreResponse.getContext());
-                    }
+    protected Response convertResponse(DtUpdateOperation.Response coreResponse) {
+        RiakCounter counter = null;
+        if (coreResponse.hasCrdtElement())
+        {
+            RiakDatatype element = coreResponse.getCrdtElement();
+            counter = element.getAsCounter();
+        }
+        BinaryValue returnedKey = coreResponse.hasGeneratedKey()
+            ? coreResponse.getGeneratedKey()
+            : null;
 
-                    return new Response(returnedCtx, counter, returnedKey);
-                }
+        Context returnedCtx = null;
+        if (coreResponse.hasContext())
+        {
+            returnedCtx = new Context(coreResponse.getContext());
+        }
 
-                @Override
-                protected Location convertQueryInfo(Location coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+        return new Response(returnedCtx, counter, returnedKey);
     }
 
     /**
@@ -116,8 +90,6 @@ private Response(Context context, RiakCounter datatype, BinaryValue generatedKey
      */
     public static final class Builder extends UpdateDatatype.Builder
     {
-        private final CounterUpdate update;
-
         /**
          * Construct a Builder for an UpdateCounter command.
          * @param location the location of the counter in Riak.
@@ -125,12 +97,11 @@ public static final class Builder extends UpdateDatatype.Builder
          */
         public Builder(Location location, CounterUpdate update)
         {
-            super(location);
+            super(location, update);
             if (update == null)
             {
                 throw new IllegalArgumentException("Update cannot be null");
             }
-            this.update = update;
         }
 
         /**
@@ -146,12 +117,11 @@ public Builder(Location location, CounterUpdate update)
          */
         public Builder(Namespace namespace, CounterUpdate update)
         {
-            super(namespace);
+            super(namespace, update);
             if (update == null)
             {
                 throw new IllegalArgumentException("Update cannot be null");
             }
-            this.update = update;
         }
 
         /**
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateDatatype.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateDatatype.java
index 8e7f05d25..5ff19e743 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateDatatype.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateDatatype.java
@@ -16,9 +16,10 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
-import com.basho.riak.client.api.RiakCommand;
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.api.cap.Quorum;
 import com.basho.riak.client.api.commands.RiakOption;
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.DtUpdateOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
@@ -34,23 +35,26 @@
  * @author Brian Roach 
  * @since 2.0
  */
-public abstract class UpdateDatatype extends RiakCommand
+public abstract class UpdateDatatype extends GenericRiakCommand
 {
     protected final Namespace namespace;
     protected final BinaryValue key;
     private final Context ctx;
     private final Map, Object> options = new HashMap<>();
+    private final DatatypeUpdate update;
 
     @SuppressWarnings("unchecked")
     UpdateDatatype(Builder builder)
     {
         this.namespace = builder.namespace;
+        this.update = builder.update;
         this.key = builder.key;
         this.ctx = builder.ctx;
         this.options.putAll(builder.options);
     }
 
-    protected final DtUpdateOperation buildCoreOperation(DatatypeUpdate update)
+    @Override
+    protected FutureOperation buildCoreOperation()
     {
         DtUpdateOperation.Builder builder;
 
@@ -152,6 +156,7 @@ public Option(String name)
     */
     public static abstract class Builder>
     {
+        private final DatatypeUpdate update;
         private final Namespace namespace;
         private BinaryValue key;
         private Context ctx;
@@ -161,7 +166,7 @@ public static abstract class Builder>
          * Constructs a builder for a datatype update.
          * @param location the location of the datatype object in Riak.
          */
-        Builder(Location location)
+        Builder(Location location, DatatypeUpdate update)
         {
             if (location == null)
             {
@@ -169,6 +174,12 @@ public static abstract class Builder>
             }
             this.namespace = location.getNamespace();
             this.key = location.getKey();
+
+            if (update == null)
+            {
+                throw new IllegalArgumentException("Update cannot be null.");
+            }
+            this.update = update;
         }
 
         /**
@@ -181,13 +192,19 @@ public static abstract class Builder>
          * @param namespace the namespace to create the datatype.
          * @see Response#getGeneratedKey()
          */
-        Builder(Namespace namespace)
+        Builder(Namespace namespace, DatatypeUpdate update)
         {
             if (namespace == null)
             {
                 throw new IllegalArgumentException("Namespace cannot be null.");
             }
             this.namespace = namespace;
+
+            if (update == null)
+            {
+                throw new IllegalArgumentException("Update cannot be null.");
+            }
+            this.update = update;
         }
 
         /**
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java
index fa8e81a64..d2ed465f1 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java
@@ -16,9 +16,6 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.operations.DtUpdateOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
@@ -51,51 +48,28 @@
  
 public class UpdateHll extends UpdateDatatype
 {
-    private final HllUpdate update;
-
     private UpdateHll(Builder builder)
     {
         super(builder);
-        this.update = builder.update;
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation(update));
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected Response convertResponse(DtUpdateOperation.Response coreResponse)
-                {
-                    RiakHll hll = null;
-                    if (coreResponse.hasCrdtElement())
-                    {
-                        RiakDatatype element = coreResponse.getCrdtElement();
-                        hll = element.getAsHll();
-                    }
-                    BinaryValue returnedKey = coreResponse.hasGeneratedKey()
-                        ? coreResponse.getGeneratedKey()
-                        : null;
-                    Context returnedCtx = null;
-                    if (coreResponse.hasContext())
-                    {
-                        returnedCtx = new Context(coreResponse.getContext());
-                    }
-                    return new Response(returnedCtx, hll, returnedKey);
-                }
-
-                @Override
-                protected Location convertQueryInfo(Location coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+    protected Response convertResponse(DtUpdateOperation.Response coreResponse) {
+        RiakHll hll = null;
+        if (coreResponse.hasCrdtElement())
+        {
+            RiakDatatype element = coreResponse.getCrdtElement();
+            hll = element.getAsHll();
+        }
+        BinaryValue returnedKey = coreResponse.hasGeneratedKey()
+            ? coreResponse.getGeneratedKey()
+            : null;
+        Context returnedCtx = null;
+        if (coreResponse.hasContext())
+        {
+            returnedCtx = new Context(coreResponse.getContext());
+        }
+        return new Response(returnedCtx, hll, returnedKey);
     }
 
     /**
@@ -103,8 +77,6 @@ protected Location convertQueryInfo(Location coreQueryInfo)
      */
     public static class Builder extends UpdateDatatype.Builder
     {
-        private final HllUpdate update;
-
         /**
          * Construct a Builder for an UpdateHll command.
          * @param location the location of the HyperLogLog in Riak.
@@ -112,12 +84,11 @@ public static class Builder extends UpdateDatatype.Builder
          */
         public Builder(Location location, HllUpdate update)
         {
-            super(location);
+            super(location, update);
             if (update == null)
             {
                 throw new IllegalArgumentException("Update cannot be null");
             }
-            this.update = update;
         }
 
         /**
@@ -132,12 +103,11 @@ public Builder(Location location, HllUpdate update)
          */
         public Builder(Namespace namespace, HllUpdate update)
         {
-            super(namespace);
+            super(namespace, update);
             if (update == null)
             {
                 throw new IllegalArgumentException("Update cannot be null");
             }
-            this.update = update;
         }
 
         /**
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateMap.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateMap.java
index d60f47bae..2865ce887 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateMap.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateMap.java
@@ -16,9 +16,6 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.operations.DtUpdateOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
@@ -52,53 +49,30 @@
  */
 public class UpdateMap extends UpdateDatatype
 {
-    private final MapUpdate update;
-
     private UpdateMap(Builder builder)
     {
         super(builder);
-        this.update = builder.update;
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation(update));
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected Response convertResponse(DtUpdateOperation.Response coreResponse)
-                {
-                    RiakMap map = null;
-                    if (coreResponse.hasCrdtElement())
-                    {
-                        RiakDatatype element = coreResponse.getCrdtElement();
-                        map = element.getAsMap();
-                    }
-                    BinaryValue returnedKey = coreResponse.hasGeneratedKey()
-                        ? coreResponse.getGeneratedKey()
-                        : null;
-
-                    Context returnedCtx = null;
-                    if (coreResponse.hasContext())
-                    {
-                        returnedCtx = new Context(coreResponse.getContext());
-                    }
+    protected Response convertResponse(DtUpdateOperation.Response coreResponse) {
+        RiakMap map = null;
+        if (coreResponse.hasCrdtElement())
+        {
+            RiakDatatype element = coreResponse.getCrdtElement();
+            map = element.getAsMap();
+        }
+        BinaryValue returnedKey = coreResponse.hasGeneratedKey()
+            ? coreResponse.getGeneratedKey()
+            : null;
 
-                    return new Response(returnedCtx, map, returnedKey);
-                }
+        Context returnedCtx = null;
+        if (coreResponse.hasContext())
+        {
+            returnedCtx = new Context(coreResponse.getContext());
+        }
 
-                @Override
-                protected Location convertQueryInfo(Location coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+        return new Response(returnedCtx, map, returnedKey);
     }
 
     public static final class Response extends UpdateDatatype.Response
@@ -114,8 +88,6 @@ private Response(Context context, RiakMap datatype, BinaryValue generatedKey)
      */
     public static final class Builder extends UpdateDatatype.Builder
     {
-        private final MapUpdate update;
-
         /**
          * Construct a Builder for an UpdateMap command.
          * @param location the location of the map in Riak.
@@ -123,12 +95,11 @@ public static final class Builder extends UpdateDatatype.Builder
          */
         public Builder(Location location, MapUpdate update)
         {
-            super(location);
+            super(location, update);
             if (update == null)
             {
                 throw new IllegalArgumentException("Update cannot be null");
             }
-            this.update = update;
         }
 
         /**
@@ -144,12 +115,11 @@ public Builder(Location location, MapUpdate update)
          */
         public Builder(Namespace namespace, MapUpdate update)
         {
-            super(namespace);
+            super(namespace, update);
             if (update == null)
             {
                 throw new IllegalArgumentException("Update cannot be null");
             }
-            this.update = update;
         }
 
         @Override
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateSet.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateSet.java
index 062d49bc0..d370dcfc6 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateSet.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateSet.java
@@ -16,9 +16,6 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.operations.DtUpdateOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
@@ -51,51 +48,28 @@
  */
 public class UpdateSet extends UpdateDatatype
 {
-    private final SetUpdate update;
-
     private UpdateSet(Builder builder)
     {
         super(builder);
-        this.update = builder.update;
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation(update));
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected Response convertResponse(DtUpdateOperation.Response coreResponse)
-                {
-                    RiakSet set = null;
-                    if (coreResponse.hasCrdtElement())
-                    {
-                        RiakDatatype element = coreResponse.getCrdtElement();
-                        set = element.getAsSet();
-                    }
-                    BinaryValue returnedKey = coreResponse.hasGeneratedKey()
-                        ? coreResponse.getGeneratedKey()
-                        : null;
-                    Context returnedCtx = null;
-                    if (coreResponse.hasContext())
-                    {
-                        returnedCtx = new Context(coreResponse.getContext());
-                    }
-                    return new Response(returnedCtx, set, returnedKey);
-                }
-
-                @Override
-                protected Location convertQueryInfo(Location coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+    protected Response convertResponse(DtUpdateOperation.Response coreResponse) {
+        RiakSet set = null;
+        if (coreResponse.hasCrdtElement())
+        {
+            RiakDatatype element = coreResponse.getCrdtElement();
+            set = element.getAsSet();
+        }
+        BinaryValue returnedKey = coreResponse.hasGeneratedKey()
+            ? coreResponse.getGeneratedKey()
+            : null;
+        Context returnedCtx = null;
+        if (coreResponse.hasContext())
+        {
+            returnedCtx = new Context(coreResponse.getContext());
+        }
+        return new Response(returnedCtx, set, returnedKey);
     }
 
     /**
@@ -103,8 +77,6 @@ protected Location convertQueryInfo(Location coreQueryInfo)
      */
     public static class Builder extends UpdateDatatype.Builder
     {
-        private final SetUpdate update;
-
         /**
          * Construct a Builder for an UpdateSet command.
          * @param location the location of the set in Riak.
@@ -112,12 +84,11 @@ public static class Builder extends UpdateDatatype.Builder
          */
         public Builder(Location location, SetUpdate update)
         {
-            super(location);
+            super(location, update);
             if (update == null)
             {
                 throw new IllegalArgumentException("Update cannot be null");
             }
-            this.update = update;
         }
 
         /**
@@ -133,12 +104,11 @@ public Builder(Location location, SetUpdate update)
          */
         public Builder(Namespace namespace, SetUpdate update)
         {
-            super(namespace);
+            super(namespace, update);
             if (update == null)
             {
                 throw new IllegalArgumentException("Update cannot be null");
             }
-            this.update = update;
         }
 
         /**
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
index 5c274428d..873f6d05d 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
@@ -18,7 +18,6 @@
 
 import com.basho.riak.client.api.StreamableRiakCommand;
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
 import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.RiakFuture;
@@ -44,7 +43,8 @@
  * @author Sergey Galkin 
  * @since 2.0
  */
-public abstract class SecondaryIndexQuery, U> extends StreamableRiakCommand
+public abstract class SecondaryIndexQuery, U> extends StreamableRiakCommand
 {
     @FunctionalInterface
     public interface StreamableResponseCreator>
@@ -210,7 +210,8 @@ public Integer getTimeout()
         return timeout;
     }
 
-    protected final SecondaryIndexQueryOperation.Query createCoreQuery()
+    @Override
+    protected SecondaryIndexQueryOperation buildCoreOperation(boolean streamResults)
     {
         IndexConverter converter = getConverter();
 
@@ -250,29 +251,25 @@ protected final SecondaryIndexQueryOperation.Query createCoreQuery()
         {
             coreQueryBuilder.withCoverageContext(coverageContext);
         }
-        return coreQueryBuilder.build();
+
+        return new SecondaryIndexQueryOperation.Builder(coreQueryBuilder.build()).streamResults(streamResults).build();
     }
 
-    protected RiakFuture executeCoreAsync(RiakCluster cluster)
+    @Override
+    protected S convertResponse(SecondaryIndexQueryOperation.Response coreResponse)
     {
-        SecondaryIndexQueryOperation.Builder builder =
-                new SecondaryIndexQueryOperation.Builder(this.createCoreQuery());
-
-        return cluster.execute(builder.build());
+        return convertResponse(coreResponse, null);
     }
 
-    protected StreamingRiakFuture executeCoreAsyncStreaming(RiakCluster cluster)
+    @Override
+    @SuppressWarnings("unchecked")
+    protected U convertInfo(SecondaryIndexQueryOperation.Query coreInfo)
     {
-        SecondaryIndexQueryOperation.Builder builder =
-                new SecondaryIndexQueryOperation.Builder(this.createCoreQuery()).streamResults(true);
-
-        return cluster.execute(builder.build());
+        return (U)SecondaryIndexQuery.this;
     }
 
     private S convertResponse(final SecondaryIndexQueryOperation.Response coreResponse,
-                                final ChunkedResponseIterator iterator)
+                              final ChunkedResponseIterator iterator)
     {
         final S response;
 
@@ -288,33 +285,13 @@ private S convertResponse(final SecondaryIndexQueryOperation.Response coreRespon
         return response;
     }
 
-    @Override
-    protected final RiakFuture executeAsync(RiakCluster cluster)
-    {
-        final RiakFuture coreFuture = executeCoreAsync(cluster);
-        final CoreFutureAdapter future =
-                new CoreFutureAdapter(coreFuture) {
-                    @Override
-                    protected S convertResponse(SecondaryIndexQueryOperation.Response coreResponse) {
-                        return SecondaryIndexQuery.this.convertResponse(coreResponse, null);
-                    }
-
-                    @SuppressWarnings("unchecked")
-                    @Override
-                    protected U convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo) {
-                        return (U)SecondaryIndexQuery.this;
-                    }
-                };
-
-        coreFuture.addListener(future);
-        return future;
-    }
-
     @Override
     protected final RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
     {
+        final SecondaryIndexQueryOperation coreOperation = buildCoreOperation(true);
+
         final StreamingRiakFuture coreFuture =
-                executeCoreAsyncStreaming(cluster);
+                cluster.execute(coreOperation);
 
         final Response[] responses = {null};
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/CoveragePlan.java b/src/main/java/com/basho/riak/client/api/commands/kv/CoveragePlan.java
index 877daaf1b..25b0c5bb7 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/CoveragePlan.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/CoveragePlan.java
@@ -15,10 +15,7 @@
  */
 package com.basho.riak.client.api.commands.kv;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.core.operations.CoveragePlanOperation;
 import com.basho.riak.client.core.query.Namespace;
 
@@ -27,7 +24,7 @@
  *
  * @author Sergey Galkin 
  */
-public class CoveragePlan  extends RiakCommand
+public class CoveragePlan  extends GenericRiakCommand
 {
     private final CoveragePlanOperation operation;
 
@@ -37,27 +34,13 @@ private CoveragePlan(Builder builder)
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        final RiakFuture coreFuture = cluster.execute(operation);
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected Response convertResponse(CoveragePlanOperation.Response coreResponse)
-                {
-                    return new Response(coreResponse);
-                }
+    protected CoveragePlanOperation buildCoreOperation() {
+        return operation;
+    }
 
-                @Override
-                protected Namespace convertQueryInfo(Namespace coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+    @Override
+    protected Response convertResponse(CoveragePlanOperation.Response coreResponse) {
+        return new Response(coreResponse);
     }
 
     /**
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/DeleteValue.java b/src/main/java/com/basho/riak/client/api/commands/kv/DeleteValue.java
index 2aa119cde..d79ba6814 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/DeleteValue.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/DeleteValue.java
@@ -15,13 +15,12 @@
  */
 package com.basho.riak.client.api.commands.kv;
 
-import com.basho.riak.client.api.RiakCommand;
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.api.cap.Quorum;
 import com.basho.riak.client.api.cap.VClock;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.operations.DeleteOperation;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
 import com.basho.riak.client.api.commands.RiakOption;
 import com.basho.riak.client.core.query.Location;
 
@@ -64,7 +63,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class DeleteValue extends RiakCommand
+public final class DeleteValue extends GenericRiakCommand
 {
     private final Location location;
     private final Map, Object> options = new HashMap<>();
@@ -77,32 +76,18 @@ public DeleteValue(Builder builder)
         this.vClock = builder.vClock;
     }
 
+    /**
+     * To ensure that this method is accessible by {@link MultiDelete#executeBaseCommandAsync(DeleteValue, RiakCluster)}
+     * it MUST be overrode.
+     */
     @Override
     protected RiakFuture executeAsync(RiakCluster cluster)
     {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected Void convertResponse(Void coreResponse)
-                {
-                    return coreResponse;
-                }
-
-                @Override
-                protected Location convertQueryInfo(Location coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+        return super.executeAsync(cluster);
     }
 
-    private DeleteOperation buildCoreOperation()
+    @Override
+    protected DeleteOperation buildCoreOperation()
     {
         DeleteOperation.Builder builder = new DeleteOperation.Builder(location);
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/FetchValue.java b/src/main/java/com/basho/riak/client/api/commands/kv/FetchValue.java
index 7fb1c1d14..c7a5caf75 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/FetchValue.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/FetchValue.java
@@ -15,13 +15,12 @@
  */
 package com.basho.riak.client.api.commands.kv;
 
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.api.cap.Quorum;
 import com.basho.riak.client.api.cap.VClock;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.operations.FetchOperation;
-import com.basho.riak.client.api.RiakCommand;
 import com.basho.riak.client.core.RiakFuture;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
 import com.basho.riak.client.api.commands.RiakOption;
 import com.basho.riak.client.core.query.Location;
 
@@ -70,7 +69,8 @@
  * @since 2.0
  * @see Response
  */
-public final class FetchValue extends RiakCommand
+public final class FetchValue extends GenericRiakCommand
 {
     private final Location location;
     private final Map, Object> options = new HashMap<>();
@@ -84,33 +84,21 @@ public final class FetchValue extends RiakCommand
     @Override
     protected final RiakFuture executeAsync(RiakCluster cluster)
     {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected Response convertResponse(FetchOperation.Response coreResponse)
-                {
-                    return new Response.Builder().withNotFound(coreResponse.isNotFound())
-                                        .withUnchanged(coreResponse.isUnchanged())
-                                        .withValues(coreResponse.getObjectList())
-                                        .withLocation(location) // for ORM
-                                        .build();
-                }
+        return super.executeAsync(cluster);
+    }
 
-                @Override
-                protected Location convertQueryInfo(Location coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+    @Override
+    protected Response convertResponse(FetchOperation.Response coreResponse)
+    {
+        return new Response.Builder().withNotFound(coreResponse.isNotFound())
+                            .withUnchanged(coreResponse.isUnchanged())
+                            .withValues(coreResponse.getObjectList())
+                            .withLocation(location) // for ORM
+                            .build();
     }
 
-    private FetchOperation buildCoreOperation()
+    @Override
+    protected FetchOperation buildCoreOperation()
     {
         FetchOperation.Builder builder = new FetchOperation.Builder(location);
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
index 467c452ff..9874abed9 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
@@ -22,7 +22,6 @@
 import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.ListKeysOperation;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
@@ -74,7 +73,8 @@
  * @author Alex Moore 
  * @since 2.0
  */
-public final class ListKeys extends StreamableRiakCommand
+public final class ListKeys extends StreamableRiakCommand
 {
     private final Namespace namespace;
     private final int timeout;
@@ -86,28 +86,8 @@ public final class ListKeys extends StreamableRiakCommand executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation(false));
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected Response convertResponse(ListKeysOperation.Response coreResponse)
-                {
-                    return new Response(namespace, coreResponse.getKeys());
-                }
-
-                @Override
-                protected Namespace convertQueryInfo(Namespace coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
+    protected Response convertResponse(ListKeysOperation.Response coreResponse) {
+        return new Response(namespace, coreResponse.getKeys());
     }
 
     @Override
@@ -126,7 +106,8 @@ protected RiakFuture executeAsyncStreaming(RiakClu
         return future;
     }
 
-    private ListKeysOperation buildCoreOperation(boolean streamResults)
+    @Override
+    protected ListKeysOperation buildCoreOperation(boolean streamResults)
     {
         ListKeysOperation.Builder builder = new ListKeysOperation.Builder(namespace);
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
index 8040332fb..c41f8b158 100644
--- a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
+++ b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
@@ -15,14 +15,12 @@
 
 import com.basho.riak.client.api.RiakException;
 import com.basho.riak.client.api.StreamableRiakCommand;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
 import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
 import com.basho.riak.client.api.convert.ConversionException;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.MapReduceOperation;
-import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.query.functions.Function;
 import com.basho.riak.client.core.util.BinaryValue;
 import com.fasterxml.jackson.core.JsonEncoding;
@@ -37,7 +35,6 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.*;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TransferQueue;
 
 /**
@@ -48,7 +45,8 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public abstract class MapReduce extends StreamableRiakCommand
+public abstract class MapReduce extends StreamableRiakCommand
 {
     private final MapReduceSpec spec;
 
@@ -59,7 +57,7 @@ protected MapReduce(MapReduceInput input, Builder builder)
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
+    protected MapReduceOperation buildCoreOperation(boolean streamResults)
     {
         BinaryValue jobSpec;
         try
@@ -72,54 +70,28 @@ protected RiakFuture executeAsync(RiakCluster cluster)
             throw new RuntimeException(e);
         }
 
-        MapReduceOperation operation = new MapReduceOperation.Builder(jobSpec).build();
-
-        final RiakFuture coreFuture = cluster.execute(operation);
-
-        CoreFutureAdapter future =
-                new CoreFutureAdapter(coreFuture)
-                {
-                    @Override
-                    protected Response convertResponse(MapReduceOperation.Response coreResponse)
-                    {
-                        return new Response(coreResponse.getResults());
-                    }
-
-                    @Override
-                    protected BinaryValue convertQueryInfo(BinaryValue coreQueryInfo)
-                    {
-                        return coreQueryInfo;
-                    }
-                };
-
-        coreFuture.addListener(future);
-
-        return future;
+        return new MapReduceOperation.Builder(jobSpec)
+                .streamResults(streamResults)
+                .build();
     }
 
     @Override
-    protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
+    protected Response convertResponse(MapReduceOperation.Response coreResponse)
     {
-        BinaryValue jobSpec;
-        try
-        {
-            String spec = writeSpec();
-            jobSpec = BinaryValue.create(spec);
-        }
-        catch (RiakException e)
-        {
-            throw new RuntimeException(e);
-        }
-
-        MapReduceOperation operation = new MapReduceOperation.Builder(jobSpec).streamResults(true).build();
+        return new Response(coreResponse.getResults());
+    }
 
+    @Override
+    protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
+    {
+        final MapReduceOperation operation = buildCoreOperation(true);
         final StreamingRiakFuture coreFuture = cluster.execute(operation);
 
-        final StreamingResponse streamingResponse = new StreamingResponse(coreFuture, timeout);
+        final Response response = new Response(coreFuture, timeout);
 
-        ImmediateCoreFutureAdapter.SameQueryInfo future =
-                new ImmediateCoreFutureAdapter.SameQueryInfo(
-                        coreFuture, streamingResponse) {};
+        ImmediateCoreFutureAdapter.SameQueryInfo future =
+                new ImmediateCoreFutureAdapter.SameQueryInfo(
+                        coreFuture, response) {};
 
         coreFuture.addListener(future);
 
@@ -376,13 +348,27 @@ public T withLinkPhase(String bucket, String tag)
     /**
      * Response from a MapReduce command.
      */
-    public static class Response
+    public static class Response implements Iterable
     {
         private final Map results;
+        private final MapReduceResponseIterator responseIterator;
+
+        Response(StreamingRiakFuture coreFuture,
+                          int pollTimeout)
+        {
+            responseIterator = new MapReduceResponseIterator(coreFuture, pollTimeout);
+            results = null;
+        }
 
         public Response(Map results)
         {
             this.results = results;
+            responseIterator = null;
+        }
+
+        public boolean isStreamable()
+        {
+            return responseIterator != null;
         }
 
         public boolean hasResultForPhase(int i)
@@ -425,22 +411,16 @@ private ArrayNode flattenResults()
             }
             return flatArray;
         }
-    }
-
-    public static class StreamingResponse implements Iterable
-    {
-        final MapReduceResponseIterator responseIterator;
-
-        StreamingResponse(StreamingRiakFuture coreFuture,
-                          int pollTimeout)
-        {
-            responseIterator = new MapReduceResponseIterator(coreFuture, pollTimeout);
-        }
 
         @Override
         public Iterator iterator()
         {
-            return responseIterator;
+            if (isStreamable()) {
+                return responseIterator;
+            }
+
+            // TODO: add support for not streamable responses
+            throw new UnsupportedOperationException("Iterating is only supported for streamable response.");
         }
 
         private class MapReduceResponseIterator implements Iterator
diff --git a/src/main/java/com/basho/riak/client/api/commands/search/DeleteIndex.java b/src/main/java/com/basho/riak/client/api/commands/search/DeleteIndex.java
index fa81b8ede..d55ab0597 100644
--- a/src/main/java/com/basho/riak/client/api/commands/search/DeleteIndex.java
+++ b/src/main/java/com/basho/riak/client/api/commands/search/DeleteIndex.java
@@ -1,9 +1,7 @@
 package com.basho.riak.client.api.commands.search;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.GenericRiakCommand;
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.YzDeleteIndexOperation;
 
 /**
@@ -11,7 +9,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class DeleteIndex extends RiakCommand
+public final class DeleteIndex extends GenericRiakCommand
 {
     private final String index;
 
@@ -21,32 +19,7 @@ public final class DeleteIndex extends RiakCommand
     }
 
     @Override
-    protected final RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-    {
-        @Override
-        protected Void convertResponse(Void coreResponse)
-        {
-            return coreResponse;
-        }
-
-        @Override
-        protected String convertQueryInfo(String coreQueryInfo)
-        {
-            return coreQueryInfo;
-        }
-            };
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    private YzDeleteIndexOperation buildCoreOperation()
-    {
+    protected FutureOperation buildCoreOperation() {
         return new YzDeleteIndexOperation.Builder(index).build();
     }
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/search/FetchIndex.java b/src/main/java/com/basho/riak/client/api/commands/search/FetchIndex.java
index cb5482fdd..c0211699b 100644
--- a/src/main/java/com/basho/riak/client/api/commands/search/FetchIndex.java
+++ b/src/main/java/com/basho/riak/client/api/commands/search/FetchIndex.java
@@ -16,6 +16,7 @@
 
 package com.basho.riak.client.api.commands.search;
 
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.api.RiakCommand;
 import com.basho.riak.client.api.commands.CoreFutureAdapter;
 import com.basho.riak.client.core.RiakCluster;
@@ -27,7 +28,8 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public class FetchIndex extends RiakCommand
+public class FetchIndex extends GenericRiakCommand
 {
     private final String index;
 
@@ -37,31 +39,7 @@ public class FetchIndex extends RiakCommand executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected YzFetchIndexOperation.Response convertResponse(YzFetchIndexOperation.Response coreResponse)
-                {
-                    return coreResponse;
-                }
-
-                @Override
-                protected String convertQueryInfo(String coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    private YzFetchIndexOperation buildCoreOperation()
+    protected YzFetchIndexOperation buildCoreOperation()
     {
         return new YzFetchIndexOperation.Builder().withIndexName(index).build();
     }
diff --git a/src/main/java/com/basho/riak/client/api/commands/search/FetchSchema.java b/src/main/java/com/basho/riak/client/api/commands/search/FetchSchema.java
index 0c224a6c1..11b7cedb3 100644
--- a/src/main/java/com/basho/riak/client/api/commands/search/FetchSchema.java
+++ b/src/main/java/com/basho/riak/client/api/commands/search/FetchSchema.java
@@ -16,10 +16,7 @@
 
 package com.basho.riak.client.api.commands.search;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.core.operations.YzGetSchemaOperation;
 
 /**
@@ -27,7 +24,8 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class FetchSchema extends RiakCommand
+public final class FetchSchema extends GenericRiakCommand
 {
     private final String schema;
 
@@ -36,32 +34,32 @@ public final class FetchSchema extends RiakCommand executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected YzGetSchemaOperation.Response convertResponse(YzGetSchemaOperation.Response coreResponse)
-                {
-                    return coreResponse;
-                }
-
-                @Override
-                protected String convertQueryInfo(String coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
-    }
+//    @Override
+//    protected RiakFuture executeAsync(RiakCluster cluster)
+//    {
+//        RiakFuture coreFuture =
+//            cluster.execute(buildCoreOperation());
+//
+//        CoreFutureAdapter future =
+//            new CoreFutureAdapter(coreFuture)
+//            {
+//                @Override
+//                protected YzGetSchemaOperation.Response convertResponse(YzGetSchemaOperation.Response coreResponse)
+//                {
+//                    return coreResponse;
+//                }
+//
+//                @Override
+//                protected String convertQueryInfo(String coreQueryInfo)
+//                {
+//                    return coreQueryInfo;
+//                }
+//            };
+//        coreFuture.addListener(future);
+//        return future;
+//    }
 
-    private YzGetSchemaOperation buildCoreOperation()
+    protected YzGetSchemaOperation buildCoreOperation()
     {
         return new YzGetSchemaOperation.Builder(schema).build();
     }
diff --git a/src/main/java/com/basho/riak/client/api/commands/search/Search.java b/src/main/java/com/basho/riak/client/api/commands/search/Search.java
index d515128da..748dfd862 100644
--- a/src/main/java/com/basho/riak/client/api/commands/search/Search.java
+++ b/src/main/java/com/basho/riak/client/api/commands/search/Search.java
@@ -15,11 +15,8 @@
  */
 package com.basho.riak.client.api.commands.search;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.api.commands.RiakOption;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.operations.SearchOperation;
 import com.basho.riak.client.core.util.BinaryValue;
 
@@ -61,7 +58,8 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class Search extends RiakCommand
+public final class Search extends GenericRiakCommand
 {
     /**
      * Enum that encapsulates the possible settings for a search command's presort setting.
@@ -103,31 +101,7 @@ public Search(Builder builder)
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected SearchOperation.Response convertResponse(SearchOperation.Response coreResponse)
-                {
-                    return coreResponse;
-                }
-
-                @Override
-                protected BinaryValue convertQueryInfo(BinaryValue coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    private SearchOperation buildCoreOperation()
+    protected SearchOperation buildCoreOperation()
     {
         SearchOperation.Builder builder = new SearchOperation.Builder(BinaryValue.create(index), query);
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/search/StoreIndex.java b/src/main/java/com/basho/riak/client/api/commands/search/StoreIndex.java
index c3657cac0..a7463de76 100644
--- a/src/main/java/com/basho/riak/client/api/commands/search/StoreIndex.java
+++ b/src/main/java/com/basho/riak/client/api/commands/search/StoreIndex.java
@@ -1,9 +1,6 @@
 package com.basho.riak.client.api.commands.search;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.core.operations.YzPutIndexOperation;
 import com.basho.riak.client.core.query.search.YokozunaIndex;
 
@@ -17,7 +14,8 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class StoreIndex extends RiakCommand
+public final class StoreIndex extends GenericRiakCommand
 {
     private final Builder cmdBuilder;
 
@@ -27,30 +25,7 @@ public final class StoreIndex extends RiakCommand
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture = cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-                new CoreFutureAdapter(coreFuture)
-                {
-                    @Override
-                    protected Void convertResponse(Void coreResponse)
-                    {
-                        return coreResponse;
-                    }
-
-                    @Override
-                    protected YokozunaIndex convertQueryInfo(YokozunaIndex coreQueryInfo)
-                    {
-                        return coreQueryInfo;
-                    }
-                };
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    private YzPutIndexOperation buildCoreOperation()
+    protected YzPutIndexOperation buildCoreOperation()
     {
         final YzPutIndexOperation.Builder opBuilder = new YzPutIndexOperation.Builder(cmdBuilder.index);
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/search/StoreSchema.java b/src/main/java/com/basho/riak/client/api/commands/search/StoreSchema.java
index bd66f9078..8fae2daef 100644
--- a/src/main/java/com/basho/riak/client/api/commands/search/StoreSchema.java
+++ b/src/main/java/com/basho/riak/client/api/commands/search/StoreSchema.java
@@ -1,9 +1,6 @@
 package com.basho.riak.client.api.commands.search;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.core.operations.YzPutSchemaOperation;
 import com.basho.riak.client.core.query.search.YokozunaSchema;
 
@@ -16,7 +13,8 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class StoreSchema extends RiakCommand
+public final class StoreSchema extends GenericRiakCommand
 {
     private final YokozunaSchema schema;
 
@@ -26,31 +24,7 @@ public final class StoreSchema extends RiakCommand
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected Void convertResponse(Void coreResponse)
-                {
-                    return coreResponse;
-                }
-
-                @Override
-                protected YokozunaSchema convertQueryInfo(YokozunaSchema coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    private YzPutSchemaOperation buildCoreOperation()
+    protected YzPutSchemaOperation buildCoreOperation()
     {
         return new YzPutSchemaOperation.Builder(schema).build();
     }
diff --git a/src/main/java/com/basho/riak/client/api/commands/timeseries/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/timeseries/ListKeys.java
index aeebf6968..062d1974e 100644
--- a/src/main/java/com/basho/riak/client/api/commands/timeseries/ListKeys.java
+++ b/src/main/java/com/basho/riak/client/api/commands/timeseries/ListKeys.java
@@ -1,9 +1,6 @@
 package com.basho.riak.client.api.commands.timeseries;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.core.operations.ts.ListKeysOperation;
 import com.basho.riak.client.core.query.timeseries.QueryResult;
 
@@ -15,7 +12,8 @@
  * @author Sergey Galkin 
  * @since 2.0.3
  */
-public class ListKeys extends RiakCommand
+public class ListKeys extends GenericRiakCommand
 {
     private final String tableName;
     private final int timeout;
@@ -27,31 +25,7 @@ private ListKeys(ListKeys.Builder builder)
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture coreFuture =
-                cluster.execute(buildCoreOperation());
-
-        CoreFutureAdapter future =
-                new CoreFutureAdapter(coreFuture)
-                {
-                    @Override
-                    protected QueryResult convertResponse(QueryResult coreResponse)
-                    {
-                        return coreResponse;
-                    }
-
-                    @Override
-                    protected String convertQueryInfo(String coreQueryInfo)
-                    {
-                        return coreQueryInfo;
-                    }
-                };
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    private ListKeysOperation buildCoreOperation()
+    protected ListKeysOperation buildCoreOperation()
     {
         ListKeysOperation.Builder builder = new ListKeysOperation.Builder(tableName);
 
diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQueryTest.java b/src/test/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQueryTest.java
index 8fc72bec6..e1cf1e53e 100644
--- a/src/test/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQueryTest.java
+++ b/src/test/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQueryTest.java
@@ -44,7 +44,7 @@ public void intIndexQueryBuildsCorrectly()
                 .withTimeout(Integer.MAX_VALUE)
                 .build();
 
-        SecondaryIndexQueryOperation.Query query = iiq.createCoreQuery();
+        SecondaryIndexQueryOperation.Query query = iiq.buildCoreOperation(false).getQueryInfo();
 
         assertEquals(ns, query.getNamespace());
         assertEquals("test_index_int", query.getIndexName().toString());
@@ -62,7 +62,7 @@ public void intIndexQueryBuildsCorrectly()
                 .withPaginationSort(true)
                 .build();
 
-        query = iiq.createCoreQuery();
+        query = iiq.buildCoreOperation(false).getQueryInfo();
 
         assertEquals(ns, query.getNamespace());
         assertEquals("test_index_int", query.getIndexName().toString());
@@ -99,7 +99,7 @@ public void binIndexQueryBuildsCorrectly()
             .withRegexTermFilter("filter")
             .build();
 
-        SecondaryIndexQueryOperation.Query query = biq.createCoreQuery();
+        SecondaryIndexQueryOperation.Query query = biq.buildCoreOperation(false).getQueryInfo();
 
         assertEquals(ns, query.getNamespace());
         assertEquals("test_index_bin", query.getIndexName().toString());
@@ -117,7 +117,7 @@ public void binIndexQueryBuildsCorrectly()
             .withRegexTermFilter("filter")
             .build();
 
-        query = biq.createCoreQuery();
+        query = biq.buildCoreOperation(false).getQueryInfo();
 
         assertEquals(ns, query.getNamespace());
         assertEquals("test_index_bin", query.getIndexName().toString());
@@ -142,7 +142,7 @@ public void rawIndexQueryBuildsCorrectly()
             new RawIndexQuery.Builder(ns, "test_index", SecondaryIndexQuery.Type._INT, indexMatch)
                 .build();
 
-        SecondaryIndexQueryOperation.Query query = riq.createCoreQuery();
+        SecondaryIndexQueryOperation.Query query = riq.buildCoreOperation(false).getQueryInfo();
 
         assertEquals(ns, query.getNamespace());
         assertEquals("test_index_int", query.getIndexName().toString());
@@ -153,7 +153,7 @@ public void rawIndexQueryBuildsCorrectly()
                                       indexStart, indexEnd)
                 .build();
 
-        query = riq.createCoreQuery();
+        query = riq.buildCoreOperation(false).getQueryInfo();
 
         assertEquals(ns, query.getNamespace());
         assertEquals("test_index_int", query.getIndexName().toString());
diff --git a/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java b/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java
index 6b385c1ff..1ad19e568 100644
--- a/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java
+++ b/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java
@@ -170,14 +170,15 @@ private void streamingErlangBucketMR(String bucketType) throws InterruptedExcept
                         .withReducePhase(Function.newErlangFunction("riak_kv_mapreduce", "reduce_sort"), true)
                         .build();
 
-        final RiakFuture streamingFuture =
+        final RiakFuture streamingFuture =
                 client.executeAsyncStreaming(bmr, 10);
 
         boolean found42 = false;
         boolean found199 = false;
         int count = 0;
 
-        final MapReduce.StreamingResponse streamingResponse = streamingFuture.get();
+        final MapReduce.Response streamingResponse = streamingFuture.get();
+        assertTrue(streamingResponse.isStreamable());
         // The streaming query should return many results which are JSON arrays, each
         // containing a piece of the array [0-199].
         // Streaming result would look like: [[0], [1,2,3], ... [..., 199]], with the outer

From 63efe16f878640058a553056fbe5c01be31bbf39 Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Tue, 8 Nov 2016 06:43:58 +0200
Subject: [PATCH 37/55] Simplification  for the case when no needs to convert
 results

---
 .../riak/client/api/AsIsRiakCommand.java      | 49 +++++++++++++++++++
 .../riak/client/api/GenericRiakCommand.java   | 23 +++++++--
 .../buckets/FetchBucketProperties.java        |  5 +-
 .../buckets/ResetBucketProperties.java        |  4 +-
 .../buckets/StoreBucketProperties.java        |  4 +-
 .../client/api/commands/kv/DeleteValue.java   |  4 +-
 .../api/commands/search/DeleteIndex.java      |  7 ++-
 .../api/commands/search/FetchIndex.java       |  9 +---
 .../api/commands/search/FetchSchema.java      | 30 +-----------
 .../client/api/commands/search/Search.java    |  5 +-
 .../api/commands/search/StoreIndex.java       |  5 +-
 .../api/commands/search/StoreSchema.java      |  5 +-
 .../api/commands/timeseries/CoveragePlan.java | 13 ++---
 .../api/commands/timeseries/CreateTable.java  | 15 ++----
 .../api/commands/timeseries/Delete.java       | 18 ++-----
 .../commands/timeseries/DescribeTable.java    | 16 ++----
 .../client/api/commands/timeseries/Fetch.java | 16 ++----
 .../api/commands/timeseries/ListKeys.java     |  5 +-
 .../client/api/commands/timeseries/Query.java | 15 ++----
 .../client/api/commands/timeseries/Store.java | 16 ++----
 20 files changed, 117 insertions(+), 147 deletions(-)
 create mode 100644 src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java

diff --git a/src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java b/src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java
new file mode 100644
index 000000000..d0f4be59b
--- /dev/null
+++ b/src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2013-2016 Basho Technologies Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.basho.riak.client.api;
+
+import com.basho.riak.client.core.FutureOperation;
+import com.basho.riak.client.core.PBStreamingFutureOperation;
+import com.basho.riak.client.core.RiakCluster;
+import com.basho.riak.client.core.RiakFuture;
+
+/**
+ * @author Sergey Galkin 
+ * @since 2.1.0
+ */
+public abstract class AsIsRiakCommand extends RiakCommand
+{
+    protected abstract FutureOperation buildCoreOperation();
+
+    protected RiakFuture executeAsync(RiakCluster cluster)
+    {
+        final FutureOperation coreOperation = buildCoreOperation();
+
+        // TODO: WE NEED TO GET RID SUCH A WEIRD IF-FORK
+        final RiakFuture coreFuture;
+        if (coreOperation instanceof PBStreamingFutureOperation)
+        {
+            coreFuture = cluster.execute((PBStreamingFutureOperation) coreOperation);
+        }
+        else
+        {
+            coreFuture = cluster.execute(coreOperation);
+        }
+
+        return coreFuture;
+    }
+
+}
diff --git a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
index 4f52c9f73..4a15de827 100644
--- a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
@@ -1,8 +1,19 @@
-package com.basho.riak.client.api;
-
-/**
- * Created by srg on 11/6/16.
+/*
+ * Copyright 2013-2016 Basho Technologies Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
+package com.basho.riak.client.api;
 
 import com.basho.riak.client.api.commands.CoreFutureAdapter;
 import com.basho.riak.client.core.FutureOperation;
@@ -11,6 +22,10 @@
 import com.basho.riak.client.core.RiakFuture;
 
 
+/**
+ * @author Sergey Galkin 
+ * @since 2.1.0
+ */
 public abstract class GenericRiakCommand extends RiakCommand
 {
     @FunctionalInterface
diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/FetchBucketProperties.java b/src/main/java/com/basho/riak/client/api/commands/buckets/FetchBucketProperties.java
index 4b521825c..4722d14ac 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/FetchBucketProperties.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/FetchBucketProperties.java
@@ -16,7 +16,7 @@
 
 package com.basho.riak.client.api.commands.buckets;
 
-import com.basho.riak.client.api.GenericRiakCommand;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.FetchBucketPropsOperation;
 import com.basho.riak.client.core.query.Namespace;
 
@@ -36,8 +36,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class FetchBucketProperties extends GenericRiakCommand
+public final class FetchBucketProperties extends AsIsRiakCommand
 {
     private final Namespace namespace;
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ResetBucketProperties.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ResetBucketProperties.java
index cce63bbef..02cfea577 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/ResetBucketProperties.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ResetBucketProperties.java
@@ -1,6 +1,6 @@
 package com.basho.riak.client.api.commands.buckets;
 
-import com.basho.riak.client.api.GenericRiakCommand;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.ResetBucketPropsOperation;
 import com.basho.riak.client.core.query.Namespace;
 
@@ -18,7 +18,7 @@
  * @author Chris Mancini 
  * @since 2.0
  */
-public class ResetBucketProperties extends GenericRiakCommand
+public class ResetBucketProperties extends AsIsRiakCommand
 {
     private final Namespace namespace;
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/StoreBucketProperties.java b/src/main/java/com/basho/riak/client/api/commands/buckets/StoreBucketProperties.java
index 52f71895e..367678939 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/StoreBucketProperties.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/StoreBucketProperties.java
@@ -16,7 +16,7 @@
 
 package com.basho.riak.client.api.commands.buckets;
 
-import com.basho.riak.client.api.GenericRiakCommand;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.StoreBucketPropsOperation;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.query.functions.Function;
@@ -36,7 +36,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class StoreBucketProperties extends GenericRiakCommand
+public final class StoreBucketProperties extends AsIsRiakCommand
 {
     private final Namespace namespace;
     private final Boolean allowMulti;
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/DeleteValue.java b/src/main/java/com/basho/riak/client/api/commands/kv/DeleteValue.java
index d79ba6814..839ea4fa7 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/DeleteValue.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/DeleteValue.java
@@ -15,7 +15,7 @@
  */
 package com.basho.riak.client.api.commands.kv;
 
-import com.basho.riak.client.api.GenericRiakCommand;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.api.cap.Quorum;
 import com.basho.riak.client.api.cap.VClock;
 import com.basho.riak.client.core.RiakCluster;
@@ -63,7 +63,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class DeleteValue extends GenericRiakCommand
+public final class DeleteValue extends AsIsRiakCommand
 {
     private final Location location;
     private final Map, Object> options = new HashMap<>();
diff --git a/src/main/java/com/basho/riak/client/api/commands/search/DeleteIndex.java b/src/main/java/com/basho/riak/client/api/commands/search/DeleteIndex.java
index d55ab0597..28484ffa7 100644
--- a/src/main/java/com/basho/riak/client/api/commands/search/DeleteIndex.java
+++ b/src/main/java/com/basho/riak/client/api/commands/search/DeleteIndex.java
@@ -1,7 +1,6 @@
 package com.basho.riak.client.api.commands.search;
 
-import com.basho.riak.client.api.GenericRiakCommand;
-import com.basho.riak.client.core.FutureOperation;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.YzDeleteIndexOperation;
 
 /**
@@ -9,7 +8,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class DeleteIndex extends GenericRiakCommand
+public final class DeleteIndex extends AsIsRiakCommand
 {
     private final String index;
 
@@ -19,7 +18,7 @@ public final class DeleteIndex extends GenericRiakCommand buildCoreOperation() {
+    protected YzDeleteIndexOperation buildCoreOperation() {
         return new YzDeleteIndexOperation.Builder(index).build();
     }
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/search/FetchIndex.java b/src/main/java/com/basho/riak/client/api/commands/search/FetchIndex.java
index c0211699b..691dbd27e 100644
--- a/src/main/java/com/basho/riak/client/api/commands/search/FetchIndex.java
+++ b/src/main/java/com/basho/riak/client/api/commands/search/FetchIndex.java
@@ -16,11 +16,7 @@
 
 package com.basho.riak.client.api.commands.search;
 
-import com.basho.riak.client.api.GenericRiakCommand;
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.YzFetchIndexOperation;
 
 /**
@@ -28,8 +24,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public class FetchIndex extends GenericRiakCommand
+public class FetchIndex extends AsIsRiakCommand
 {
     private final String index;
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/search/FetchSchema.java b/src/main/java/com/basho/riak/client/api/commands/search/FetchSchema.java
index 11b7cedb3..a5ce1df54 100644
--- a/src/main/java/com/basho/riak/client/api/commands/search/FetchSchema.java
+++ b/src/main/java/com/basho/riak/client/api/commands/search/FetchSchema.java
@@ -16,7 +16,7 @@
 
 package com.basho.riak.client.api.commands.search;
 
-import com.basho.riak.client.api.GenericRiakCommand;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.YzGetSchemaOperation;
 
 /**
@@ -24,8 +24,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class FetchSchema extends GenericRiakCommand
+public final class FetchSchema extends AsIsRiakCommand
 {
     private final String schema;
 
@@ -34,31 +33,6 @@ public final class FetchSchema extends GenericRiakCommand executeAsync(RiakCluster cluster)
-//    {
-//        RiakFuture coreFuture =
-//            cluster.execute(buildCoreOperation());
-//
-//        CoreFutureAdapter future =
-//            new CoreFutureAdapter(coreFuture)
-//            {
-//                @Override
-//                protected YzGetSchemaOperation.Response convertResponse(YzGetSchemaOperation.Response coreResponse)
-//                {
-//                    return coreResponse;
-//                }
-//
-//                @Override
-//                protected String convertQueryInfo(String coreQueryInfo)
-//                {
-//                    return coreQueryInfo;
-//                }
-//            };
-//        coreFuture.addListener(future);
-//        return future;
-//    }
-
     protected YzGetSchemaOperation buildCoreOperation()
     {
         return new YzGetSchemaOperation.Builder(schema).build();
diff --git a/src/main/java/com/basho/riak/client/api/commands/search/Search.java b/src/main/java/com/basho/riak/client/api/commands/search/Search.java
index 748dfd862..62512fc75 100644
--- a/src/main/java/com/basho/riak/client/api/commands/search/Search.java
+++ b/src/main/java/com/basho/riak/client/api/commands/search/Search.java
@@ -15,7 +15,7 @@
  */
 package com.basho.riak.client.api.commands.search;
 
-import com.basho.riak.client.api.GenericRiakCommand;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.api.commands.RiakOption;
 import com.basho.riak.client.core.operations.SearchOperation;
 import com.basho.riak.client.core.util.BinaryValue;
@@ -58,8 +58,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class Search extends GenericRiakCommand
+public final class Search extends AsIsRiakCommand
 {
     /**
      * Enum that encapsulates the possible settings for a search command's presort setting.
diff --git a/src/main/java/com/basho/riak/client/api/commands/search/StoreIndex.java b/src/main/java/com/basho/riak/client/api/commands/search/StoreIndex.java
index a7463de76..04e668387 100644
--- a/src/main/java/com/basho/riak/client/api/commands/search/StoreIndex.java
+++ b/src/main/java/com/basho/riak/client/api/commands/search/StoreIndex.java
@@ -1,6 +1,6 @@
 package com.basho.riak.client.api.commands.search;
 
-import com.basho.riak.client.api.GenericRiakCommand;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.YzPutIndexOperation;
 import com.basho.riak.client.core.query.search.YokozunaIndex;
 
@@ -14,8 +14,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class StoreIndex extends GenericRiakCommand
+public final class StoreIndex extends AsIsRiakCommand
 {
     private final Builder cmdBuilder;
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/search/StoreSchema.java b/src/main/java/com/basho/riak/client/api/commands/search/StoreSchema.java
index 8fae2daef..5ec4749a7 100644
--- a/src/main/java/com/basho/riak/client/api/commands/search/StoreSchema.java
+++ b/src/main/java/com/basho/riak/client/api/commands/search/StoreSchema.java
@@ -1,6 +1,6 @@
 package com.basho.riak.client.api.commands.search;
 
-import com.basho.riak.client.api.GenericRiakCommand;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.YzPutSchemaOperation;
 import com.basho.riak.client.core.query.search.YokozunaSchema;
 
@@ -13,8 +13,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class StoreSchema extends GenericRiakCommand
+public final class StoreSchema extends AsIsRiakCommand
 {
     private final YokozunaSchema schema;
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/timeseries/CoveragePlan.java b/src/main/java/com/basho/riak/client/api/commands/timeseries/CoveragePlan.java
index 49a3428e2..cd1dee016 100644
--- a/src/main/java/com/basho/riak/client/api/commands/timeseries/CoveragePlan.java
+++ b/src/main/java/com/basho/riak/client/api/commands/timeseries/CoveragePlan.java
@@ -15,13 +15,12 @@
  */
 package com.basho.riak.client.api.commands.timeseries;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.AsIsRiakCommand;
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.ts.CoveragePlanOperation;
 import com.basho.riak.client.core.query.timeseries.CoveragePlanResult;
 
-public class CoveragePlan extends RiakCommand
+public class CoveragePlan extends AsIsRiakCommand
 {
     private final CoveragePlanOperation operation;
 
@@ -31,10 +30,8 @@ private CoveragePlan(Builder builder)
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture future = cluster.execute(operation);
-        return future;
+    protected FutureOperation buildCoreOperation() {
+        return operation;
     }
 
     public static class Builder extends CoveragePlanOperation.AbstractBuilder
diff --git a/src/main/java/com/basho/riak/client/api/commands/timeseries/CreateTable.java b/src/main/java/com/basho/riak/client/api/commands/timeseries/CreateTable.java
index 2f4453a77..b3538c826 100644
--- a/src/main/java/com/basho/riak/client/api/commands/timeseries/CreateTable.java
+++ b/src/main/java/com/basho/riak/client/api/commands/timeseries/CreateTable.java
@@ -15,9 +15,8 @@
  */
 package com.basho.riak.client.api.commands.timeseries;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.AsIsRiakCommand;
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.ts.CreateTableOperation;
 import com.basho.riak.client.core.query.timeseries.TableDefinition;
 
@@ -28,7 +27,7 @@
  * @author Sergey Galkin 
  * @since 2.0.6
  */
-public class CreateTable extends RiakCommand
+public class CreateTable extends AsIsRiakCommand
 {
     private final Builder builder;
 
@@ -38,12 +37,8 @@ private CreateTable(Builder builder)
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        final RiakFuture future =
-                cluster.execute(builder.buildOperation());
-
-        return future;
+    protected FutureOperation buildCoreOperation() {
+        return builder.buildOperation();
     }
 
     public static class Builder extends CreateTableOperation.AbstractBuilder
diff --git a/src/main/java/com/basho/riak/client/api/commands/timeseries/Delete.java b/src/main/java/com/basho/riak/client/api/commands/timeseries/Delete.java
index 60e774080..f2e1df4db 100644
--- a/src/main/java/com/basho/riak/client/api/commands/timeseries/Delete.java
+++ b/src/main/java/com/basho/riak/client/api/commands/timeseries/Delete.java
@@ -1,8 +1,7 @@
 package com.basho.riak.client.api.commands.timeseries;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.AsIsRiakCommand;
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.ts.DeleteOperation;
 import com.basho.riak.client.core.query.timeseries.Cell;
 
@@ -14,7 +13,7 @@
  * @author Sergey Galkin 
  * @since 2.0.3
  */
-public class Delete extends RiakCommand
+public class Delete extends AsIsRiakCommand
 {
     private final Builder builder;
 
@@ -24,16 +23,7 @@ private Delete(Builder builder)
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture future =
-                cluster.execute(buildCoreOperation());
-
-        return future;
-    }
-
-    private DeleteOperation buildCoreOperation()
-    {
+    protected FutureOperation buildCoreOperation() {
         final DeleteOperation.Builder opBuilder =
                 new DeleteOperation.Builder(this.builder.tableName, builder.keyValues);
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/timeseries/DescribeTable.java b/src/main/java/com/basho/riak/client/api/commands/timeseries/DescribeTable.java
index a7d524ad7..d539631bc 100644
--- a/src/main/java/com/basho/riak/client/api/commands/timeseries/DescribeTable.java
+++ b/src/main/java/com/basho/riak/client/api/commands/timeseries/DescribeTable.java
@@ -1,8 +1,6 @@
 package com.basho.riak.client.api.commands.timeseries;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.ts.DescribeTableOperation;
 import com.basho.riak.client.core.query.timeseries.TableDefinition;
 
@@ -13,7 +11,7 @@
  * @author Alex Moore 
   * @since 2.0.4
  */
-public class DescribeTable extends RiakCommand
+public class DescribeTable extends AsIsRiakCommand
 {
     private final String tableName;
 
@@ -34,15 +32,7 @@ public DescribeTable(String tableName)
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture future =
-                cluster.execute(buildCoreOperation());
-
-        return future;
-    }
-
-    private DescribeTableOperation buildCoreOperation()
+    protected DescribeTableOperation buildCoreOperation()
     {
         return new DescribeTableOperation(this.tableName);
     }
diff --git a/src/main/java/com/basho/riak/client/api/commands/timeseries/Fetch.java b/src/main/java/com/basho/riak/client/api/commands/timeseries/Fetch.java
index b81778c3a..c54836165 100644
--- a/src/main/java/com/basho/riak/client/api/commands/timeseries/Fetch.java
+++ b/src/main/java/com/basho/riak/client/api/commands/timeseries/Fetch.java
@@ -1,8 +1,6 @@
 package com.basho.riak.client.api.commands.timeseries;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.ts.FetchOperation;
 import com.basho.riak.client.core.query.timeseries.Cell;
 import com.basho.riak.client.core.query.timeseries.QueryResult;
@@ -15,7 +13,7 @@
  * @author Sergey Galkin 
  * @since 2.0.3
  */
-public class Fetch extends RiakCommand
+public class Fetch extends AsIsRiakCommand
 {
     private final Builder builder;
 
@@ -25,15 +23,7 @@ private Fetch(Builder builder)
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture future =
-                cluster.execute(buildCoreOperation());
-
-        return future;
-    }
-
-    private FetchOperation buildCoreOperation()
+    protected FetchOperation buildCoreOperation()
     {
         final FetchOperation.Builder opBuilder =
                 new FetchOperation.Builder(this.builder.tableName, builder.keyValues);
diff --git a/src/main/java/com/basho/riak/client/api/commands/timeseries/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/timeseries/ListKeys.java
index 062d1974e..d1568b3de 100644
--- a/src/main/java/com/basho/riak/client/api/commands/timeseries/ListKeys.java
+++ b/src/main/java/com/basho/riak/client/api/commands/timeseries/ListKeys.java
@@ -1,6 +1,6 @@
 package com.basho.riak.client.api.commands.timeseries;
 
-import com.basho.riak.client.api.GenericRiakCommand;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.ts.ListKeysOperation;
 import com.basho.riak.client.core.query.timeseries.QueryResult;
 
@@ -12,8 +12,7 @@
  * @author Sergey Galkin 
  * @since 2.0.3
  */
-public class ListKeys extends GenericRiakCommand
+public class ListKeys extends AsIsRiakCommand
 {
     private final String tableName;
     private final int timeout;
diff --git a/src/main/java/com/basho/riak/client/api/commands/timeseries/Query.java b/src/main/java/com/basho/riak/client/api/commands/timeseries/Query.java
index 74bd09013..9334c4c15 100644
--- a/src/main/java/com/basho/riak/client/api/commands/timeseries/Query.java
+++ b/src/main/java/com/basho/riak/client/api/commands/timeseries/Query.java
@@ -1,13 +1,9 @@
 package com.basho.riak.client.api.commands.timeseries;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.ts.QueryOperation;
-import com.basho.riak.client.core.operations.ts.QueryOperation.Builder;
 import com.basho.riak.client.core.query.timeseries.QueryResult;
 import com.basho.riak.client.core.util.BinaryValue;
-import com.google.protobuf.ByteString;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -24,7 +20,7 @@
  * @author Sergey Galkin 
  * @since 2.0.3
  */
-public class Query extends RiakCommand
+public class Query extends AsIsRiakCommand
 {
     private final Builder builder;
 
@@ -34,12 +30,7 @@ private Query(Builder builder)
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        return cluster.execute(buildCoreOperation());
-    }
-
-    private QueryOperation buildCoreOperation()
+    protected QueryOperation buildCoreOperation()
     {
         return new QueryOperation.Builder(builder.queryText)
                                            .withCoverageContext(builder.coverageContext)
diff --git a/src/main/java/com/basho/riak/client/api/commands/timeseries/Store.java b/src/main/java/com/basho/riak/client/api/commands/timeseries/Store.java
index ea1f06935..d0e7ea57d 100644
--- a/src/main/java/com/basho/riak/client/api/commands/timeseries/Store.java
+++ b/src/main/java/com/basho/riak/client/api/commands/timeseries/Store.java
@@ -1,8 +1,6 @@
 package com.basho.riak.client.api.commands.timeseries;
 
-import com.basho.riak.client.api.RiakCommand;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.AsIsRiakCommand;
 import com.basho.riak.client.core.operations.ts.StoreOperation;
 import com.basho.riak.client.core.query.timeseries.Row;
 
@@ -21,7 +19,7 @@
  * @author Sergey Galkin 
  * @since 2.0.3
  */
-public class Store extends RiakCommand
+public class Store extends AsIsRiakCommand
 {
     private final Builder builder;
 
@@ -31,15 +29,7 @@ private Store (Builder builder)
     }
 
     @Override
-    protected RiakFuture executeAsync(RiakCluster cluster)
-    {
-        RiakFuture future =
-                cluster.execute(buildCoreOperation());
-
-        return future;
-    }
-
-    private StoreOperation buildCoreOperation()
+    protected StoreOperation buildCoreOperation()
     {
         return new StoreOperation.Builder(builder.tableName)
                 .withRows(builder.rows)

From 52795d41d3c07ab6df839910da7147d99a3de1d3 Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Tue, 8 Nov 2016 07:19:53 +0200
Subject: [PATCH 38/55] Generalize streamable responses

---
 .../com/basho/riak/client/api/RiakClient.java |  2 +-
 .../client/api/StreamableRiakCommand.java     |  4 +-
 .../api/commands/buckets/ListBuckets.java     | 55 ++++++++--------
 .../commands/indexes/SecondaryIndexQuery.java |  4 +-
 .../riak/client/api/commands/kv/ListKeys.java | 62 ++++++++++---------
 .../api/commands/mapreduce/MapReduce.java     |  2 +-
 .../buckets/itest/ITestListBuckets.java       |  4 +-
 .../commands/buckets/itest/ITestListKeys.java |  4 +-
 8 files changed, 74 insertions(+), 63 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/RiakClient.java b/src/main/java/com/basho/riak/client/api/RiakClient.java
index 71bfc3584..82ad3d9d3 100644
--- a/src/main/java/com/basho/riak/client/api/RiakClient.java
+++ b/src/main/java/com/basho/riak/client/api/RiakClient.java
@@ -435,7 +435,7 @@ public  RiakFuture executeAsync(RiakCommand command)
      * @return a RiakFuture for the operation
      * @see RiakFuture
      */
-    public  RiakFuture executeAsyncStreaming(StreamableRiakCommand command, int timeoutMS)
+    public  RiakFuture executeAsyncStreaming(StreamableRiakCommand command, int timeoutMS)
     {
         return command.executeAsyncStreaming(cluster, timeoutMS);
     }
diff --git a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
index 6cc36e91f..bf7136ae6 100644
--- a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
@@ -33,7 +33,7 @@
  * @author Brian Roach 
  * @since 2.0
  */
-public abstract class StreamableRiakCommand extends GenericRiakCommand
+public abstract class StreamableRiakCommand extends GenericRiakCommand
 {
     public StreamableRiakCommand() {
     }
@@ -42,7 +42,7 @@ public StreamableRiakCommand(Converter responseConverter, Converter executeAsyncStreaming(RiakCluster cluster, int timeout);
+    protected abstract RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout);
 
     @Override
     protected final FutureOperation buildCoreOperation() {
diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
index c010d6ce8..80be5e143 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
@@ -63,7 +63,7 @@
  * @author Alex Moore 
  * @since 2.0
  */
-public final class ListBuckets extends StreamableRiakCommand
 {
     private final int timeout;
@@ -81,15 +81,15 @@ protected Response convertResponse(ListBucketsOperation.Response coreResponse) {
     }
 
     @Override
-    protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
+    protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
     {
         StreamingRiakFuture coreFuture =
                 cluster.execute(buildCoreOperation(true));
 
-        final StreamingResponse streamingResponse = new StreamingResponse(type, timeout, coreFuture);
+        final Response streamingResponse = new Response(type, timeout, coreFuture);
 
-        ImmediateCoreFutureAdapter.SameQueryInfo future =
-                new ImmediateCoreFutureAdapter.SameQueryInfo
+        ImmediateCoreFutureAdapter.SameQueryInfo future =
+                new ImmediateCoreFutureAdapter.SameQueryInfo
                         (coreFuture, streamingResponse) {};
 
         coreFuture.addListener(future);
@@ -131,42 +131,47 @@ protected ListBucketsOperation buildCoreOperation(boolean streamResults)
      */
     public static class Response implements Iterable
     {
+        private final ChunkedResponseIterator
+                chunkedResponseIterator;
         private final BinaryValue type;
         private final List buckets;
 
-        public Response(BinaryValue type, List buckets)
+        Response(BinaryValue type,
+                          int pollTimeout,
+                          StreamingRiakFuture coreFuture)
         {
+            this.chunkedResponseIterator = new ChunkedResponseIterator<>(coreFuture,
+                    pollTimeout,
+                    (bucketName) -> new Namespace(type, bucketName),
+                    (response) -> response.getBuckets().iterator());
+
             this.type = type;
-            this.buckets = buckets;
+            this.buckets = null;
         }
 
-        @Override
-        public Iterator iterator()
+        public Response(BinaryValue type, List buckets)
         {
-            return new Itr(buckets.iterator(), type);
+            this.chunkedResponseIterator = null;
+            this.type = type;
+            this.buckets = buckets;
         }
-    }
 
-    public static class StreamingResponse extends Response
-    {
-        private final ChunkedResponseIterator
-                chunkedResponseIterator;
-
-        StreamingResponse(BinaryValue type,
-                          int pollTimeout,
-                          StreamingRiakFuture coreFuture)
+        public boolean isStreamable()
         {
-            super(type, null);
-            chunkedResponseIterator = new ChunkedResponseIterator<>(coreFuture,
-                                                                    pollTimeout,
-                                                                    (bucketName) -> new Namespace(type, bucketName),
-                                                                    (response) -> response.getBuckets().iterator());
+            return chunkedResponseIterator != null;
         }
 
         @Override
         public Iterator iterator()
         {
-            return chunkedResponseIterator;
+            if (isStreamable())
+            {
+                assert  chunkedResponseIterator != null;
+                return chunkedResponseIterator;
+            }
+
+            assert  buckets != null;
+            return new Itr(buckets.iterator(), type);
         }
     }
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
index 873f6d05d..32a95b342 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
@@ -43,7 +43,7 @@
  * @author Sergey Galkin 
  * @since 2.0
  */
-public abstract class SecondaryIndexQuery, U> extends StreamableRiakCommand, U> extends StreamableRiakCommand
 {
     @FunctionalInterface
@@ -55,6 +55,7 @@ R createResponse(Namespace queryLocation,
                          ChunkedResponseIterator chunkedResponseIterator);
     }
 
+    @FunctionalInterface
     public interface GatherableResponseCreator>
     {
         R createResponse(Namespace queryLocation,
@@ -678,6 +679,7 @@ public boolean isStreamable()
         public Iterator iterator()
         {
             if (isStreamable()) {
+                assert chunkedResponseIterator != null;
                 return (Iterator)chunkedResponseIterator;
             }
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
index 9874abed9..69927cea0 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
@@ -73,7 +73,7 @@
  * @author Alex Moore 
  * @since 2.0
  */
-public final class ListKeys extends StreamableRiakCommand
 {
     private final Namespace namespace;
@@ -91,15 +91,15 @@ protected Response convertResponse(ListKeysOperation.Response coreResponse) {
     }
 
     @Override
-    protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
+    protected RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
     {
         StreamingRiakFuture coreFuture =
                 cluster.execute(buildCoreOperation(true));
 
-        final ListKeys.StreamingResponse streamingResponse = new ListKeys.StreamingResponse(namespace, timeout, coreFuture);
+        final Response streamingResponse = new Response(namespace, timeout, coreFuture);
 
-        ImmediateCoreFutureAdapter.SameQueryInfo future =
-                new ImmediateCoreFutureAdapter.SameQueryInfo(
+        ImmediateCoreFutureAdapter.SameQueryInfo future =
+                new ImmediateCoreFutureAdapter.SameQueryInfo(
                         coreFuture, streamingResponse) {};
 
         coreFuture.addListener(future);
@@ -123,6 +123,8 @@ protected ListKeysOperation buildCoreOperation(boolean streamResults)
 
     public static class Response implements Iterable
     {
+        private final ChunkedResponseIterator
+                chunkedResponseIterator;
         private final Namespace namespace;
         private final List keys;
 
@@ -130,11 +132,37 @@ public Response(Namespace namespace, List keys)
         {
             this.namespace = namespace;
             this.keys = keys;
+            this.chunkedResponseIterator = null;
+        }
+
+        Response(Namespace namespace,
+                          int pollTimeout,
+                          StreamingRiakFuture coreFuture)
+        {
+            chunkedResponseIterator = new ChunkedResponseIterator<>(coreFuture,
+                    pollTimeout,
+                    (key) -> new Location(namespace, key),
+                    (nextChunk) -> nextChunk.getKeys().iterator());
+
+            this.namespace = namespace;
+            this.keys = null;
+        }
+
+        public boolean isStreamable()
+        {
+            return chunkedResponseIterator != null;
         }
 
         @Override
         public Iterator iterator()
         {
+            if (isStreamable())
+            {
+                assert chunkedResponseIterator != null;
+                return chunkedResponseIterator;
+            }
+
+            assert keys != null;
             return new Itr(namespace, keys.iterator());
         }
     }
@@ -170,30 +198,6 @@ public void remove()
         }
     }
 
-    public static class StreamingResponse extends Response
-    {
-        private final ChunkedResponseIterator
-                chunkedResponseIterator;
-
-        StreamingResponse(Namespace namespace,
-                          int pollTimeout,
-                          StreamingRiakFuture coreFuture)
-        {
-            super(namespace, null);
-            chunkedResponseIterator = new ChunkedResponseIterator<>(coreFuture,
-                                                                    pollTimeout,
-                                                                    (key) -> new Location(namespace, key),
-                                                                    (nextChunk) -> nextChunk.getKeys().iterator());
-        }
-
-        @Override
-        public Iterator iterator()
-        {
-            return chunkedResponseIterator;
-        }
-
-    }
-
     /**
      * Used to construct a ListKeys command.
      */
diff --git a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
index c41f8b158..184b39975 100644
--- a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
+++ b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
@@ -45,7 +45,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public abstract class MapReduce extends StreamableRiakCommand
 {
     private final MapReduceSpec spec;
diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java
index 0d3621f84..9a93105ca 100644
--- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java
+++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListBuckets.java
@@ -111,10 +111,10 @@ private void testListBucketsStreaming(Namespace namespace) throws InterruptedExc
     {
         ListBuckets listBucketsCommand = new ListBuckets.Builder(namespace.getBucketType()).build();
 
-        final RiakFuture streamingFuture =
+        final RiakFuture streamingFuture =
                 client.executeAsyncStreaming(listBucketsCommand, 500);
 
-        final ListBuckets.StreamingResponse streamResponse = streamingFuture.get();
+        final ListBuckets.Response streamResponse = streamingFuture.get();
         final Iterator iterator = streamResponse.iterator();
 
         assumeTrue(iterator.hasNext());
diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java
index bd16bd1fa..dec479329 100644
--- a/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java
+++ b/src/test/java/com/basho/riak/client/api/commands/buckets/itest/ITestListKeys.java
@@ -71,10 +71,10 @@ public void testLargeStreamingListKeys() throws ExecutionException, InterruptedE
 
         ListKeys lk = new ListKeys.Builder(typedNamespace).build();
 
-        final RiakFuture streamFuture =
+        final RiakFuture streamFuture =
                 client.executeAsyncStreaming(lk, 200);
 
-        final ListKeys.StreamingResponse streamingResponse = streamFuture.get();
+        final ListKeys.Response streamingResponse = streamFuture.get();
 
         int count = 0;
         boolean foundLastKey = false;

From 3e5c7b02be463222652509b8f9dd70b269af993e Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Tue, 8 Nov 2016 10:46:15 +0200
Subject: [PATCH 39/55] Get rid of unsafe default response and infor
 conversions in GenericRiakCommand

---
 .../riak/client/api/GenericRiakCommand.java   | 20 +++++++++----------
 .../client/api/StreamableRiakCommand.java     |  8 ++++++++
 .../api/commands/buckets/ListBuckets.java     |  4 ++--
 .../api/commands/datatypes/FetchCounter.java  |  2 +-
 .../api/commands/datatypes/FetchDatatype.java |  5 +++--
 .../api/commands/datatypes/FetchHll.java      |  2 +-
 .../api/commands/datatypes/FetchMap.java      |  2 +-
 .../api/commands/datatypes/FetchSet.java      |  2 +-
 .../api/commands/datatypes/UpdateCounter.java |  2 +-
 .../commands/datatypes/UpdateDatatype.java    |  3 ++-
 .../api/commands/datatypes/UpdateHll.java     |  2 +-
 .../api/commands/datatypes/UpdateMap.java     |  2 +-
 .../api/commands/datatypes/UpdateSet.java     |  2 +-
 .../client/api/commands/kv/CoveragePlan.java  |  3 ++-
 .../client/api/commands/kv/FetchValue.java    |  4 ++--
 .../riak/client/api/commands/kv/ListKeys.java |  4 ++--
 .../api/commands/mapreduce/MapReduce.java     |  4 ++--
 17 files changed, 41 insertions(+), 30 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
index 4a15de827..532521202 100644
--- a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
@@ -28,6 +28,14 @@
  */
 public abstract class GenericRiakCommand extends RiakCommand
 {
+    public static abstract class GenericRiakCommandWithSameInfo extends GenericRiakCommand
+    {
+        @Override
+        protected I convertInfo(I coreInfo) {
+            return coreInfo;
+        }
+    }
+
     @FunctionalInterface
     protected interface Converter
     {
@@ -88,15 +96,7 @@ protected I convertQueryInfo(CoreI coreQueryInfo)
         return future;
     }
 
-    @SuppressWarnings("unchecked")
-    protected R convertResponse(CoreR coreResponse)
-    {
-        return (R)coreResponse;
-    }
+    protected abstract R convertResponse(CoreR coreResponse);
 
-    @SuppressWarnings("unchecked")
-    protected I convertInfo(CoreI coreInfo)
-    {
-        return (I)coreInfo;
-    }
+    protected abstract I convertInfo(CoreI coreInfo);
 }
\ No newline at end of file
diff --git a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
index bf7136ae6..72d7c8f99 100644
--- a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
@@ -35,6 +35,14 @@
  */
 public abstract class StreamableRiakCommand extends GenericRiakCommand
 {
+    public static abstract class StreamableRiakCommandWithSameInfo extends StreamableRiakCommand
+    {
+        @Override
+        protected I convertInfo(I coreInfo) {
+            return coreInfo;
+        }
+    }
+
     public StreamableRiakCommand() {
     }
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
index 80be5e143..a31c23b3c 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
@@ -63,8 +63,8 @@
  * @author Alex Moore 
  * @since 2.0
  */
-public final class ListBuckets extends StreamableRiakCommand
+public final class ListBuckets extends StreamableRiakCommand.StreamableRiakCommandWithSameInfo
 {
     private final int timeout;
     private final BinaryValue type;
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchCounter.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchCounter.java
index b47df947b..fc40d1f42 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchCounter.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchCounter.java
@@ -37,7 +37,7 @@
  * @author Brian Roach 
  * @since 2.0
  */
-public final class FetchCounter extends FetchDatatype
+public final class FetchCounter extends FetchDatatype
 {
     private FetchCounter(Builder builder)
     {
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchDatatype.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchDatatype.java
index a77b50716..a78c9c492 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchDatatype.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchDatatype.java
@@ -31,7 +31,8 @@
   * @author Dave Rusek 
   * @since 2.0
   */
-public abstract class FetchDatatype extends GenericRiakCommand
+public abstract class FetchDatatype extends
+         GenericRiakCommand.GenericRiakCommandWithSameInfo
 {
     private final Location location;
     private final Map, Object> options = new HashMap<>();
@@ -48,7 +49,7 @@ protected FetchDatatype(Builder builder)
         this.options.putAll(builder.options);
     }
 
-    public  FetchDatatype withOption(Option option, V value)
+    public  FetchDatatype withOption(Option option, V value)
     {
         options.put(option, value);
         return this;
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java
index b5e0b5fa3..48e2157a1 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java
@@ -38,7 +38,7 @@
  * @since 2.1.0
  */
  
-public final class FetchHll extends FetchDatatype
+public final class FetchHll extends FetchDatatype
 {
     private FetchHll(Builder builder)
     {
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchMap.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchMap.java
index 627947117..a66ef394c 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchMap.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchMap.java
@@ -39,7 +39,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class FetchMap extends FetchDatatype
+public final class FetchMap extends FetchDatatype
 {
     private FetchMap(Builder builder)
     {
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchSet.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchSet.java
index 34d9b1763..a626b8538 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchSet.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchSet.java
@@ -38,7 +38,7 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class FetchSet extends FetchDatatype
+public final class FetchSet extends FetchDatatype
 {
     private FetchSet(Builder builder)
     {
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateCounter.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateCounter.java
index 6c4c269d3..ca5bd28f8 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateCounter.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateCounter.java
@@ -46,7 +46,7 @@
  * @author Brian Roach 
  * @since 2.0
  */
-public class UpdateCounter extends UpdateDatatype
+public class UpdateCounter extends UpdateDatatype
 {
     private UpdateCounter(Builder builder)
     {
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateDatatype.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateDatatype.java
index 5ff19e743..2f3849299 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateDatatype.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateDatatype.java
@@ -35,7 +35,8 @@
  * @author Brian Roach 
  * @since 2.0
  */
-public abstract class UpdateDatatype extends GenericRiakCommand
+public abstract class UpdateDatatype
+        extends GenericRiakCommand.GenericRiakCommandWithSameInfo
 {
     protected final Namespace namespace;
     protected final BinaryValue key;
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java
index d2ed465f1..09ec41ae0 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java
@@ -46,7 +46,7 @@
  * @since 2.1.0
  */
  
-public class UpdateHll extends UpdateDatatype
+public class UpdateHll extends UpdateDatatype
 {
     private UpdateHll(Builder builder)
     {
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateMap.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateMap.java
index 2865ce887..afc89d963 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateMap.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateMap.java
@@ -47,7 +47,7 @@
  * @author Brian Roach 
  * @since 2.0
  */
-public class UpdateMap extends UpdateDatatype
+public class UpdateMap extends UpdateDatatype
 {
     private UpdateMap(Builder builder)
     {
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateSet.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateSet.java
index d370dcfc6..cf7455904 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateSet.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateSet.java
@@ -46,7 +46,7 @@
  * @author Brian Roach 
  * @since 2.0
  */
-public class UpdateSet extends UpdateDatatype
+public class UpdateSet extends UpdateDatatype
 {
     private UpdateSet(Builder builder)
     {
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/CoveragePlan.java b/src/main/java/com/basho/riak/client/api/commands/kv/CoveragePlan.java
index 25b0c5bb7..61a940843 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/CoveragePlan.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/CoveragePlan.java
@@ -24,7 +24,8 @@
  *
  * @author Sergey Galkin 
  */
-public class CoveragePlan  extends GenericRiakCommand
+public class CoveragePlan  extends GenericRiakCommand.GenericRiakCommandWithSameInfo
 {
     private final CoveragePlanOperation operation;
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/FetchValue.java b/src/main/java/com/basho/riak/client/api/commands/kv/FetchValue.java
index c7a5caf75..5e0711861 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/FetchValue.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/FetchValue.java
@@ -69,8 +69,8 @@
  * @since 2.0
  * @see Response
  */
-public final class FetchValue extends GenericRiakCommand
+public final class FetchValue extends GenericRiakCommand.GenericRiakCommandWithSameInfo
 {
     private final Location location;
     private final Map, Object> options = new HashMap<>();
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
index 69927cea0..633f8c980 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
@@ -73,8 +73,8 @@
  * @author Alex Moore 
  * @since 2.0
  */
-public final class ListKeys extends StreamableRiakCommand
+public final class ListKeys extends StreamableRiakCommand.StreamableRiakCommandWithSameInfo
 {
     private final Namespace namespace;
     private final int timeout;
diff --git a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
index 184b39975..f49041b6b 100644
--- a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
+++ b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
@@ -45,8 +45,8 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public abstract class MapReduce extends StreamableRiakCommand
+public abstract class MapReduce extends StreamableRiakCommand.StreamableRiakCommandWithSameInfo
 {
     private final MapReduceSpec spec;
 

From d97183ad70e6fec5de2cef72157c85e7c4806ebd Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Tue, 8 Nov 2016 11:07:30 +0200
Subject: [PATCH 40/55] Get rid of unused GenericRiakCommand constructor

---
 .../riak/client/api/GenericRiakCommand.java   | 25 ++-----------------
 .../client/api/StreamableRiakCommand.java     |  7 ------
 2 files changed, 2 insertions(+), 30 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
index 532521202..39d467633 100644
--- a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
@@ -36,27 +36,6 @@ protected I convertInfo(I coreInfo) {
         }
     }
 
-    @FunctionalInterface
-    protected interface Converter
-    {
-        T convert(O source);
-    }
-
-    private Converter responseConverter;
-    private Converter infoConverter;
-
-    public GenericRiakCommand()
-    {
-        this.responseConverter = this::convertResponse;
-        this.infoConverter = this::convertInfo;
-    }
-
-    public GenericRiakCommand(final Converter responseConverter, final Converter infoConverter)
-    {
-        this.responseConverter = responseConverter;
-        this.infoConverter = infoConverter;
-    }
-
     protected abstract FutureOperation buildCoreOperation();
 
     protected RiakFuture executeAsync(RiakCluster cluster)
@@ -83,13 +62,13 @@ protected RiakFuture executeAsync(RiakCluster cluster)
                     @Override
                     protected R convertResponse(CoreR coreResponse)
                     {
-                        return responseConverter.convert(coreResponse);
+                        return GenericRiakCommand.this.convertResponse(coreResponse);
                     }
 
                     @Override
                     protected I convertQueryInfo(CoreI coreQueryInfo)
                     {
-                        return infoConverter.convert(coreQueryInfo);
+                        return GenericRiakCommand.this.convertInfo(coreQueryInfo);
                     }
                 };
         coreFuture.addListener(future);
diff --git a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
index 72d7c8f99..fc367b119 100644
--- a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
@@ -43,13 +43,6 @@ protected I convertInfo(I coreInfo) {
         }
     }
 
-    public StreamableRiakCommand() {
-    }
-
-    public StreamableRiakCommand(Converter responseConverter, Converter infoConverter) {
-        super(responseConverter, infoConverter);
-    }
-
     protected abstract RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout);
 
     @Override

From 76a6ee440caa5b65042ce4da1a401b7281e37ed6 Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Tue, 8 Nov 2016 14:25:26 +0200
Subject: [PATCH 41/55] Request argument was added to the
 GenericRiakCommand.convertResponse signature

---
 .../riak/client/api/GenericRiakCommand.java   |  4 +-
 .../api/commands/buckets/ListBuckets.java     |  5 +-
 .../api/commands/datatypes/FetchCounter.java  |  5 +-
 .../api/commands/datatypes/FetchHll.java      |  5 +-
 .../api/commands/datatypes/FetchMap.java      |  5 +-
 .../api/commands/datatypes/FetchSet.java      |  4 +-
 .../api/commands/datatypes/UpdateCounter.java |  5 +-
 .../api/commands/datatypes/UpdateHll.java     |  5 +-
 .../api/commands/datatypes/UpdateMap.java     |  5 +-
 .../api/commands/datatypes/UpdateSet.java     |  5 +-
 .../commands/indexes/SecondaryIndexQuery.java |  4 +-
 .../client/api/commands/kv/CoveragePlan.java  |  5 +-
 .../client/api/commands/kv/FetchValue.java    |  4 +-
 .../riak/client/api/commands/kv/ListKeys.java |  5 +-
 .../client/api/commands/kv/StoreValue.java    | 69 ++++++++-----------
 .../api/commands/mapreduce/MapReduce.java     |  4 +-
 16 files changed, 82 insertions(+), 57 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
index 39d467633..974c923b5 100644
--- a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
@@ -62,7 +62,7 @@ protected RiakFuture executeAsync(RiakCluster cluster)
                     @Override
                     protected R convertResponse(CoreR coreResponse)
                     {
-                        return GenericRiakCommand.this.convertResponse(coreResponse);
+                        return GenericRiakCommand.this.convertResponse(coreOperation, coreResponse);
                     }
 
                     @Override
@@ -75,7 +75,7 @@ protected I convertQueryInfo(CoreI coreQueryInfo)
         return future;
     }
 
-    protected abstract R convertResponse(CoreR coreResponse);
+    protected abstract R convertResponse(FutureOperation request, CoreR coreResponse);
 
     protected abstract I convertInfo(CoreI coreInfo);
 }
\ No newline at end of file
diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
index a31c23b3c..e1428d12e 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
@@ -18,6 +18,7 @@
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
 import com.basho.riak.client.api.StreamableRiakCommand;
 import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.StreamingRiakFuture;
@@ -76,7 +77,9 @@ public final class ListBuckets extends StreamableRiakCommand.StreamableRiakComma
     }
 
     @Override
-    protected Response convertResponse(ListBucketsOperation.Response coreResponse) {
+    protected Response convertResponse(FutureOperation request,
+                                       ListBucketsOperation.Response coreResponse)
+    {
         return new Response(type, coreResponse.getBuckets());
     }
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchCounter.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchCounter.java
index fc40d1f42..7a47c3d05 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchCounter.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchCounter.java
@@ -16,6 +16,7 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.DtFetchOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.crdt.types.RiakCounter;
@@ -45,7 +46,9 @@ private FetchCounter(Builder builder)
     }
 
     @Override
-    protected Response convertResponse(DtFetchOperation.Response coreResponse) {
+    protected Response convertResponse(FutureOperation request,
+                                       DtFetchOperation.Response coreResponse)
+    {
         RiakDatatype element = coreResponse.getCrdtElement();
 
         Context context = null;
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java
index 48e2157a1..60c3dcdef 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchHll.java
@@ -16,6 +16,7 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.DtFetchOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.crdt.types.RiakDatatype;
@@ -46,7 +47,9 @@ private FetchHll(Builder builder)
     }
 
     @Override
-    protected RiakHll convertResponse(DtFetchOperation.Response coreResponse) {
+    protected RiakHll convertResponse(FutureOperation request,
+                                      DtFetchOperation.Response coreResponse)
+    {
         RiakDatatype element = coreResponse.getCrdtElement();
 
         RiakHll datatype = extractDatatype(element);
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchMap.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchMap.java
index a66ef394c..3f4101130 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchMap.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchMap.java
@@ -16,6 +16,7 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.DtFetchOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.crdt.types.RiakDatatype;
@@ -53,7 +54,9 @@ public RiakMap extractDatatype(RiakDatatype element)
     }
 
     @Override
-    protected Response convertResponse(DtFetchOperation.Response coreResponse) {
+    protected Response convertResponse(FutureOperation request,
+                                       DtFetchOperation.Response coreResponse)
+    {
         RiakDatatype element = coreResponse.getCrdtElement();
 
         Context context = null;
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchSet.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchSet.java
index a626b8538..aed1ea64d 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchSet.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/FetchSet.java
@@ -16,6 +16,7 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.DtFetchOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.crdt.types.RiakDatatype;
@@ -46,7 +47,8 @@ private FetchSet(Builder builder)
     }
 
     @Override
-    protected Response convertResponse(DtFetchOperation.Response coreResponse) {
+    protected Response convertResponse(FutureOperation request, DtFetchOperation.Response coreResponse)
+    {
         RiakDatatype element = coreResponse.getCrdtElement();
 
         Context context = null;
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateCounter.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateCounter.java
index ca5bd28f8..19842e891 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateCounter.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateCounter.java
@@ -16,6 +16,7 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.DtUpdateOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
@@ -54,7 +55,9 @@ private UpdateCounter(Builder builder)
     }
 
     @Override
-    protected Response convertResponse(DtUpdateOperation.Response coreResponse) {
+    protected Response convertResponse(FutureOperation request,
+                                       DtUpdateOperation.Response coreResponse)
+    {
         RiakCounter counter = null;
         if (coreResponse.hasCrdtElement())
         {
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java
index 09ec41ae0..034e22f88 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateHll.java
@@ -16,6 +16,7 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.DtUpdateOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
@@ -54,7 +55,9 @@ private UpdateHll(Builder builder)
     }
 
     @Override
-    protected Response convertResponse(DtUpdateOperation.Response coreResponse) {
+    protected Response convertResponse(FutureOperation request,
+                                       DtUpdateOperation.Response coreResponse)
+    {
         RiakHll hll = null;
         if (coreResponse.hasCrdtElement())
         {
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateMap.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateMap.java
index afc89d963..ba7786a40 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateMap.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateMap.java
@@ -16,6 +16,7 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.DtUpdateOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
@@ -55,7 +56,9 @@ private UpdateMap(Builder builder)
     }
 
     @Override
-    protected Response convertResponse(DtUpdateOperation.Response coreResponse) {
+    protected Response convertResponse(FutureOperation request,
+                                       DtUpdateOperation.Response coreResponse)
+    {
         RiakMap map = null;
         if (coreResponse.hasCrdtElement())
         {
diff --git a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateSet.java b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateSet.java
index cf7455904..e5305d2f4 100644
--- a/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateSet.java
+++ b/src/main/java/com/basho/riak/client/api/commands/datatypes/UpdateSet.java
@@ -16,6 +16,7 @@
 
 package com.basho.riak.client.api.commands.datatypes;
 
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.DtUpdateOperation;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
@@ -54,7 +55,9 @@ private UpdateSet(Builder builder)
     }
 
     @Override
-    protected Response convertResponse(DtUpdateOperation.Response coreResponse) {
+    protected Response convertResponse(FutureOperation request,
+                                       DtUpdateOperation.Response coreResponse)
+    {
         RiakSet set = null;
         if (coreResponse.hasCrdtElement())
         {
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
index 32a95b342..359549331 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
@@ -19,6 +19,7 @@
 import com.basho.riak.client.api.StreamableRiakCommand;
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
 import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.StreamingRiakFuture;
@@ -257,7 +258,8 @@ protected SecondaryIndexQueryOperation buildCoreOperation(boolean streamResults)
     }
 
     @Override
-    protected S convertResponse(SecondaryIndexQueryOperation.Response coreResponse)
+    protected S convertResponse(FutureOperation request, SecondaryIndexQueryOperation.Response coreResponse)
     {
         return convertResponse(coreResponse, null);
     }
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/CoveragePlan.java b/src/main/java/com/basho/riak/client/api/commands/kv/CoveragePlan.java
index 61a940843..e2122955d 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/CoveragePlan.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/CoveragePlan.java
@@ -16,6 +16,7 @@
 package com.basho.riak.client.api.commands.kv;
 
 import com.basho.riak.client.api.GenericRiakCommand;
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.operations.CoveragePlanOperation;
 import com.basho.riak.client.core.query.Namespace;
 
@@ -40,7 +41,9 @@ protected CoveragePlanOperation buildCoreOperation() {
     }
 
     @Override
-    protected Response convertResponse(CoveragePlanOperation.Response coreResponse) {
+    protected Response convertResponse(FutureOperation request,
+                                       CoveragePlanOperation.Response coreResponse)
+    {
         return new Response(coreResponse);
     }
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/FetchValue.java b/src/main/java/com/basho/riak/client/api/commands/kv/FetchValue.java
index 5e0711861..8e1876527 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/FetchValue.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/FetchValue.java
@@ -18,6 +18,7 @@
 import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.api.cap.Quorum;
 import com.basho.riak.client.api.cap.VClock;
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.operations.FetchOperation;
 import com.basho.riak.client.core.RiakFuture;
@@ -88,7 +89,8 @@ protected final RiakFuture executeAsync(RiakCluster cluster)
     }
 
     @Override
-    protected Response convertResponse(FetchOperation.Response coreResponse)
+    protected Response convertResponse(FutureOperation request,
+                                       FetchOperation.Response coreResponse)
     {
         return new Response.Builder().withNotFound(coreResponse.isNotFound())
                             .withUnchanged(coreResponse.isUnchanged())
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
index 633f8c980..c9df6281f 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
@@ -18,6 +18,7 @@
 import com.basho.riak.client.api.StreamableRiakCommand;
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
 import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.StreamingRiakFuture;
@@ -86,7 +87,9 @@ public final class ListKeys extends StreamableRiakCommand.StreamableRiakCommandW
     }
 
     @Override
-    protected Response convertResponse(ListKeysOperation.Response coreResponse) {
+    protected Response convertResponse(FutureOperation request,
+                                       ListKeysOperation.Response coreResponse)
+    {
         return new Response(namespace, coreResponse.getKeys());
     }
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/StoreValue.java b/src/main/java/com/basho/riak/client/api/commands/kv/StoreValue.java
index 660d83fd4..3b7e17739 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/StoreValue.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/StoreValue.java
@@ -15,16 +15,16 @@
  */
 package com.basho.riak.client.api.commands.kv;
 
+import com.basho.riak.client.api.GenericRiakCommand;
 import com.basho.riak.client.api.cap.Quorum;
 import com.basho.riak.client.api.cap.VClock;
 import com.basho.riak.client.api.convert.Converter;
 import com.basho.riak.client.api.convert.Converter.OrmExtracted;
 import com.basho.riak.client.api.convert.ConverterFactory;
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.operations.StoreOperation;
-import com.basho.riak.client.api.RiakCommand;
 import com.basho.riak.client.core.RiakFuture;
-import com.basho.riak.client.api.commands.CoreFutureAdapter;
 import com.basho.riak.client.api.commands.RiakOption;
 import com.basho.riak.client.core.util.BinaryValue;
 
@@ -69,7 +69,8 @@
  * @author Dave Rusek 
  * @since 2.0
  */
-public final class StoreValue extends RiakCommand
+public final class StoreValue extends GenericRiakCommand.GenericRiakCommandWithSameInfo
 {
     private final Namespace namespace;
     private final BinaryValue key;
@@ -88,9 +89,32 @@ public final class StoreValue extends RiakCommand
         this.vclock = builder.vclock;
     }
 
-    @SuppressWarnings("unchecked")
+    @Override
+    protected Response convertResponse(FutureOperation request,
+                                       StoreOperation.Response coreResponse)
+    {
+        Location loc = request.getQueryInfo();
+        if (coreResponse.hasGeneratedKey())
+        {
+            loc = new Location(loc.getNamespace(), coreResponse.getGeneratedKey());
+        }
+
+        return new Response.Builder()
+                .withValues(coreResponse.getObjectList())
+                .withGeneratedKey(loc.getKey())
+                .withLocation(loc) // for ORM
+                .build();
+    }
+
     @Override
     protected RiakFuture executeAsync(RiakCluster cluster)
+    {
+        return super.executeAsync(cluster);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    protected StoreOperation buildCoreOperation()
     {
         Converter converter;
 
@@ -112,43 +136,6 @@ protected RiakFuture executeAsync(RiakCluster cluster)
             orm.getRiakObject().setVClock(vclock);
         }
 
-        RiakFuture coreFuture =
-            cluster.execute(buildCoreOperation(orm));
-
-        CoreFutureAdapter future =
-            new CoreFutureAdapter(coreFuture)
-            {
-                @Override
-                protected Response convertResponse(StoreOperation.Response coreResponse)
-                {
-                    Namespace ns = orm.getNamespace();
-                    BinaryValue key = orm.getKey();
-                    if (coreResponse.hasGeneratedKey())
-                    {
-                        key = coreResponse.getGeneratedKey();
-                    }
-
-                    Location loc = new Location(ns, key);
-
-                    return new Response.Builder()
-                        .withValues(coreResponse.getObjectList())
-                        .withGeneratedKey(coreResponse.getGeneratedKey())
-                        .withLocation(loc) // for ORM
-                        .build();
-                }
-
-                @Override
-                protected Location convertQueryInfo(Location coreQueryInfo)
-                {
-                    return coreQueryInfo;
-                }
-            };
-        coreFuture.addListener(future);
-        return future;
-    }
-
-    private StoreOperation buildCoreOperation(OrmExtracted orm)
-    {
         StoreOperation.Builder builder;
 
         if (orm.hasKey())
diff --git a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
index f49041b6b..dadf877bf 100644
--- a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
+++ b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
@@ -17,6 +17,7 @@
 import com.basho.riak.client.api.StreamableRiakCommand;
 import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
 import com.basho.riak.client.api.convert.ConversionException;
+import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.RiakCluster;
 import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.StreamingRiakFuture;
@@ -76,7 +77,8 @@ protected MapReduceOperation buildCoreOperation(boolean streamResults)
     }
 
     @Override
-    protected Response convertResponse(MapReduceOperation.Response coreResponse)
+    protected Response convertResponse(FutureOperation request,
+                                       MapReduceOperation.Response coreResponse)
     {
         return new Response(coreResponse.getResults());
     }

From 7cdf043e755cba92120ab81833ff192fac7a8e7a Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Tue, 8 Nov 2016 19:53:49 +0200
Subject: [PATCH 42/55] Generalize
 StreamableRiakCommand.executeAsyncStreaming()

---
 .../client/api/StreamableRiakCommand.java     |  36 +++++-
 .../api/commands/buckets/ListBuckets.java     |  17 +--
 .../commands/indexes/BigIntIndexQuery.java    |  10 +-
 .../api/commands/indexes/BinIndexQuery.java   |  11 +-
 .../api/commands/indexes/IntIndexQuery.java   |  10 +-
 .../api/commands/indexes/RawIndexQuery.java   |  10 +-
 .../commands/indexes/SecondaryIndexQuery.java | 103 +++++-------------
 .../api/commands/kv/FullBucketRead.java       |  10 +-
 .../riak/client/api/commands/kv/ListKeys.java |  17 +--
 .../api/commands/mapreduce/MapReduce.java     |  18 +--
 10 files changed, 95 insertions(+), 147 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
index fc367b119..ab46943eb 100644
--- a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
@@ -16,9 +16,8 @@
 
 package com.basho.riak.client.api;
 
-import com.basho.riak.client.core.FutureOperation;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
+import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
+import com.basho.riak.client.core.*;
 
 /*
  * The base class for all Streamable Riak Commands.
@@ -31,6 +30,7 @@
  * @param  The query info type
  * @author Dave Rusek
  * @author Brian Roach 
+ * @author Sergey Galkin 
  * @since 2.0
  */
 public abstract class StreamableRiakCommand extends GenericRiakCommand
@@ -43,12 +43,38 @@ protected I convertInfo(I coreInfo) {
         }
     }
 
-    protected abstract RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout);
+    protected abstract R createResponse(int timeout, StreamingRiakFuture coreFuture);
+
+    protected abstract  FutureOperation buildCoreOperation(boolean streamResults);
 
     @Override
     protected final FutureOperation buildCoreOperation() {
         return buildCoreOperation(false);
     }
 
-    protected abstract  FutureOperation buildCoreOperation(boolean streamResults);
+    protected final RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
+    {
+        final PBStreamingFutureOperation coreOperation = (PBStreamingFutureOperation)buildCoreOperation(true);
+        final StreamingRiakFuture coreFuture = cluster.execute(coreOperation);
+
+        final R r = createResponse(timeout, coreFuture);
+
+        final ImmediateCoreFutureAdapter future = new ImmediateCoreFutureAdapter(coreFuture, r)
+        {
+            @Override
+            protected R convertResponse(CoreR response)
+            {
+                return StreamableRiakCommand.this.convertResponse(coreOperation, response);
+            }
+
+            @Override
+            protected I convertQueryInfo(CoreI coreQueryInfo)
+            {
+                return StreamableRiakCommand.this.convertInfo(coreQueryInfo);
+            }
+        };
+
+        coreFuture.addListener(future);
+        return future;
+    }
 }
diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
index e1428d12e..cc67e50c4 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
@@ -17,10 +17,7 @@
 
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
 import com.basho.riak.client.api.StreamableRiakCommand;
-import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
 import com.basho.riak.client.core.FutureOperation;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.ListBucketsOperation;
 import com.basho.riak.client.core.query.Namespace;
@@ -84,19 +81,9 @@ protected Response convertResponse(FutureOperation executeAsyncStreaming(RiakCluster cluster, int timeout)
+    protected Response createResponse(int timeout, StreamingRiakFuture coreFuture)
     {
-        StreamingRiakFuture coreFuture =
-                cluster.execute(buildCoreOperation(true));
-
-        final Response streamingResponse = new Response(type, timeout, coreFuture);
-
-        ImmediateCoreFutureAdapter.SameQueryInfo future =
-                new ImmediateCoreFutureAdapter.SameQueryInfo
-                        (coreFuture, streamingResponse) {};
-
-        coreFuture.addListener(future);
-        return future;
+        return new Response(type, timeout, coreFuture);
     }
 
     @Override
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java
index 52933a788..382567ea2 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BigIntIndexQuery.java
@@ -16,7 +16,7 @@
 
 package com.basho.riak.client.api.commands.indexes;
 
-import com.basho.riak.client.api.commands.ChunkedResponseIterator;
+import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
@@ -198,11 +198,13 @@ public BigIntIndexQuery build()
 
     public static class Response extends SecondaryIndexQuery.Response>
     {
-        protected Response(Namespace queryLocation, IndexConverter converter, ChunkedResponseIterator chunkedResponseIterator) {
-            super(queryLocation, converter, chunkedResponseIterator);
+        Response(Namespace queryLocation, IndexConverter converter, int timeout, StreamingRiakFuture coreFuture)
+        {
+            super(queryLocation, converter, timeout, coreFuture);
         }
 
-        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter) {
+        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter)
+        {
             super(queryLocation, coreResponse, converter);
         }
     }
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java
index 4bb1c1ed7..dfb881948 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/BinIndexQuery.java
@@ -16,7 +16,7 @@
 
 package com.basho.riak.client.api.commands.indexes;
 
-import com.basho.riak.client.api.commands.ChunkedResponseIterator;
+import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.query.indexes.IndexNames;
@@ -236,12 +236,13 @@ public BinIndexQuery build()
 
     public static class Response extends SecondaryIndexQuery.Response>
     {
-
-        protected Response(Namespace queryLocation, IndexConverter converter, ChunkedResponseIterator chunkedResponseIterator) {
-            super(queryLocation, converter, chunkedResponseIterator);
+        Response(Namespace queryLocation, IndexConverter converter, int timeout, StreamingRiakFuture coreFuture)
+        {
+            super(queryLocation, converter, timeout, coreFuture);
         }
 
-        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter) {
+        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter)
+        {
             super(queryLocation, coreResponse, converter);
         }
     }
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java
index 50ac2dca3..5ae04345f 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java
@@ -16,7 +16,7 @@
 
 package com.basho.riak.client.api.commands.indexes;
 
-import com.basho.riak.client.api.commands.ChunkedResponseIterator;
+import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
@@ -195,11 +195,13 @@ public IntIndexQuery build()
 
     public static class Response extends SecondaryIndexQuery.Response>
     {
-        protected Response(Namespace queryLocation, IndexConverter converter, ChunkedResponseIterator chunkedResponseIterator) {
-            super(queryLocation, converter, chunkedResponseIterator);
+        Response(Namespace queryLocation, IndexConverter converter, int timeout, StreamingRiakFuture coreFuture)
+        {
+            super(queryLocation, converter, timeout, coreFuture);
         }
 
-        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter) {
+        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter)
+        {
             super(queryLocation, coreResponse, converter);
         }
     }
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java
index fa6714eef..54efce718 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/RawIndexQuery.java
@@ -16,7 +16,7 @@
 
 package com.basho.riak.client.api.commands.indexes;
 
-import com.basho.riak.client.api.commands.ChunkedResponseIterator;
+import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
@@ -147,11 +147,13 @@ public RawIndexQuery build()
 
     public static class Response extends SecondaryIndexQuery.Response>
     {
-        protected Response(Namespace queryLocation, IndexConverter converter, ChunkedResponseIterator chunkedResponseIterator) {
-            super(queryLocation, converter, chunkedResponseIterator);
+        Response(Namespace queryLocation, IndexConverter converter, int timeout, StreamingRiakFuture coreFuture)
+        {
+            super(queryLocation, converter, timeout, coreFuture);
         }
 
-        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter) {
+        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter)
+        {
             super(queryLocation, coreResponse, converter);
         }
     }
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
index 359549331..acb0ca7ce 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
@@ -18,10 +18,7 @@
 
 import com.basho.riak.client.api.StreamableRiakCommand;
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
-import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
 import com.basho.riak.client.core.FutureOperation;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
 import com.basho.riak.client.core.query.Location;
@@ -44,16 +41,16 @@
  * @author Sergey Galkin 
  * @since 2.0
  */
-public abstract class SecondaryIndexQuery, U> extends StreamableRiakCommand
+public abstract class SecondaryIndexQuery, U extends SecondaryIndexQuery>
+        extends StreamableRiakCommand
 {
     @FunctionalInterface
     public interface StreamableResponseCreator>
     {
-
         R createResponse(Namespace queryLocation,
                          IndexConverter converter,
-                         ChunkedResponseIterator chunkedResponseIterator);
+                         int timeout,
+                         StreamingRiakFuture coreFuture);
     }
 
     @FunctionalInterface
@@ -261,7 +258,7 @@ protected SecondaryIndexQueryOperation buildCoreOperation(boolean streamResults)
     protected S convertResponse(FutureOperation request, SecondaryIndexQueryOperation.Response coreResponse)
     {
-        return convertResponse(coreResponse, null);
+        return gatherableResponseCreator.createResponse(namespace, coreResponse, getConverter());
     }
 
     @Override
@@ -271,60 +268,10 @@ protected U convertInfo(SecondaryIndexQueryOperation.Query coreInfo)
         return (U)SecondaryIndexQuery.this;
     }
 
-    private S convertResponse(final SecondaryIndexQueryOperation.Response coreResponse,
-                              final ChunkedResponseIterator iterator)
-    {
-        final S response;
-
-        if (iterator != null)
-        {
-            response = streamableResponseCreator.createResponse(namespace, getConverter(), iterator);
-        }
-        else
-        {
-            response = gatherableResponseCreator.createResponse(namespace, coreResponse, getConverter());
-        }
-
-        return response;
-    }
-
     @Override
-    protected final RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
+    protected S createResponse(int timeout, StreamingRiakFuture coreFuture)
     {
-        final SecondaryIndexQueryOperation coreOperation = buildCoreOperation(true);
-
-        final StreamingRiakFuture coreFuture =
-                cluster.execute(coreOperation);
-
-        final Response[] responses = {null};
-
-        final ChunkedResponseIterator iterator = new ChunkedResponseIterator(
-            coreFuture, timeout, null,
-            SecondaryIndexQueryOperation.Response::iterator,
-            SecondaryIndexQueryOperation.Response::getContinuation)
-        {
-            @SuppressWarnings("unchecked")
-            @Override
-            public Response.Entry next() {
-                final SecondaryIndexQueryOperation.Response.Entry coreEntity = currentIterator.next();
-                return responses[0].createEntry(namespace, coreEntity, getConverter());
-            }
-        };
-
-
-        final S response = convertResponse(null, iterator);
-        responses[0] = response;
-
-        final ImmediateCoreFutureAdapter future = new ImmediateCoreFutureAdapter(coreFuture, response) {
-            @SuppressWarnings("unchecked")
-            @Override
-            protected U convertQueryInfo(SecondaryIndexQueryOperation.Query coreQueryInfo) {
-                return (U)SecondaryIndexQuery.this;
-            }
-        };
-
-        coreFuture.addListener(future);
-        return future;
+        return streamableResponseCreator.createResponse(namespace, getConverter(), timeout, coreFuture);
     }
 
     @Override
@@ -647,29 +594,35 @@ public static class Response> implements Iterable
         final protected Namespace queryLocation;
         private final ChunkedResponseIterator chunkedResponseIterator;
 
-        private Response(Namespace queryLocation,
-                         IndexConverter converter,
-                         ChunkedResponseIterator chunkedResponseIterator,
-                         SecondaryIndexQueryOperation.Response coreResponse)
+        protected Response(final Namespace queryLocation, IndexConverter converter, final int timeout,
+                 final StreamingRiakFuture coreFuture)
         {
-            this.converter = converter;
             this.queryLocation = queryLocation;
-            this.chunkedResponseIterator = chunkedResponseIterator;
-            this.coreResponse = coreResponse;
-        }
-
-        protected Response(Namespace queryLocation,
-                           IndexConverter converter,
-                           ChunkedResponseIterator chunkedResponseIterator)
-        {
-            this(queryLocation, converter, chunkedResponseIterator, null);
+            this.converter = converter;
+            this.coreResponse = null;
+            chunkedResponseIterator = new ChunkedResponseIterator(
+                    coreFuture, timeout, null,
+                    SecondaryIndexQueryOperation.Response::iterator,
+                    SecondaryIndexQueryOperation.Response::getContinuation)
+            {
+                @SuppressWarnings("unchecked")
+                @Override
+                public Response.Entry next()
+                {
+                    final SecondaryIndexQueryOperation.Response.Entry coreEntity = currentIterator.next();
+                    return Response.this.createEntry(Response.this.queryLocation, coreEntity, converter);
+                }
+            };
         }
 
         protected Response(Namespace queryLocation,
                            SecondaryIndexQueryOperation.Response coreResponse,
                            IndexConverter converter)
         {
-            this(queryLocation, converter, null, coreResponse);
+            this.queryLocation = queryLocation;
+            this.converter = converter;
+            this.coreResponse = coreResponse;
+            chunkedResponseIterator = null;
         }
 
         public boolean isStreamable()
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java b/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java
index faed8bd48..62856cd10 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/FullBucketRead.java
@@ -15,8 +15,8 @@
  */
 package com.basho.riak.client.api.commands.kv;
 
-import com.basho.riak.client.api.commands.ChunkedResponseIterator;
 import com.basho.riak.client.api.commands.indexes.SecondaryIndexQuery;
+import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.FetchOperation;
 import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
 import com.basho.riak.client.core.query.Location;
@@ -201,11 +201,13 @@ public static class Response extends SecondaryIndexQuery.Response convertedList = null;
 
-        protected Response(Namespace queryLocation, IndexConverter converter, ChunkedResponseIterator chunkedResponseIterator) {
-            super(queryLocation, converter, chunkedResponseIterator);
+        protected Response(Namespace queryLocation, IndexConverter converter, final int timeout, StreamingRiakFuture coreFuture)
+        {
+            super(queryLocation, converter, timeout, coreFuture);
         }
 
-        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter) {
+        protected Response(Namespace queryLocation, SecondaryIndexQueryOperation.Response coreResponse, IndexConverter converter)
+        {
             super(queryLocation, coreResponse, converter);
         }
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
index c9df6281f..9e982278f 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
@@ -17,10 +17,7 @@
 
 import com.basho.riak.client.api.StreamableRiakCommand;
 import com.basho.riak.client.api.commands.ChunkedResponseIterator;
-import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
 import com.basho.riak.client.core.FutureOperation;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.ListKeysOperation;
 import com.basho.riak.client.core.query.Location;
@@ -94,19 +91,9 @@ protected Response convertResponse(FutureOperation executeAsyncStreaming(RiakCluster cluster, int timeout)
+    protected Response createResponse(int timeout, StreamingRiakFuture coreFuture)
     {
-        StreamingRiakFuture coreFuture =
-                cluster.execute(buildCoreOperation(true));
-
-        final Response streamingResponse = new Response(namespace, timeout, coreFuture);
-
-        ImmediateCoreFutureAdapter.SameQueryInfo future =
-                new ImmediateCoreFutureAdapter.SameQueryInfo(
-                        coreFuture, streamingResponse) {};
-
-        coreFuture.addListener(future);
-        return future;
+        return new Response(namespace, timeout, coreFuture);
     }
 
     @Override
diff --git a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
index dadf877bf..a999b214a 100644
--- a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
+++ b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java
@@ -15,11 +15,8 @@
 
 import com.basho.riak.client.api.RiakException;
 import com.basho.riak.client.api.StreamableRiakCommand;
-import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
 import com.basho.riak.client.api.convert.ConversionException;
 import com.basho.riak.client.core.FutureOperation;
-import com.basho.riak.client.core.RiakCluster;
-import com.basho.riak.client.core.RiakFuture;
 import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.MapReduceOperation;
 import com.basho.riak.client.core.query.functions.Function;
@@ -84,20 +81,9 @@ protected Response convertResponse(FutureOperation executeAsyncStreaming(RiakCluster cluster, int timeout)
+    protected Response createResponse(int timeout, StreamingRiakFuture coreFuture)
     {
-        final MapReduceOperation operation = buildCoreOperation(true);
-        final StreamingRiakFuture coreFuture = cluster.execute(operation);
-
-        final Response response = new Response(coreFuture, timeout);
-
-        ImmediateCoreFutureAdapter.SameQueryInfo future =
-                new ImmediateCoreFutureAdapter.SameQueryInfo(
-                        coreFuture, response) {};
-
-        coreFuture.addListener(future);
-
-        return future;
+        return new Response(coreFuture, timeout);
     }
 
     /**

From 45f55c4a10fdb9559a928044bf44405131180ba2 Mon Sep 17 00:00:00 2001
From: Alex Moore 
Date: Tue, 8 Nov 2016 15:37:49 -0500
Subject: [PATCH 43/55] Remove if / subtype casting from Command execution

---
 .../com/basho/riak/client/api/AsIsRiakCommand.java  | 13 +------------
 .../basho/riak/client/api/GenericRiakCommand.java   | 13 ++-----------
 .../com/basho/riak/client/core/RiakCluster.java     |  5 ++++-
 .../api/commands/buckets/ListBucketsTest.java       | 10 ++++++----
 4 files changed, 13 insertions(+), 28 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java b/src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java
index d0f4be59b..17fb0a13f 100644
--- a/src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java
@@ -32,18 +32,7 @@ protected RiakFuture executeAsync(RiakCluster cluster)
     {
         final FutureOperation coreOperation = buildCoreOperation();
 
-        // TODO: WE NEED TO GET RID SUCH A WEIRD IF-FORK
-        final RiakFuture coreFuture;
-        if (coreOperation instanceof PBStreamingFutureOperation)
-        {
-            coreFuture = cluster.execute((PBStreamingFutureOperation) coreOperation);
-        }
-        else
-        {
-            coreFuture = cluster.execute(coreOperation);
-        }
-
-        return coreFuture;
+        return cluster.execute(coreOperation);
     }
 
 }
diff --git a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
index 974c923b5..c6f25b545 100644
--- a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java
@@ -43,16 +43,7 @@ protected RiakFuture executeAsync(RiakCluster cluster)
         final FutureOperation coreOperation = buildCoreOperation();
         assert coreOperation != null;
 
-        // TODO: WE NEED TO GET RID SUCH A WEIRD IF-FORK
-        final RiakFuture coreFuture;
-        if (coreOperation instanceof PBStreamingFutureOperation)
-        {
-            coreFuture = cluster.execute((PBStreamingFutureOperation) coreOperation);
-        }
-        else
-        {
-            coreFuture = cluster.execute(coreOperation);
-        }
+        final RiakFuture coreFuture = cluster.execute(coreOperation);
 
         assert coreFuture != null;
 
@@ -78,4 +69,4 @@ protected I convertQueryInfo(CoreI coreQueryInfo)
     protected abstract R convertResponse(FutureOperation request, CoreR coreResponse);
 
     protected abstract I convertInfo(CoreI coreInfo);
-}
\ No newline at end of file
+}
diff --git a/src/main/java/com/basho/riak/client/core/RiakCluster.java b/src/main/java/com/basho/riak/client/core/RiakCluster.java
index 772a7684b..3613a9939 100644
--- a/src/main/java/com/basho/riak/client/core/RiakCluster.java
+++ b/src/main/java/com/basho/riak/client/core/RiakCluster.java
@@ -226,7 +226,10 @@ public  RiakFuture execute(FutureOperation operation)
 
     public  StreamingRiakFuture execute(PBStreamingFutureOperation operation)
     {
-        executeFutureOperation(operation);
+        execute((FutureOperation)operation);
+        // N.B. Currently the operation and future are one in the same,
+        // so we can just return the operation to accomplish our
+        // StreamingRiakFuture return type contract.
         return operation;
     }
 
diff --git a/src/test/java/com/basho/riak/client/api/commands/buckets/ListBucketsTest.java b/src/test/java/com/basho/riak/client/api/commands/buckets/ListBucketsTest.java
index 2cb7bbc7e..e37b123ec 100644
--- a/src/test/java/com/basho/riak/client/api/commands/buckets/ListBucketsTest.java
+++ b/src/test/java/com/basho/riak/client/api/commands/buckets/ListBucketsTest.java
@@ -21,6 +21,7 @@
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -47,21 +48,22 @@ public void init() throws Exception
         when(mockFuture.isCancelled()).thenReturn(false);
         when(mockFuture.isDone()).thenReturn(true);
         when(mockFuture.isSuccess()).thenReturn(true);
-        when(mockCluster.execute(any(PBStreamingFutureOperation.class))).thenReturn(mockFuture);
+        doReturn(mockFuture).when(mockCluster).execute(any(FutureOperation.class));
         client = new RiakClient(mockCluster);
     }
 
+    @SuppressWarnings("unchecked")
     private void testListBuckets(String bucketType) throws Exception
     {
         final BinaryValue type = BinaryValue.createFromUtf8(bucketType);
         ListBuckets.Builder list = new ListBuckets.Builder(type);
         client.execute(list.build());
 
-        ArgumentCaptor captor =
-                ArgumentCaptor.forClass(ListBucketsOperation.class);
+        ArgumentCaptor captor =
+                ArgumentCaptor.forClass(FutureOperation.class);
         verify(mockCluster).execute(captor.capture());
 
-        ListBucketsOperation operation = captor.getValue();
+        ListBucketsOperation operation = (ListBucketsOperation)captor.getValue();
         RiakKvPB.RpbListBucketsReq.Builder builder =
                 (RiakKvPB.RpbListBucketsReq.Builder) Whitebox.getInternalState(operation, "reqBuilder");
 

From 12c894483bdf64de388c66c5f14b25609412b85e Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Wed, 9 Nov 2016 07:19:45 +0200
Subject: [PATCH 44/55] Make PBStreamingFutureOperation to be a part of
 StreamableRiakCommand contract

---
 .../java/com/basho/riak/client/api/StreamableRiakCommand.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
index ab46943eb..1aa21f527 100644
--- a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
@@ -45,7 +45,7 @@ protected I convertInfo(I coreInfo) {
 
     protected abstract R createResponse(int timeout, StreamingRiakFuture coreFuture);
 
-    protected abstract  FutureOperation buildCoreOperation(boolean streamResults);
+    protected abstract  PBStreamingFutureOperation buildCoreOperation(boolean streamResults);
 
     @Override
     protected final FutureOperation buildCoreOperation() {
@@ -54,7 +54,7 @@ protected I convertInfo(I coreInfo) {
 
     protected final RiakFuture executeAsyncStreaming(RiakCluster cluster, int timeout)
     {
-        final PBStreamingFutureOperation coreOperation = (PBStreamingFutureOperation)buildCoreOperation(true);
+        final PBStreamingFutureOperation coreOperation = buildCoreOperation(true);
         final StreamingRiakFuture coreFuture = cluster.execute(coreOperation);
 
         final R r = createResponse(timeout, coreFuture);

From 944fb90b3b0169a8de83ffcce555cdfa453c276f Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Wed, 9 Nov 2016 09:31:50 +0200
Subject: [PATCH 45/55] Refactor streaming responses to re-use
 ConvertibleIterator

---
 .../api/commands/buckets/ListBuckets.java     | 41 ++++---------------
 .../commands/indexes/SecondaryIndexQuery.java | 11 ++++-
 .../riak/client/api/commands/kv/ListKeys.java | 41 ++++---------------
 .../core/query/ConvertibleIterator.java       | 35 ++++++++++++++++
 .../query/timeseries/ConvertibleIterable.java |  8 ++--
 ...tor.java => ConvertibleIteratorUtils.java} | 31 ++------------
 .../core/query/timeseries/QueryResult.java    |  3 +-
 .../client/core/query/timeseries/Row.java     |  2 +-
 8 files changed, 72 insertions(+), 100 deletions(-)
 create mode 100644 src/main/java/com/basho/riak/client/core/query/ConvertibleIterator.java
 rename src/main/java/com/basho/riak/client/core/query/timeseries/{ConvertibleIterator.java => ConvertibleIteratorUtils.java} (82%)

diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
index cc67e50c4..15f3ed898 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
@@ -20,6 +20,7 @@
 import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.ListBucketsOperation;
+import com.basho.riak.client.core.query.ConvertibleIterator;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
 
@@ -161,38 +162,14 @@ public Iterator iterator()
             }
 
             assert  buckets != null;
-            return new Itr(buckets.iterator(), type);
-        }
-    }
-
-    private static class Itr implements Iterator
-    {
-        private final Iterator iterator;
-        private final BinaryValue type;
-
-        private Itr(Iterator iterator, BinaryValue type)
-        {
-            this.iterator = iterator;
-            this.type = type;
-        }
-
-        @Override
-        public boolean hasNext()
-        {
-            return iterator.hasNext();
-        }
-
-        @Override
-        public Namespace next()
-        {
-            BinaryValue bucket = iterator.next();
-            return new Namespace(type, bucket);
-        }
-
-        @Override
-        public void remove()
-        {
-            iterator.remove();
+            return new ConvertibleIterator(buckets.iterator())
+            {
+                @Override
+                protected Namespace convert(BinaryValue bucket)
+                {
+                    return new Namespace(type, bucket);
+                }
+            };
         }
     }
 
diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
index acb0ca7ce..9a69ec195 100644
--- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
+++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java
@@ -21,6 +21,7 @@
 import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.SecondaryIndexQueryOperation;
+import com.basho.riak.client.core.query.ConvertibleIterator;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
@@ -638,8 +639,14 @@ public Iterator iterator()
                 return (Iterator)chunkedResponseIterator;
             }
 
-            // TODO: add support for not streamable responses
-            throw new UnsupportedOperationException("Iterating is only supported for streamable response.");
+            return new ConvertibleIterator(coreResponse.getEntryList().iterator())
+            {
+                @Override
+                protected E convert(SecondaryIndexQueryOperation.Response.Entry e)
+                {
+                    return createEntry(queryLocation, e, converter);
+                }
+            };
         }
 
         /**
diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
index 9e982278f..da230c8df 100644
--- a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
+++ b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java
@@ -20,6 +20,7 @@
 import com.basho.riak.client.core.FutureOperation;
 import com.basho.riak.client.core.StreamingRiakFuture;
 import com.basho.riak.client.core.operations.ListKeysOperation;
+import com.basho.riak.client.core.query.ConvertibleIterator;
 import com.basho.riak.client.core.query.Location;
 import com.basho.riak.client.core.query.Namespace;
 import com.basho.riak.client.core.util.BinaryValue;
@@ -153,38 +154,14 @@ public Iterator iterator()
             }
 
             assert keys != null;
-            return new Itr(namespace, keys.iterator());
-        }
-    }
-
-    private static class Itr implements Iterator
-    {
-        private final Iterator iterator;
-        private final Namespace namespace;
-
-        private Itr(Namespace namespace, Iterator iterator)
-        {
-            this.iterator = iterator;
-            this.namespace = namespace;
-        }
-
-        @Override
-        public boolean hasNext()
-        {
-            return iterator.hasNext();
-        }
-
-        @Override
-        public Location next()
-        {
-            BinaryValue key = iterator.next();
-            return new Location(namespace, key);
-        }
-
-        @Override
-        public void remove()
-        {
-            iterator.remove();
+            return new ConvertibleIterator(keys.iterator())
+            {
+                @Override
+                protected Location convert(BinaryValue key)
+                {
+                    return new Location(namespace, key);
+                }
+            };
         }
     }
 
diff --git a/src/main/java/com/basho/riak/client/core/query/ConvertibleIterator.java b/src/main/java/com/basho/riak/client/core/query/ConvertibleIterator.java
new file mode 100644
index 000000000..7a93c349b
--- /dev/null
+++ b/src/main/java/com/basho/riak/client/core/query/ConvertibleIterator.java
@@ -0,0 +1,35 @@
+package com.basho.riak.client.core.query;
+
+import com.basho.riak.protobuf.RiakTsPB;
+
+import java.util.Iterator;
+
+/**
+ * @author Sergey Galkin 
+ * @author Alex Moore 
+ * @since 2.0.3
+ */
+public abstract class ConvertibleIterator implements Iterator {
+    private final Iterator iterator;
+
+    public ConvertibleIterator(Iterator iterator) {
+        this.iterator = iterator;
+    }
+
+    abstract protected D convert(S source);
+
+    @Override
+    public final boolean hasNext() {
+        return iterator.hasNext();
+    }
+
+    @Override
+    public final D next() {
+        return convert(iterator.next());
+    }
+
+    @Override
+    public final void remove() {
+        throw new UnsupportedOperationException();
+    }
+}
\ No newline at end of file
diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIterable.java b/src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIterable.java
index 19bb07e42..0edd900a3 100644
--- a/src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIterable.java
+++ b/src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIterable.java
@@ -28,7 +28,7 @@ public ImmutableIterablePBCell(Iterable source)
         @Override
         public Iterator iterator()
         {
-            return ConvertibleIterator.iterateAsPbCell(this.source.iterator());
+            return ConvertibleIteratorUtils.iterateAsPbCell(this.source.iterator());
         }
     }
 
@@ -42,7 +42,7 @@ public ImmutableIterablePBRow(Iterable source)
         @Override
         public Iterator iterator()
         {
-            return ConvertibleIterator.iterateAsPbRow(this.source.iterator());
+            return ConvertibleIteratorUtils.iterateAsPbRow(this.source.iterator());
         }
     }
 
@@ -56,7 +56,7 @@ public ImmutableIterableRow(Iterable source)
         @Override
         public Iterator iterator()
         {
-            return ConvertibleIterator.iterateAsRow(this.source.iterator());
+            return ConvertibleIteratorUtils.iterateAsRow(this.source.iterator());
         }
     }
 
@@ -70,7 +70,7 @@ public ImmutableIterableCell(Iterable source)
         @Override
         public Iterator iterator()
         {
-            return ConvertibleIterator.iterateAsCell(this.source.iterator());
+            return ConvertibleIteratorUtils.iterateAsCell(this.source.iterator());
         }
     }
 
diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIterator.java b/src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIteratorUtils.java
similarity index 82%
rename from src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIterator.java
rename to src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIteratorUtils.java
index 70be94d38..df861d69b 100644
--- a/src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIterator.java
+++ b/src/main/java/com/basho/riak/client/core/query/timeseries/ConvertibleIteratorUtils.java
@@ -1,5 +1,6 @@
 package com.basho.riak.client.core.query.timeseries;
 
+import com.basho.riak.client.core.query.ConvertibleIterator;
 import com.basho.riak.protobuf.RiakTsPB;
 
 import java.util.Iterator;
@@ -9,35 +10,11 @@
  * @author Alex Moore 
  * @since 2.0.3
  */
-public abstract class ConvertibleIterator implements Iterator
+class ConvertibleIteratorUtils
 {
-    private static final RiakTsPB.TsCell NullTSCell = RiakTsPB.TsCell.newBuilder().build();
-    private final Iterator iterator;
-
-    public ConvertibleIterator(Iterator iterator)
-    {
-        this.iterator = iterator;
-    }
-
-    abstract protected D convert(S source);
-
-    @Override
-    public final boolean hasNext()
-    {
-        return iterator.hasNext();
-    }
-
-    @Override
-    public final D next()
-    {
-        return convert(iterator.next());
-    }
+    private ConvertibleIteratorUtils(){}
 
-    @Override
-    public final void remove()
-    {
-        throw new UnsupportedOperationException();
-    }
+    private static final RiakTsPB.TsCell NullTSCell = RiakTsPB.TsCell.newBuilder().build();
 
     private static class ImmutablePBCellIterator extends ConvertibleIterator
     {
diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/QueryResult.java b/src/main/java/com/basho/riak/client/core/query/timeseries/QueryResult.java
index eb5f09e96..9eea203bc 100644
--- a/src/main/java/com/basho/riak/client/core/query/timeseries/QueryResult.java
+++ b/src/main/java/com/basho/riak/client/core/query/timeseries/QueryResult.java
@@ -4,7 +4,6 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -82,7 +81,7 @@ public Iterator iterator()
         }
         else
         {
-            return ConvertibleIterator.iterateAsRow(this.pbRows.iterator());
+            return ConvertibleIteratorUtils.iterateAsRow(this.pbRows.iterator());
         }
     }
 
diff --git a/src/main/java/com/basho/riak/client/core/query/timeseries/Row.java b/src/main/java/com/basho/riak/client/core/query/timeseries/Row.java
index efd102050..340ae4aca 100644
--- a/src/main/java/com/basho/riak/client/core/query/timeseries/Row.java
+++ b/src/main/java/com/basho/riak/client/core/query/timeseries/Row.java
@@ -102,7 +102,7 @@ public Iterator iterator()
         }
         else // if (pbRow != null)
         {
-            return ConvertibleIterator.iterateAsCell(pbRow.getCellsList().iterator());
+            return ConvertibleIteratorUtils.iterateAsCell(pbRow.getCellsList().iterator());
         }
     }
 

From 67e56cb5845f03f5f72c976d11e063935eb2500a Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Thu, 10 Nov 2016 08:29:15 +0200
Subject: [PATCH 46/55] Simplify ChunkedResponseIterator initial loading

---
 .../api/commands/ChunkedResponseIterator.java | 33 ++-----------------
 1 file changed, 2 insertions(+), 31 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java
index dc4a96623..f286507d0 100644
--- a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java
+++ b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java
@@ -57,37 +57,8 @@ public ChunkedResponseIterator(StreamingRiakFuture coreFuture,
         this.getNextIterator = getNextIteratorFn;
         this.getContinuationFn = getContinuationFn;
 
-
-        // Check & clear interrupted flag so we don't get an
-        // InterruptedException every time if the user
-        // doesn't clear it / deal with it.
-        boolean interrupted = Thread.interrupted();
-        try
-        {
-            boolean lastLoadInterrupted;
-            do
-            {
-                lastLoadInterrupted = false;
-                try
-                {
-                    tryLoadNextChunkIterator();
-                }
-                catch (InterruptedException ex)
-                {
-                    interrupted = true;
-                    lastLoadInterrupted = true;
-                }
-            } while (lastLoadInterrupted);
-        }
-        finally
-        {
-            if (interrupted)
-            {
-                // Reset interrupted flag if we came in with it
-                // or we were interrupted while waiting.
-                Thread.currentThread().interrupt();
-            }
-        }
+        // to kick of initial loading
+        hasNext();
     }
 
     @Override

From 22655d4319bafdc4ce7a0e0ca230650e179f67ea Mon Sep 17 00:00:00 2001
From: Sergey Galkin 
Date: Thu, 10 Nov 2016 12:16:06 +0200
Subject: [PATCH 47/55] Generalize StreamableResponse

---
 .../com/basho/riak/client/api/RiakClient.java |  2 +-
 .../client/api/StreamableRiakCommand.java     | 42 ++++++++++++++++++-
 .../api/commands/buckets/ListBuckets.java     | 20 +++------
 .../commands/indexes/SecondaryIndexQuery.java | 18 +++-----
 .../riak/client/api/commands/kv/ListKeys.java | 20 +++------
 .../api/commands/mapreduce/MapReduce.java     |  3 +-
 6 files changed, 58 insertions(+), 47 deletions(-)

diff --git a/src/main/java/com/basho/riak/client/api/RiakClient.java b/src/main/java/com/basho/riak/client/api/RiakClient.java
index 82ad3d9d3..84fe79e11 100644
--- a/src/main/java/com/basho/riak/client/api/RiakClient.java
+++ b/src/main/java/com/basho/riak/client/api/RiakClient.java
@@ -435,7 +435,7 @@ public  RiakFuture executeAsync(RiakCommand command)
      * @return a RiakFuture for the operation
      * @see RiakFuture
      */
-    public  RiakFuture executeAsyncStreaming(StreamableRiakCommand command, int timeoutMS)
+    public  RiakFuture executeAsyncStreaming(StreamableRiakCommand command, int timeoutMS)
     {
         return command.executeAsyncStreaming(cluster, timeoutMS);
     }
diff --git a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
index 1aa21f527..ba6bf058f 100644
--- a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
+++ b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java
@@ -16,9 +16,12 @@
 
 package com.basho.riak.client.api;
 
+import com.basho.riak.client.api.commands.ChunkedResponseIterator;
 import com.basho.riak.client.api.commands.ImmediateCoreFutureAdapter;
 import com.basho.riak.client.core.*;
 
+import java.util.Iterator;
+
 /*
  * The base class for all Streamable Riak Commands.
  * Allows the user to either use {@link RiakCommand#executeAsync} and return a "batch-mode" result
@@ -33,9 +36,9 @@
  * @author Sergey Galkin 
  * @since 2.0
  */
-public abstract class StreamableRiakCommand extends GenericRiakCommand
+public abstract class StreamableRiakCommand extends GenericRiakCommand
 {
-    public static abstract class StreamableRiakCommandWithSameInfo extends StreamableRiakCommand
+    public static abstract class StreamableRiakCommandWithSameInfo extends StreamableRiakCommand
     {
         @Override
         protected I convertInfo(I coreInfo) {
@@ -43,6 +46,41 @@ protected I convertInfo(I coreInfo) {
         }
     }
 
+    public static abstract class StreamableResponse implements Iterable
+    {
+        protected ChunkedResponseIterator chunkedResponseIterator;
+
+        /**
+         * Constructor for streamable response
+         * @param chunkedResponseIterator
+         */
+        protected StreamableResponse(ChunkedResponseIterator chunkedResponseIterator) {
+            this.chunkedResponseIterator = chunkedResponseIterator;
+        }
+
+        /**
+         * Constructor for not streamable response.
+         */
+        protected StreamableResponse()
+        {
+        }
+
+        public boolean isStreamable()
+        {
+            return chunkedResponseIterator != null;
+        }
+
+        @Override
+        public Iterator iterator() {
+            if (isStreamable()) {
+                assert chunkedResponseIterator != null;
+                return chunkedResponseIterator;
+            }
+
+            throw new UnsupportedOperationException("Iterating is only supported for streamable response");
+        }
+    }
+
     protected abstract R createResponse(int timeout, StreamingRiakFuture coreFuture);
 
     protected abstract  PBStreamingFutureOperation buildCoreOperation(boolean streamResults);
diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
index 15f3ed898..986f65237 100644
--- a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
+++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java
@@ -120,10 +120,8 @@ protected ListBucketsOperation buildCoreOperation(boolean streamResults)
      *
*

*/ - public static class Response implements Iterable + public static class Response extends StreamableRiakCommand.StreamableResponse { - private final ChunkedResponseIterator - chunkedResponseIterator; private final BinaryValue type; private final List buckets; @@ -131,10 +129,10 @@ public static class Response implements Iterable int pollTimeout, StreamingRiakFuture coreFuture) { - this.chunkedResponseIterator = new ChunkedResponseIterator<>(coreFuture, + super(new ChunkedResponseIterator<>(coreFuture, pollTimeout, (bucketName) -> new Namespace(type, bucketName), - (response) -> response.getBuckets().iterator()); + (response) -> response.getBuckets().iterator())); this.type = type; this.buckets = null; @@ -142,23 +140,15 @@ public static class Response implements Iterable public Response(BinaryValue type, List buckets) { - this.chunkedResponseIterator = null; this.type = type; this.buckets = buckets; } - public boolean isStreamable() - { - return chunkedResponseIterator != null; - } - @Override public Iterator iterator() { - if (isStreamable()) - { - assert chunkedResponseIterator != null; - return chunkedResponseIterator; + if (isStreamable()) { + return super.iterator(); } assert buckets != null; diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java index 9a69ec195..a15482a27 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java @@ -588,12 +588,12 @@ protected T withReturnBody(boolean returnBody) * * @param The type contained in the resposne. */ - public static class Response> implements Iterable + public static class Response> + extends StreamableResponse { final protected IndexConverter converter; final protected SecondaryIndexQueryOperation.Response coreResponse; final protected Namespace queryLocation; - private final ChunkedResponseIterator chunkedResponseIterator; protected Response(final Namespace queryLocation, IndexConverter converter, final int timeout, final StreamingRiakFuture coreFuture) @@ -601,14 +601,14 @@ protected Response(final Namespace queryLocation, IndexConverter converter, f this.queryLocation = queryLocation; this.converter = converter; this.coreResponse = null; - chunkedResponseIterator = new ChunkedResponseIterator( + chunkedResponseIterator = new ChunkedResponseIterator( coreFuture, timeout, null, SecondaryIndexQueryOperation.Response::iterator, SecondaryIndexQueryOperation.Response::getContinuation) { @SuppressWarnings("unchecked") @Override - public Response.Entry next() + public E next() { final SecondaryIndexQueryOperation.Response.Entry coreEntity = currentIterator.next(); return Response.this.createEntry(Response.this.queryLocation, coreEntity, converter); @@ -623,20 +623,12 @@ protected Response(Namespace queryLocation, this.queryLocation = queryLocation; this.converter = converter; this.coreResponse = coreResponse; - chunkedResponseIterator = null; } - public boolean isStreamable() - { - return chunkedResponseIterator != null; - } - - @SuppressWarnings("unchecked") public Iterator iterator() { if (isStreamable()) { - assert chunkedResponseIterator != null; - return (Iterator)chunkedResponseIterator; + return super.iterator(); } return new ConvertibleIterator(coreResponse.getEntryList().iterator()) diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java index da230c8df..496514984 100644 --- a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java +++ b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java @@ -112,10 +112,8 @@ protected ListKeysOperation buildCoreOperation(boolean streamResults) return builder.build(); } - public static class Response implements Iterable + public static class Response extends StreamableRiakCommand.StreamableResponse { - private final ChunkedResponseIterator - chunkedResponseIterator; private final Namespace namespace; private final List keys; @@ -123,34 +121,26 @@ public Response(Namespace namespace, List keys) { this.namespace = namespace; this.keys = keys; - this.chunkedResponseIterator = null; } Response(Namespace namespace, int pollTimeout, StreamingRiakFuture coreFuture) { - chunkedResponseIterator = new ChunkedResponseIterator<>(coreFuture, + super(new ChunkedResponseIterator<>(coreFuture, pollTimeout, (key) -> new Location(namespace, key), - (nextChunk) -> nextChunk.getKeys().iterator()); + (nextChunk) -> nextChunk.getKeys().iterator())); this.namespace = namespace; this.keys = null; } - public boolean isStreamable() - { - return chunkedResponseIterator != null; - } - @Override public Iterator iterator() { - if (isStreamable()) - { - assert chunkedResponseIterator != null; - return chunkedResponseIterator; + if (isStreamable()) { + return super.iterator(); } assert keys != null; diff --git a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java index a999b214a..f836afcfc 100644 --- a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java +++ b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java @@ -336,7 +336,7 @@ public T withLinkPhase(String bucket, String tag) /** * Response from a MapReduce command. */ - public static class Response implements Iterable + public static class Response extends StreamableRiakCommand.StreamableResponse { private final Map results; private final MapReduceResponseIterator responseIterator; @@ -354,6 +354,7 @@ public Response(Map results) responseIterator = null; } + @Override public boolean isStreamable() { return responseIterator != null; From 3afa145867dcf5d711855dc88b1a06192e6d02c1 Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Thu, 10 Nov 2016 19:12:27 +0200 Subject: [PATCH 48/55] fix iteration through chunked response --- .../api/commands/ChunkedResponseIterator.java | 8 +- .../core/ChunkedResponseIteratorTest.java | 113 +++++++++++------- 2 files changed, 77 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java index f286507d0..182bf7b3f 100644 --- a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java +++ b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java @@ -20,6 +20,7 @@ import com.basho.riak.client.core.util.BinaryValue; import java.util.Iterator; +import java.util.NoSuchElementException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TransferQueue; import java.util.function.Function; @@ -121,7 +122,12 @@ public BinaryValue getContinuation() @Override public FinalT next() { - return createNext.apply(currentIterator.next()); + if (hasNext()) + { + return createNext.apply(currentIterator.next()); + } + + throw new NoSuchElementException(); } private boolean tryLoadNextChunkIterator() throws InterruptedException diff --git a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java index 5b73bd865..bce865b6b 100644 --- a/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java +++ b/src/test/java/com/basho/riak/client/core/ChunkedResponseIteratorTest.java @@ -2,22 +2,24 @@ import com.basho.riak.client.api.commands.ChunkedResponseIterator; +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; -import org.mockito.invocation.InvocationOnMock; +import org.mockito.Matchers; +import org.mockito.Mock; import org.mockito.stubbing.Answer; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.TransferQueue; import static org.junit.Assert.*; -import static org.powermock.api.mockito.PowerMockito.mock; import static org.powermock.api.mockito.PowerMockito.when; +import static org.mockito.Matchers.any; /** * @author Alex Moore @@ -26,6 +28,23 @@ @PrepareForTest(PBStreamingFutureOperation.class) public class ChunkedResponseIteratorTest { + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Mock + private TransferQueue fakeQueue; + + @Mock + private PBStreamingFutureOperation coreFuture; + + private final int timeout = 1000; + + @Before + public void initializeMOcks() + { + when(coreFuture.getResultsQueue()).thenReturn(fakeQueue); + } + /* NB: Disable this test if you want to use code coverage tools. The Thread interrupt it generates triggers a shutdown hook race bug in Java, @@ -35,17 +54,11 @@ public class ChunkedResponseIteratorTest @Test public void testInterruptedExceptionUponInitialisation() throws InterruptedException { - int timeout = 1000; - Thread testThread = new Thread(() -> { try { - @SuppressWarnings("unchecked") - TransferQueue fakeQueue = - (TransferQueue) mock(TransferQueue.class); - - when(fakeQueue.poll(timeout, TimeUnit.MILLISECONDS)) + when(fakeQueue.poll(any(Long.class), any(TimeUnit.class))) .thenThrow(new InterruptedException()) .thenReturn( new FakeResponse() { @@ -56,12 +69,6 @@ public Iterator iterator() } }); - @SuppressWarnings("unchecked") - PBStreamingFutureOperation coreFuture = - (PBStreamingFutureOperation) mock(PBStreamingFutureOperation.class); - - when(coreFuture.getResultsQueue()).thenReturn(fakeQueue); - // ChunkedResponseIterator polls the response queue when created, // so we'll use that to simulate a Thread interrupt. new ChunkedResponseIterator<>(coreFuture, @@ -87,17 +94,11 @@ public Iterator iterator() @Test public void testInterruptedExceptionUponNextChunkLoad() throws InterruptedException { - int timeout = 1000; - Thread testThread = new Thread(() -> { try { - @SuppressWarnings("unchecked") - TransferQueue fakeQueue = - (TransferQueue) mock(TransferQueue.class); - - when(fakeQueue.poll(timeout, TimeUnit.MILLISECONDS)) + when(fakeQueue.poll(any(Long.class), any(TimeUnit.class))) .thenReturn( new FakeResponse() { @Override @@ -116,12 +117,6 @@ public Iterator iterator() } }); - @SuppressWarnings("unchecked") - PBStreamingFutureOperation coreFuture = - (PBStreamingFutureOperation) mock(PBStreamingFutureOperation.class); - - when(coreFuture.getResultsQueue()).thenReturn(fakeQueue); - ChunkedResponseIterator chunkedResponseIterator = new ChunkedResponseIterator<>(coreFuture, timeout, Long::new, FakeResponse::iterator); @@ -149,22 +144,13 @@ public Iterator iterator() @Test public void testConcurrentDoneAndInterruptedException() throws InterruptedException { - int timeout = 1000; - Thread testThread = new Thread(() -> { try { - @SuppressWarnings("unchecked") - PBStreamingFutureOperation coreFuture = - (PBStreamingFutureOperation) mock(PBStreamingFutureOperation.class); when(coreFuture.isDone()).thenReturn(false); - @SuppressWarnings("unchecked") - TransferQueue fakeQueue = - (TransferQueue) mock(TransferQueue.class); - - when(fakeQueue.poll(timeout, TimeUnit.MILLISECONDS)) + when(fakeQueue.poll(any(Long.class), any(TimeUnit.class))) .thenReturn( new FakeResponse() { @Override @@ -180,8 +166,6 @@ public Iterator iterator() throw new InterruptedException(); }); - when(coreFuture.getResultsQueue()).thenReturn(fakeQueue); - ChunkedResponseIterator chunkedResponseIterator = new ChunkedResponseIterator<>(coreFuture, timeout, Long::new, FakeResponse::iterator); @@ -207,5 +191,48 @@ public Iterator iterator() assertFalse(Thread.currentThread().isInterrupted()); } + @Test(timeout = 5000) + public void checkProperIterationThroughChunkedResponse() throws InterruptedException { + when(fakeQueue.poll(any(Long.class), any(TimeUnit.class))) + // Simulate first chunk + .thenReturn( + new FakeResponse() { + @Override + public Iterator iterator() + { + return Arrays.asList(1,2).iterator(); + } + }) + // Simulate next chunk + .thenReturn( + new FakeResponse() { + @Override + public Iterator iterator() + { + return Arrays.asList(3,4).iterator(); + } + }) + // Simulate completion + .thenAnswer(invocationOnMock -> + { + when(coreFuture.isDone()).thenReturn(true); + when(fakeQueue.isEmpty()).thenReturn(true); + return null; + }); + + final ChunkedResponseIterator iterator = + new ChunkedResponseIterator<>(coreFuture, 50, Long::new, FakeResponse::iterator); + + assertEquals(1l, iterator.next().longValue()); + assertEquals(2l, iterator.next().longValue()); + assertEquals(3l, iterator.next().longValue()); + assertEquals(4l, iterator.next().longValue()); + + assertFalse(iterator.hasNext()); + + exception.expect(NoSuchElementException.class); + iterator.next(); + } + static abstract class FakeResponse implements Iterable {} } From e3139561029f9c3834bf0fc0a27f5b61b26bfd44 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Fri, 11 Nov 2016 15:05:08 -0500 Subject: [PATCH 49/55] Adding assert around chunk queue insertion --- .../com/basho/riak/client/core/PBStreamingFutureOperation.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/basho/riak/client/core/PBStreamingFutureOperation.java b/src/main/java/com/basho/riak/client/core/PBStreamingFutureOperation.java index d0484790c..c8f9e7602 100644 --- a/src/main/java/com/basho/riak/client/core/PBStreamingFutureOperation.java +++ b/src/main/java/com/basho/riak/client/core/PBStreamingFutureOperation.java @@ -59,7 +59,8 @@ protected void processMessage(ResponseType decodedMessage) final ReturnType r = processStreamingChunk(decodedMessage); assert this.responseQueue != null; - responseQueue.offer(r); + final boolean chunkAdded = responseQueue.offer(r); + assert chunkAdded; } abstract protected ReturnType processStreamingChunk(ResponseType rawResponseChunk); From 51bff206320ac5c597fe6cecb3e89f55ccd63f35 Mon Sep 17 00:00:00 2001 From: Sergey Galkin Date: Tue, 15 Nov 2016 14:54:41 +0200 Subject: [PATCH 50/55] Adding integration tests for streamable full bucket read. --- .../indexes/itest/ITestFullBucketRead.java | 112 +++++++++++++----- 1 file changed, 81 insertions(+), 31 deletions(-) diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestFullBucketRead.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestFullBucketRead.java index 792ae119a..84adfc4ad 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestFullBucketRead.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestFullBucketRead.java @@ -145,38 +145,41 @@ public void readPlainTextValuesWithReturnBody() public void readPlainTextValuesWithCoverageContext() throws ExecutionException, InterruptedException, UnknownHostException { - final Map results = performFBReadWithCoverageContext(false, false); + final Map results = performFBReadWithCoverageContext(false, false, false); assertEquals(NUMBER_OF_TEST_VALUES, results.size()); - for (int i=0; i results = performFBReadWithCoverageContext(false, false, true); + assertEquals(NUMBER_OF_TEST_VALUES, results.size()); + + verifyResults(results, false); } @Test public void readPlainTextValuesWithCoverageContextContinuouslyWithReturnBody() throws ExecutionException, InterruptedException, UnknownHostException { - final Map results = performFBReadWithCoverageContext(true, true); + final Map results = performFBReadWithCoverageContext(true, true, false); assertEquals(NUMBER_OF_TEST_VALUES, results.size()); - for (int i=0; i results = performFBReadWithCoverageContext(true, true, true); + assertEquals(NUMBER_OF_TEST_VALUES, results.size()); + + verifyResults(results, true); } @Test @@ -216,7 +219,31 @@ public void queryDataByUsingAnAlternateCoveragePlan() } } - private Map performFBReadWithCoverageContext(boolean withContinuation,boolean withReturnBody) + private static void verifyResults(final Map results, boolean withReturnBody ) + { + for (int i=0; i performFBReadWithCoverageContext(boolean withContinuation, boolean withReturnBody, + boolean useStreaming) throws UnknownHostException, ExecutionException, InterruptedException { final Map> chunkedKeys = new HashMap<>(); @@ -240,7 +267,7 @@ private Map performFBReadWithCoverageContext(boolean withCon for (CoverageEntry ce : coveragePlan.hostEntries(host)) { final Map> keys = - retrieveChunkedKeysForCoverageEntry(rc, ce, withContinuation, withReturnBody); + retrieveChunkedKeysForCoverageEntry(rc, ce, withContinuation, withReturnBody, useStreaming); chunkedKeys.putAll(keys); } } @@ -272,7 +299,7 @@ private Map readDataForCoverageEntry(CoverageEntry ce) Map> keys = Collections.emptyMap(); try { - keys = retrieveChunkedKeysForCoverageEntry(rc, ce, true, true); + keys = retrieveChunkedKeysForCoverageEntry(rc, ce, true, true, false); } finally { @@ -286,9 +313,11 @@ private Map readDataForCoverageEntry(CoverageEntry ce) private Map> retrieveChunkedKeysForCoverageEntry(RiakClient rc, CoverageEntry ce, boolean withContinuation, - boolean withReturnBody) + boolean withReturnBody, + boolean useStreaming) throws ExecutionException, InterruptedException { + final int pollTimeout = 100; // <-- in ms in case when useStreaming == true final Map> chunkedKeys = new HashMap<>(); if (!withContinuation) { @@ -296,8 +325,21 @@ private Map> retrieveChunkedKeysForCoverageEntry(Riak .withReturnBody(withReturnBody) .build(); - final FullBucketRead.Response readResponse = rc.execute(cmd2); - chunkedKeys.put(ce, readResponse.getEntries()); + final FullBucketRead.Response readResponse; + if (!useStreaming) + { + readResponse = rc.execute(cmd2); + } + else + { + readResponse = rc.executeAsyncStreaming(cmd2, pollTimeout).get(); + } + + assertEquals(useStreaming, readResponse.isStreamable()); + final List list = new LinkedList<>(); + readResponse.forEach( e -> list.add(e)); + + chunkedKeys.put(ce, list); } else { @@ -313,17 +355,25 @@ private Map> retrieveChunkedKeysForCoverageEntry(Riak .withContinuation(continuation) .build(); - final FullBucketRead.Response r = rc.execute(cmd2); - final List entries; + final FullBucketRead.Response r; - if (r.hasEntries()) + if (!useStreaming) { - entries = r.getEntries(); - data.addAll(entries); + r = rc.execute(cmd2); } else { - entries = Collections.EMPTY_LIST; + r = rc.executeAsyncStreaming(cmd2, pollTimeout).get(); + } + + assertEquals(useStreaming, r.isStreamable()); + + final List entries = new LinkedList<>(); + + if (r.hasEntries()) + { + r.forEach(e-> entries.add(e)); + data.addAll(entries); } logger.debug("FullBucketRead query(ce={}, token={}) returns:\n token: {}\n entries: {}", From 002ba0e8bf1869c72307d64f2b8626d6e24aef33 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 15 Nov 2016 14:21:32 -0500 Subject: [PATCH 51/55] Fix bug where calling 2i getEntries() while streaming would throw NullPointerException. --- .../commands/indexes/SecondaryIndexQuery.java | 41 +++++++++++++++---- .../indexes/itest/ITestBigIntIndexQuery.java | 1 + .../indexes/itest/ITestBinIndexQuery.java | 3 ++ .../indexes/itest/ITestIntIndexQuery.java | 1 + .../indexes/itest/ITestRawIndexQuery.java | 1 + 5 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java index a15482a27..699f79a34 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java @@ -625,6 +625,16 @@ protected Response(Namespace queryLocation, this.coreResponse = coreResponse; } + /** + * Get an iterator over the result data. + * + * If using the streaming API, this method will block + * and wait for more data if none is immediately available. + * It is also advisable to check {@link Thread#isInterrupted()} + * in environments where thread interrupts must be obeyed. + * + * @return an iterator over the result data. + */ public Iterator iterator() { if (isStreamable()) { @@ -644,6 +654,10 @@ protected E convert(SecondaryIndexQueryOperation.Response.Entry e) /** * Check if this response has a continuation. * + * If using the streaming API, this property's value + * may change while data is being received, therefore + * it is best to call it after the operation is complete. + * * @return true if the response contains a continuation. */ public boolean hasContinuation() @@ -659,6 +673,10 @@ public boolean hasContinuation() /** * Get the continuation from this response. * + * If using the streaming API, this property's value + * may change while data is being received, therefore + * it is best to call it after the operation is complete. + * * @return the continuation, or null if none is present. */ public BinaryValue getContinuation() @@ -674,6 +692,11 @@ public BinaryValue getContinuation() /** * Check is this response contains any entries. * + * If using the streaming API, this method will block + * and wait for more data if none is immediately available. + * It is also advisable to check {@link Thread#isInterrupted()} + * in environments where thread interrupts must be obeyed. + * * @return true if entries are present, false otherwise. */ public boolean hasEntries() @@ -686,8 +709,19 @@ public boolean hasEntries() return !coreResponse.getEntryList().isEmpty(); } + /** + * Get a list of the result entries for this response. + * If using the streaming API this method will return an empty list. + * + * @return A list of result entries. + */ public final List getEntries() { + if(isStreamable()) + { + return new ArrayList<>(0); + } + final List coreEntries = coreResponse.getEntryList(); final List convertedList = new ArrayList<>(coreEntries.size()); @@ -699,13 +733,6 @@ public final List getEntries() return convertedList; } - /** - * Factory method. - * @param location - * @param coreEntry - * @param converter - * @return - */ @SuppressWarnings("unchecked") protected E createEntry(Location location, SecondaryIndexQueryOperation.Response.Entry coreEntry, IndexConverter converter) { diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java index 3f311253d..0cff732b4 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java @@ -107,6 +107,7 @@ public void testStreamingRangeQuery() throws ExecutionException, InterruptedExce final BigIntIndexQuery.Response streamingResponse = streamingFuture.get(); assertTrue(streamingResponse.hasEntries()); + assertTrue(streamingResponse.getEntries().isEmpty()); final String expectedObjectKey = objectKey(1); boolean found = false; diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java index 7ad99e301..07cecc5e5 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java @@ -21,6 +21,7 @@ import com.basho.riak.client.api.annotations.RiakIndex; import com.basho.riak.client.api.annotations.RiakKey; import com.basho.riak.client.api.commands.indexes.BinIndexQuery; +import com.basho.riak.client.api.commands.indexes.SecondaryIndexQuery; import com.basho.riak.client.api.commands.kv.StoreValue; import com.basho.riak.client.core.RiakFuture; import com.basho.riak.client.core.query.Location; @@ -28,6 +29,7 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.util.List; import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertEquals; @@ -105,6 +107,7 @@ public void testStreamingRangeQuery() throws ExecutionException, InterruptedExce final BinIndexQuery.Response streamingResponse = streamingFuture.get(); assertTrue(streamingResponse.hasEntries()); + assertTrue(streamingResponse.getEntries().isEmpty()); final String expectedObjectKey = objectKey(1); final String expectedIndexKey = indexKey(1); diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java index a33c6e9b8..2008a1957 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java @@ -113,6 +113,7 @@ public void testStreamingRangeQuery() throws ExecutionException, InterruptedExce final IntIndexQuery.Response streamingResponse = streamingFuture.get(); assertTrue(streamingResponse.hasEntries()); + assertTrue(streamingResponse.getEntries().isEmpty()); final String expectedObjectKey = objectKey(1); boolean found = false; diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java index 723972f8a..6ad18151a 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java @@ -182,6 +182,7 @@ public void testBucketIndexQueryStreaming() throws InterruptedException, Executi final BinIndexQuery.Response streamingResponse = indexResult.get(); assertTrue(streamingResponse.hasEntries()); + assertTrue(streamingResponse.getEntries().isEmpty()); assertEquals(100, StreamSupport.stream(streamingResponse.spliterator(), false).count()); // Assert everything was consumed From a13798bced142d79535ab97c2a0820720487d4b6 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 15 Nov 2016 14:22:25 -0500 Subject: [PATCH 52/55] Pedantic whitespace formatting. --- src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java | 1 - .../java/com/basho/riak/client/api/GenericRiakCommand.java | 2 -- .../basho/riak/client/api/commands/buckets/ListBuckets.java | 1 - .../basho/riak/client/api/commands/indexes/IntIndexQuery.java | 1 - .../java/com/basho/riak/client/api/commands/kv/ListKeys.java | 3 ++- .../basho/riak/client/api/commands/mapreduce/MapReduce.java | 2 +- .../basho/riak/client/core/operations/ListKeysOperation.java | 1 - .../basho/riak/client/core/operations/MapReduceOperation.java | 1 - .../client/core/operations/SecondaryIndexQueryOperation.java | 1 - .../client/api/commands/buckets/itest/ITestListBuckets.java | 2 +- .../basho/riak/client/core/ChunkedResponseIteratorTest.java | 1 - .../core/operations/itest/ITestListBucketsOperation.java | 2 -- .../client/core/operations/itest/ITestListKeysOperation.java | 3 --- .../client/core/operations/itest/ITestMapReduceOperation.java | 1 - 14 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java b/src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java index 17fb0a13f..b21dd82c5 100644 --- a/src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java +++ b/src/main/java/com/basho/riak/client/api/AsIsRiakCommand.java @@ -34,5 +34,4 @@ protected RiakFuture executeAsync(RiakCluster cluster) return cluster.execute(coreOperation); } - } diff --git a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java index c6f25b545..18cc53cbd 100644 --- a/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java +++ b/src/main/java/com/basho/riak/client/api/GenericRiakCommand.java @@ -17,11 +17,9 @@ import com.basho.riak.client.api.commands.CoreFutureAdapter; import com.basho.riak.client.core.FutureOperation; -import com.basho.riak.client.core.PBStreamingFutureOperation; import com.basho.riak.client.core.RiakCluster; import com.basho.riak.client.core.RiakFuture; - /** * @author Sergey Galkin * @since 2.1.0 diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java index 986f65237..3e9e7ffd3 100644 --- a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java +++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java @@ -213,5 +213,4 @@ public ListBuckets build() return new ListBuckets(this); } } - } diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java index 5ae04345f..1b6eb9cee 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/IntIndexQuery.java @@ -21,7 +21,6 @@ import com.basho.riak.client.core.query.Namespace; import com.basho.riak.client.core.util.BinaryValue; - /** * Performs a 2i query where the 2i index keys are numeric. * diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java index 496514984..3e86929b6 100644 --- a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java +++ b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java @@ -139,7 +139,8 @@ public Response(Namespace namespace, List keys) @Override public Iterator iterator() { - if (isStreamable()) { + if (isStreamable()) + { return super.iterator(); } diff --git a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java index f836afcfc..a61708d85 100644 --- a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java +++ b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java @@ -457,7 +457,7 @@ public boolean hasNext() } finally { - if(interrupted) + if (interrupted) { // Reset interrupted flag if we came in with it // or we were interrupted while waiting. diff --git a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java index edbd12cbc..9a0619b68 100644 --- a/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java +++ b/src/main/java/com/basho/riak/client/core/operations/ListKeysOperation.java @@ -30,7 +30,6 @@ public class ListKeysOperation extends PBStreamingFutureOperation expectedBuckets = storeObjects(bucketType, bucketCount); @@ -210,5 +209,4 @@ private List storeObjects(String bucketType, int bucketCount) throw return bucketNames; } - } diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java index b81bab2d4..f13f1b6d4 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestListKeysOperation.java @@ -91,7 +91,6 @@ public void testListKeyTestType() throws InterruptedException, ExecutionExceptio testListSingleKey(namedBucketType); } - @Test public void testListKeyTestTypeStreaming() throws InterruptedException, ExecutionException { @@ -125,8 +124,6 @@ public void testLargeKeyListTestTypeStreaming() throws InterruptedException, Exe testListKeysStreaming(namedBucketType, 1000); } - - private void testListNoKeys(String bucketType) throws InterruptedException, ExecutionException { final Namespace ns = setupBucket(bucketType, 0); diff --git a/src/test/java/com/basho/riak/client/core/operations/itest/ITestMapReduceOperation.java b/src/test/java/com/basho/riak/client/core/operations/itest/ITestMapReduceOperation.java index 0abdfcbba..8336ae68b 100644 --- a/src/test/java/com/basho/riak/client/core/operations/itest/ITestMapReduceOperation.java +++ b/src/test/java/com/basho/riak/client/core/operations/itest/ITestMapReduceOperation.java @@ -135,7 +135,6 @@ private Map> testBasicStreamingMR(Namespace names final TransferQueue resultsQueue = streamFuture.getResultsQueue(); - Map> resultMap = new LinkedHashMap<>(); while (!streamFuture.isDone()) From ac3d7e202810d3cf747ca439065597674255c041 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 15 Nov 2016 14:37:35 -0500 Subject: [PATCH 53/55] Switch 2i getEntries() to throw IllegalStateException. --- .../api/commands/indexes/SecondaryIndexQuery.java | 5 +++-- .../indexes/itest/ITestBigIntIndexQuery.java | 12 +++++++++++- .../commands/indexes/itest/ITestBinIndexQuery.java | 12 +++++++++++- .../commands/indexes/itest/ITestIntIndexQuery.java | 12 +++++++++++- .../commands/indexes/itest/ITestRawIndexQuery.java | 13 ++++++++++++- 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java index 699f79a34..f6a8075d6 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java @@ -27,6 +27,7 @@ import com.basho.riak.client.core.util.BinaryValue; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -711,15 +712,15 @@ public boolean hasEntries() /** * Get a list of the result entries for this response. - * If using the streaming API this method will return an empty list. * * @return A list of result entries. + * @throws IllegalStateException when called while using the streaming API. */ public final List getEntries() { if(isStreamable()) { - return new ArrayList<>(0); + throw new IllegalStateException("Use the iterator() while using the streaming API"); } final List coreEntries = coreResponse.getEntryList(); diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java index 0cff732b4..672d8ce34 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBigIntIndexQuery.java @@ -107,7 +107,17 @@ public void testStreamingRangeQuery() throws ExecutionException, InterruptedExce final BigIntIndexQuery.Response streamingResponse = streamingFuture.get(); assertTrue(streamingResponse.hasEntries()); - assertTrue(streamingResponse.getEntries().isEmpty()); + + boolean caught = false; + try + { + streamingResponse.getEntries(); + } + catch (IllegalStateException unused) + { + caught = true; + } + assertTrue(caught); final String expectedObjectKey = objectKey(1); boolean found = false; diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java index 07cecc5e5..c92ac3d81 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestBinIndexQuery.java @@ -107,7 +107,17 @@ public void testStreamingRangeQuery() throws ExecutionException, InterruptedExce final BinIndexQuery.Response streamingResponse = streamingFuture.get(); assertTrue(streamingResponse.hasEntries()); - assertTrue(streamingResponse.getEntries().isEmpty()); + + boolean caught = false; + try + { + streamingResponse.getEntries(); + } + catch (IllegalStateException unused) + { + caught = true; + } + assertTrue(caught); final String expectedObjectKey = objectKey(1); final String expectedIndexKey = indexKey(1); diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java index 2008a1957..a5a68093b 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestIntIndexQuery.java @@ -113,7 +113,17 @@ public void testStreamingRangeQuery() throws ExecutionException, InterruptedExce final IntIndexQuery.Response streamingResponse = streamingFuture.get(); assertTrue(streamingResponse.hasEntries()); - assertTrue(streamingResponse.getEntries().isEmpty()); + + boolean caught = false; + try + { + streamingResponse.getEntries(); + } + catch (IllegalStateException unused) + { + caught = true; + } + assertTrue(caught); final String expectedObjectKey = objectKey(1); boolean found = false; diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java index 6ad18151a..3fa722604 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestRawIndexQuery.java @@ -182,7 +182,18 @@ public void testBucketIndexQueryStreaming() throws InterruptedException, Executi final BinIndexQuery.Response streamingResponse = indexResult.get(); assertTrue(streamingResponse.hasEntries()); - assertTrue(streamingResponse.getEntries().isEmpty()); + + boolean caught = false; + try + { + streamingResponse.getEntries(); + } + catch (IllegalStateException unused) + { + caught = true; + } + assertTrue(caught); + assertEquals(100, StreamSupport.stream(streamingResponse.spliterator(), false).count()); // Assert everything was consumed From 78c43f1547b70b377a42f744f9b789581328b856 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 15 Nov 2016 15:06:43 -0500 Subject: [PATCH 54/55] Adding javadocs to new public API pieces --- .../com/basho/riak/client/api/RiakClient.java | 2 + .../client/api/StreamableRiakCommand.java | 19 ++++++- .../api/commands/ChunkedResponseIterator.java | 52 +++++++++++++++++++ .../api/commands/mapreduce/MapReduce.java | 24 +++++++++ 4 files changed, 95 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/RiakClient.java b/src/main/java/com/basho/riak/client/api/RiakClient.java index 84fe79e11..2146beef8 100644 --- a/src/main/java/com/basho/riak/client/api/RiakClient.java +++ b/src/main/java/com/basho/riak/client/api/RiakClient.java @@ -155,6 +155,7 @@ * * @author Dave Rusek * @author Brian Roach + * @author Alex Moore * @author Sergey Galkin * @since 2.0 */ @@ -433,6 +434,7 @@ public RiakFuture executeAsync(RiakCommand command) * the result iterator can block the consuming thread from seeing the done() * status until the timeout is reached. * @return a RiakFuture for the operation + * @since 2.1.0 * @see RiakFuture */ public RiakFuture executeAsyncStreaming(StreamableRiakCommand command, int timeoutMS) diff --git a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java index ba6bf058f..b24c6b2f4 100644 --- a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java +++ b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java @@ -31,8 +31,7 @@ * @param The response type returned by "streaming mode" {@link executeAsyncStreaming} * @param The response type returned by the "batch mode" @{link executeAsync} * @param The query info type - * @author Dave Rusek - * @author Brian Roach + * @author Alex Moore * @author Sergey Galkin * @since 2.0 */ @@ -65,11 +64,27 @@ protected StreamableResponse() { } + /** + * Whether the results are to be streamed back. + * If true, results will appear in this class's iterator. + * If false, they will appear in the original result collection. + * @return true if the results are to be streamed. + */ public boolean isStreamable() { return chunkedResponseIterator != null; } + /** + * Get an iterator over the result data. + * + * If using the streaming API, this method will block + * and wait for more data if none is immediately available. + * It is also advisable to check {@link Thread#isInterrupted()} + * in environments where thread interrupts must be obeyed. + * + * @return an iterator over the result data. + */ @Override public Iterator iterator() { if (isStreamable()) { diff --git a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java index 182bf7b3f..59f20a4fc 100644 --- a/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java +++ b/src/main/java/com/basho/riak/client/api/commands/ChunkedResponseIterator.java @@ -25,6 +25,24 @@ import java.util.concurrent.TransferQueue; import java.util.function.Function; +/** + * Transforms a stream of response chunks to a Iterable of response items. + * + * When iterating over this class's {@link Iterator} this class will lazily walk + * through the response chunks's iterators and convert the items. + * It will also wait for more response chunks if none are available. + * + * Since this class polls for new "streaming" data, it is advisable + * to check {@link Thread#isInterrupted()} while using this class's + * {@link Iterator} in environments where thread interrupts must be obeyed. + * + * @param The final converted type that this class exposes as part of its iterator. + * @param The type of the response chunks, contains an Iterable<{@link CoreT}> + * @param The raw response type, will get converted to {@link FinalT}. + * @author Alex Moore + * @author Sergey Galkin + * @since 2.1.0 + */ public class ChunkedResponseIterator, CoreT> implements Iterator { private final int timeout; @@ -62,6 +80,19 @@ public ChunkedResponseIterator(StreamingRiakFuture coreFuture, hasNext(); } + /** + * Returns {@code true} if the iteration has more elements. + * (In other words, returns {@code true} if {@link #next} would + * return an element rather than throwing an exception.) + * + * This method will block and wait for more data if none is immediately available. + * + * Riak Java Client Note: Since this class polls for + * new "streaming" data, it is advisable to check {@link Thread#isInterrupted()} + * in environments where thread interrupts must be obeyed. + * + * @return {@code true} if the iteration has more elements + */ @Override public boolean hasNext() { @@ -109,16 +140,37 @@ private boolean possibleChunksRemaining() return !coreFuture.isDone() || !chunkQueue.isEmpty(); } + /** + * Returns whether this response contains a continuation. + * Only run this once the operation is complete, otherwise it will return true as it's + * @return Whether this response has a continuation. + */ public boolean hasContinuation() { return continuation != null || possibleChunksRemaining(); } + /** + * Returns the current value of the continuation. + * Only run this once the operation is complete, or else you will get a null value. + * @return The continuation value (if any). + */ public BinaryValue getContinuation() { return continuation; } + /** + * Returns the next element in the iteration. + * This method will block and wait for more data if none is immediately available. + * + * Riak Java Client Note: Since this class polls for + * new "streaming" data, it is advisable to check {@link Thread#isInterrupted()} + * in environments where thread interrupts must be obeyed. + * + * @return the next element in the iteration + * @throws NoSuchElementException if the iteration has no more elements + */ @Override public FinalT next() { diff --git a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java index a61708d85..83868872e 100644 --- a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java +++ b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java @@ -426,6 +426,19 @@ private class MapReduceResponseIterator implements Iterator this.pollTimeout = pollTimeout; } + /** + * Returns {@code true} if the iteration has more elements. + * (In other words, returns {@code true} if {@link #next} would + * return an element rather than throwing an exception.) + * + * This method will block and wait for more data if none is immediately available. + * + * Riak Java Client Note: Since this class polls for + * new "streaming" data, it is advisable to check {@link Thread#isInterrupted()} + * in environments where thread interrupts must be obeyed. + * + * @return {@code true} if the iteration has more elements + */ @Override public boolean hasNext() { @@ -478,6 +491,17 @@ private boolean peekWaitForNextQueueEntry() throws InterruptedException return !resultsQueue.isEmpty(); } + /** + * Returns the next element in the iteration. + * This method will block and wait for more data if none is immediately available. + * + * Riak Java Client Note: Since this class polls for + * new "streaming" data, it is advisable to check {@link Thread#isInterrupted()} + * in environments where thread interrupts must be obeyed. + * + * @return the next element in the iteration + * @throws NoSuchElementException if the iteration has no more elements + */ @Override public Response next() { From a797ba638cadcb363bae371c9158c29ed3aeba10 Mon Sep 17 00:00:00 2001 From: Alex Moore Date: Tue, 15 Nov 2016 15:10:02 -0500 Subject: [PATCH 55/55] Changing isStreamable to isStreaming, as we cannot treat a streaming result as a non-streaming one --- .../basho/riak/client/api/StreamableRiakCommand.java | 4 ++-- .../riak/client/api/commands/buckets/ListBuckets.java | 2 +- .../api/commands/indexes/SecondaryIndexQuery.java | 11 +++++------ .../basho/riak/client/api/commands/kv/ListKeys.java | 2 +- .../riak/client/api/commands/mapreduce/MapReduce.java | 4 ++-- .../commands/indexes/itest/ITestFullBucketRead.java | 4 ++-- .../api/commands/itest/ITestBucketMapReduce.java | 7 +------ 7 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java index b24c6b2f4..ac983b63b 100644 --- a/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java +++ b/src/main/java/com/basho/riak/client/api/StreamableRiakCommand.java @@ -70,7 +70,7 @@ protected StreamableResponse() * If false, they will appear in the original result collection. * @return true if the results are to be streamed. */ - public boolean isStreamable() + public boolean isStreaming() { return chunkedResponseIterator != null; } @@ -87,7 +87,7 @@ public boolean isStreamable() */ @Override public Iterator iterator() { - if (isStreamable()) { + if (isStreaming()) { assert chunkedResponseIterator != null; return chunkedResponseIterator; } diff --git a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java index 3e9e7ffd3..80a33da31 100644 --- a/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java +++ b/src/main/java/com/basho/riak/client/api/commands/buckets/ListBuckets.java @@ -147,7 +147,7 @@ public Response(BinaryValue type, List buckets) @Override public Iterator iterator() { - if (isStreamable()) { + if (isStreaming()) { return super.iterator(); } diff --git a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java index f6a8075d6..52f6dba5b 100644 --- a/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java +++ b/src/main/java/com/basho/riak/client/api/commands/indexes/SecondaryIndexQuery.java @@ -27,7 +27,6 @@ import com.basho.riak.client.core.util.BinaryValue; import java.util.ArrayList; -import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -638,7 +637,7 @@ protected Response(Namespace queryLocation, */ public Iterator iterator() { - if (isStreamable()) { + if (isStreaming()) { return super.iterator(); } @@ -663,7 +662,7 @@ protected E convert(SecondaryIndexQueryOperation.Response.Entry e) */ public boolean hasContinuation() { - if (isStreamable()) + if (isStreaming()) { return chunkedResponseIterator.hasContinuation(); } @@ -682,7 +681,7 @@ public boolean hasContinuation() */ public BinaryValue getContinuation() { - if (isStreamable()) + if (isStreaming()) { return chunkedResponseIterator.getContinuation(); } @@ -702,7 +701,7 @@ public BinaryValue getContinuation() */ public boolean hasEntries() { - if (isStreamable()) + if (isStreaming()) { return chunkedResponseIterator.hasNext(); } @@ -718,7 +717,7 @@ public boolean hasEntries() */ public final List getEntries() { - if(isStreamable()) + if(isStreaming()) { throw new IllegalStateException("Use the iterator() while using the streaming API"); } diff --git a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java index 3e86929b6..beaf3bc71 100644 --- a/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java +++ b/src/main/java/com/basho/riak/client/api/commands/kv/ListKeys.java @@ -139,7 +139,7 @@ public Response(Namespace namespace, List keys) @Override public Iterator iterator() { - if (isStreamable()) + if (isStreaming()) { return super.iterator(); } diff --git a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java index 83868872e..ca13a51b2 100644 --- a/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java +++ b/src/main/java/com/basho/riak/client/api/commands/mapreduce/MapReduce.java @@ -355,7 +355,7 @@ public Response(Map results) } @Override - public boolean isStreamable() + public boolean isStreaming() { return responseIterator != null; } @@ -404,7 +404,7 @@ private ArrayNode flattenResults() @Override public Iterator iterator() { - if (isStreamable()) { + if (isStreaming()) { return responseIterator; } diff --git a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestFullBucketRead.java b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestFullBucketRead.java index 84adfc4ad..5da5123d4 100644 --- a/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestFullBucketRead.java +++ b/src/test/java/com/basho/riak/client/api/commands/indexes/itest/ITestFullBucketRead.java @@ -335,7 +335,7 @@ private Map> retrieveChunkedKeysForCoverageEntry(Riak readResponse = rc.executeAsyncStreaming(cmd2, pollTimeout).get(); } - assertEquals(useStreaming, readResponse.isStreamable()); + assertEquals(useStreaming, readResponse.isStreaming()); final List list = new LinkedList<>(); readResponse.forEach( e -> list.add(e)); @@ -366,7 +366,7 @@ private Map> retrieveChunkedKeysForCoverageEntry(Riak r = rc.executeAsyncStreaming(cmd2, pollTimeout).get(); } - assertEquals(useStreaming, r.isStreamable()); + assertEquals(useStreaming, r.isStreaming()); final List entries = new LinkedList<>(); diff --git a/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java b/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java index 1ad19e568..dbd056547 100644 --- a/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java +++ b/src/test/java/com/basho/riak/client/api/commands/itest/ITestBucketMapReduce.java @@ -18,7 +18,6 @@ import com.basho.riak.client.api.RiakClient; import com.basho.riak.client.core.RiakFuture; -import com.basho.riak.client.core.operations.itest.ITestAutoCleanupBase; import com.basho.riak.client.api.commands.buckets.StoreBucketProperties; import com.basho.riak.client.api.commands.kv.StoreValue; import com.basho.riak.client.api.commands.mapreduce.BucketMapReduce; @@ -34,13 +33,9 @@ import com.basho.riak.client.api.commands.mapreduce.filters.TokenizeFilter; import com.basho.riak.client.core.query.functions.Function; import com.basho.riak.client.core.util.BinaryValue; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; -import java.util.List; import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; import org.junit.*; @@ -178,7 +173,7 @@ private void streamingErlangBucketMR(String bucketType) throws InterruptedExcept int count = 0; final MapReduce.Response streamingResponse = streamingFuture.get(); - assertTrue(streamingResponse.isStreamable()); + assertTrue(streamingResponse.isStreaming()); // The streaming query should return many results which are JSON arrays, each // containing a piece of the array [0-199]. // Streaming result would look like: [[0], [1,2,3], ... [..., 199]], with the outer