Skip to content

Commit 57dc060

Browse files
author
Sharon Shabtai
authored
Merge pull request #216 from scribe/233
JavaCLI-233 : Make calls paginate
2 parents b5e8a8d + 8d52ff5 commit 57dc060

File tree

6 files changed

+104
-40
lines changed

6 files changed

+104
-40
lines changed

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
buildscript {
1717

1818
buildscript {
19-
ext.kotlin_version = '1.3.72'
19+
ext.kotlin_version = '1.4.10'
2020
}
2121

2222
repositories {

‎ds3_java_cli/src/main/java/com/spectralogic/ds3cli/command/GetBucket.java‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
import com.spectralogic.ds3client.models.Contents;
3232
import com.spectralogic.ds3client.networking.FailedRequestException;
3333
import org.apache.commons.cli.Option;
34-
34+
import java.util.ArrayList;
3535
import java.util.List;
3636

3737
import static com.spectralogic.ds3cli.ArgumentFactory.BUCKET;
@@ -42,6 +42,7 @@ public class GetBucket extends CliCommand<GetBucketResult> {
4242

4343
private final static ImmutableList<Option> requiredArgs = ImmutableList.of(BUCKET);
4444
private final static ImmutableList<Option> optionalArgs = ImmutableList.of(PREFIX, SHOW_VERSIONS);
45+
private final static int maxKeys = 1000;
4546

4647
private String bucket;
4748
private String prefix;
@@ -60,22 +61,42 @@ public CliCommand init(final Arguments args) throws Exception {
6061
@Override
6162
public GetBucketResult call() throws Exception {
6263
try {
64+
final List<Contents> contents = new ArrayList<>();
6365
// GetBucketDetail to get both name and id
64-
final GetBucketRequest getBucketRequest = new GetBucketRequest(bucket);
65-
getBucketRequest.withVersions(showVersion);
66-
getBucketRequest.withPrefix(prefix);
67-
final GetBucketSpectraS3Request getBucketSpectraS3Request = new GetBucketSpectraS3Request(bucket);
68-
final GetBucketSpectraS3Response response = getClient().getBucketSpectraS3(getBucketSpectraS3Request);
69-
final Bucket bucketDetails = response.getBucketResult();
70-
final GetBucketResponse bucket = getClient().getBucket(getBucketRequest);
66+
GetBucketRequest getBucketRequest = new GetBucketRequest(bucket)
67+
.withVersions(showVersion)
68+
.withPrefix(prefix)
69+
.withMaxKeys(maxKeys);
70+
71+
GetBucketResponse bucketResponse = getClient().getBucket(getBucketRequest);
7172

72-
final List<Contents> contents;
7373
if (showVersion) {
74-
contents = bucket.getListBucketResult().getVersionedObjects();
74+
contents.addAll(bucketResponse.getListBucketResult().getVersionedObjects());
7575
} else {
76-
contents = bucket.getListBucketResult().getObjects();
76+
contents.addAll(bucketResponse.getListBucketResult().getObjects());
77+
}
78+
79+
String marker = bucketResponse.getListBucketResult().getNextMarker();
80+
while (marker != null) {
81+
getBucketRequest = new GetBucketRequest(this.bucket)
82+
.withVersions(showVersion)
83+
.withPrefix(prefix)
84+
.withMaxKeys(maxKeys)
85+
.withMarker(marker);
86+
87+
bucketResponse = getClient().getBucket(getBucketRequest);
88+
89+
if (showVersion) {
90+
contents.addAll(bucketResponse.getListBucketResult().getVersionedObjects());
91+
} else {
92+
contents.addAll(bucketResponse.getListBucketResult().getObjects());
93+
}
94+
marker = bucketResponse.getListBucketResult().getNextMarker();
7795
}
7896

97+
final GetBucketSpectraS3Request getBucketSpectraS3Request = new GetBucketSpectraS3Request(this.bucket);
98+
final GetBucketSpectraS3Response response = getClient().getBucketSpectraS3(getBucketSpectraS3Request);
99+
final Bucket bucketDetails = response.getBucketResult();
79100
return new GetBucketResult(bucketDetails, contents);
80101
} catch (final FailedRequestException e) {
81102
if (e.getStatusCode() == 404) {

‎ds3_java_cli/src/main/java/com/spectralogic/ds3cli/command/GetObjectsOnTape.java‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,22 @@
2525
import com.spectralogic.ds3cli.views.json.DataView;
2626
import com.spectralogic.ds3client.commands.spectrads3.GetBlobsOnTapeSpectraS3Request;
2727
import com.spectralogic.ds3client.commands.spectrads3.GetBlobsOnTapeSpectraS3Response;
28+
import com.spectralogic.ds3client.models.BulkObject;
2829
import com.spectralogic.ds3client.networking.FailedRequestException;
2930
import org.apache.commons.cli.Option;
3031

3132
import java.io.IOException;
33+
import java.util.ArrayList;
34+
import java.util.List;
3235

3336
import static com.spectralogic.ds3cli.ArgumentFactory.ID;
37+
import static com.spectralogic.ds3cli.ArgumentFactory.PREFIX;
3438

3539

3640
public class GetObjectsOnTape extends CliCommand<GetObjectsOnTapeResult> {
3741

3842
private final static ImmutableList<Option> requiredArgs = ImmutableList.of(ID);
43+
private final static int maxKeys = 1000;
3944

4045
// Barcode or tape ID
4146
private String tapeId;
@@ -48,20 +53,31 @@ public CliCommand init(final Arguments args) throws Exception {
4853
processCommandOptions(requiredArgs, EMPTY_LIST, args);
4954

5055
this.tapeId = args.getId();
51-
return this;
56+
return this;
5257
}
5358

5459
@Override
5560
public GetObjectsOnTapeResult call() throws CommandException, IOException {
61+
final List<BulkObject> objects = new ArrayList<>();
62+
int offset = 0;
5663
try {
57-
58-
final GetBlobsOnTapeSpectraS3Response response
59-
= getClient().getBlobsOnTapeSpectraS3(new GetBlobsOnTapeSpectraS3Request(this.tapeId));
60-
61-
return new GetObjectsOnTapeResult(this.tapeId, response.getBulkObjectListResult().getObjects());
64+
while (true) {
65+
final GetBlobsOnTapeSpectraS3Request request = new GetBlobsOnTapeSpectraS3Request(this.tapeId)
66+
.withPageOffset(offset)
67+
.withPageLength(maxKeys);
68+
final GetBlobsOnTapeSpectraS3Response response = getClient().getBlobsOnTapeSpectraS3(request);
69+
final List<BulkObject> objectList = response.getBulkObjectListResult().getObjects();
70+
if (objectList.isEmpty()) {
71+
break;
72+
} else {
73+
objects.addAll(response.getBulkObjectListResult().getObjects());
74+
}
75+
offset = objects.size();
76+
}
77+
return new GetObjectsOnTapeResult(this.tapeId, objects);
6278
} catch (final FailedRequestException e) {
6379
if (e.getStatusCode() == 404) {
64-
throw new CommandException("Unknown tape '" + this.tapeId +"'", e);
80+
throw new CommandException("Unknown tape '" + this.tapeId + "'", e);
6581
}
6682
throw e;
6783
}

‎gradle/wrapper/gradle-wrapper.jar‎

4.19 KB
Binary file not shown.

‎gradlew‎

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,21 @@
11
#!/usr/bin/env sh
22

3+
#
4+
# Copyright 2015 the original author or authors.
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# https://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
319
##############################################################################
420
##
521
## Gradle start up script for UN*X
@@ -28,7 +44,7 @@ APP_NAME="Gradle"
2844
APP_BASE_NAME=`basename "$0"`
2945

3046
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
31-
DEFAULT_JVM_OPTS=""
47+
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
3248

3349
# Use the maximum available, or set MAX_FD != -1 to use that value.
3450
MAX_FD="maximum"
@@ -109,8 +125,8 @@ if $darwin; then
109125
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
110126
fi
111127

112-
# For Cygwin, switch paths to Windows format before running java
113-
if $cygwin ; then
128+
# For Cygwin or MSYS, switch paths to Windows format before running java
129+
if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
114130
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
115131
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
116132
JAVACMD=`cygpath --unix "$JAVACMD"`
@@ -138,19 +154,19 @@ if $cygwin ; then
138154
else
139155
eval `echo args$i`="\"$arg\""
140156
fi
141-
i=$((i+1))
157+
i=`expr $i + 1`
142158
done
143159
case $i in
144-
(0) set -- ;;
145-
(1) set -- "$args0" ;;
146-
(2) set -- "$args0" "$args1" ;;
147-
(3) set -- "$args0" "$args1" "$args2" ;;
148-
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
149-
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
150-
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
151-
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
152-
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
153-
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
160+
0) set -- ;;
161+
1) set -- "$args0" ;;
162+
2) set -- "$args0" "$args1" ;;
163+
3) set -- "$args0" "$args1" "$args2" ;;
164+
4) set -- "$args0" "$args1" "$args2" "$args3" ;;
165+
5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
166+
6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
167+
7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
168+
8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
169+
9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
154170
esac
155171
fi
156172

@@ -159,14 +175,9 @@ save () {
159175
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160176
echo " "
161177
}
162-
APP_ARGS=$(save "$@")
178+
APP_ARGS=`save "$@"`
163179

164180
# Collect all arguments for the java command, following the shell quoting and substitution rules
165181
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166182

167-
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168-
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169-
cd "$(dirname "$0")"
170-
fi
171-
172183
exec "$JAVACMD" "$@"

‎gradlew.bat‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,19 @@
1+
@rem
2+
@rem Copyright 2015 the original author or authors.
3+
@rem
4+
@rem Licensed under the Apache License, Version 2.0 (the "License");
5+
@rem you may not use this file except in compliance with the License.
6+
@rem You may obtain a copy of the License at
7+
@rem
8+
@rem https://www.apache.org/licenses/LICENSE-2.0
9+
@rem
10+
@rem Unless required by applicable law or agreed to in writing, software
11+
@rem distributed under the License is distributed on an "AS IS" BASIS,
12+
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
@rem See the License for the specific language governing permissions and
14+
@rem limitations under the License.
15+
@rem
16+
117
@if "%DEBUG%" == "" @echo off
218
@rem ##########################################################################
319
@rem
@@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0
1430
set APP_HOME=%DIRNAME%
1531

1632
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
17-
set DEFAULT_JVM_OPTS=
33+
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
1834

1935
@rem Find java.exe
2036
if defined JAVA_HOME goto findJavaFromJavaHome

0 commit comments

Comments
 (0)