Skip to content

Commit cd7eb9f

Browse files
qqmyersqqmyers
committed
Support for multipart uploads
1 parent 14146b3 commit cd7eb9f

File tree

9 files changed

+648
-214
lines changed

9 files changed

+648
-214
lines changed

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<modelVersion>4.0.0</modelVersion>
55
<groupId>DVUploader</groupId>
66
<artifactId>DVUploader</artifactId>
7-
<version>1.0.9</version>
7+
<version>1.1.0</version>
88
<properties>
99
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
1010
</properties>

‎src/main/java/org/sead/uploader/AbstractUploader.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.io.PrintWriter;
2323
import java.net.MalformedURLException;
2424
import java.net.URL;
25+
import java.text.DecimalFormat;
2526
import java.util.ArrayList;
2627
import java.util.HashMap;
2728
import java.util.HashSet;
@@ -104,6 +105,13 @@ public static void println(String s) {
104105
}
105106
return;
106107
}
108+
static DecimalFormat decimalFormat = new DecimalFormat("#.00");
109+
110+
public static void printStatus(float s) {
111+
System.out.print("\rProgress: " + decimalFormat.format(s*100) + "%");
112+
System.out.flush();
113+
return;
114+
}
107115

108116
public void parseArgs(String[] args) {
109117

‎src/main/java/org/sead/uploader/dataverse/DVUploader.java

Lines changed: 432 additions & 197 deletions
Large diffs are not rendered by default.

‎src/main/java/org/sead/uploader/dataverse/HttpPartUploadJob.java

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
* To change this license header, choose License Headers in Project Properties.
3+
* To change this template file, choose Tools | Templates
4+
* and open the template in the editor.
5+
*/
6+
package org.sead.uploader.dataverse;
7+
8+
import java.io.IOException;
9+
import java.io.InputStream;
10+
import java.util.Map;
11+
import org.apache.http.HttpEntity;
12+
import org.apache.http.client.methods.CloseableHttpResponse;
13+
import org.apache.http.client.methods.HttpPut;
14+
import org.apache.http.client.protocol.HttpClientContext;
15+
import org.apache.http.entity.InputStreamEntity;
16+
import org.apache.http.impl.client.CloseableHttpClient;
17+
import org.apache.http.util.EntityUtils;
18+
import static org.sead.uploader.AbstractUploader.println;
19+
import org.sead.uploader.util.Resource;
20+
21+
/**
22+
*
23+
* @author Jim
24+
*/
25+
public class HttpPartUploadJob implements Runnable {
26+
27+
int partNo;
28+
long size;
29+
String signedUrl;
30+
Resource file;
31+
Map eTags;
32+
33+
static private long partSize = -1;
34+
static private CloseableHttpClient httpClient = null;
35+
static private HttpClientContext localContext = null;
36+
37+
public static void setHttpClient(CloseableHttpClient client) {
38+
httpClient = client;
39+
}
40+
41+
public static void setHttpClientContext(HttpClientContext context) {
42+
localContext = context;
43+
}
44+
45+
public static void setPartSize(long ps) {
46+
partSize=ps;
47+
}
48+
49+
public HttpPartUploadJob(int partNo, String url, Resource file, long size, Map eTags) throws IllegalStateException {
50+
if ((size == -1) || (httpClient == null) || (localContext == null)) {
51+
throw new IllegalStateException("partSize not set");
52+
}
53+
this.partNo = partNo;
54+
this.signedUrl = url;
55+
this.file = file;
56+
this.size=size;
57+
this.eTags = eTags;
58+
}
59+
60+
/*
61+
* (non-Javadoc)
62+
*
63+
* @see java.lang.Runnable#run()
64+
*/
65+
public void run() {
66+
int retries = 3;
67+
//println("Starting upload of part: " + partNo);
68+
while (retries > 0) {
69+
try (InputStream is = file.getInputStream((partNo - 1) * partSize, size)) {
70+
71+
HttpPut httpput = new HttpPut(signedUrl);
72+
httpput.setEntity(new InputStreamEntity(is, size));
73+
CloseableHttpResponse putResponse = httpClient.execute(httpput);
74+
int putStatus = putResponse.getStatusLine().getStatusCode();
75+
String putRes = null;
76+
HttpEntity putEntity = putResponse.getEntity();
77+
if (putEntity != null) {
78+
putRes = EntityUtils.toString(putEntity);
79+
}
80+
if (putStatus == 200) {
81+
//Part successfully stored - parse the eTag from the response and it it to the Map
82+
String eTag = putResponse.getFirstHeader("ETag").getValue();
83+
eTag= eTag.replace("\"","");
84+
eTags.put(Integer.toString(partNo), eTag);
85+
retries = 0;
86+
//println("Completed upload of part: " + partNo);
87+
} else {
88+
if (putStatus >= 500) {
89+
println("Upload of part: " + partNo + " failed with status: " + putStatus + " (skipping)");
90+
println("Error response: " + putResponse.getStatusLine() + " : " + putRes);
91+
retries--;
92+
} else {
93+
println("Upload of part: " + partNo + " failed with status: " + putStatus + " (retrying)");
94+
println("Error response: " + putResponse.getStatusLine() + " : " + putRes);
95+
96+
retries--;
97+
}
98+
}
99+
100+
} catch (IOException e) {
101+
e.printStackTrace(System.out);
102+
println("Error uploading part: " + partNo + " : " + e.getMessage());
103+
retries = 0;
104+
}
105+
}
106+
}
107+
}

‎src/main/java/org/sead/uploader/dataverse/MD5Job.java

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* To change this license header, choose License Headers in Project Properties.
3+
* To change this template file, choose Tools | Templates
4+
* and open the template in the editor.
5+
*/
6+
package org.sead.uploader.dataverse;
7+
8+
import java.io.IOException;
9+
import java.io.InputStream;
10+
import java.security.DigestInputStream;
11+
import java.security.MessageDigest;
12+
import java.security.NoSuchAlgorithmException;
13+
import java.util.Map;
14+
import org.apache.commons.codec.binary.Hex;
15+
import org.apache.http.HttpEntity;
16+
import org.apache.http.client.methods.CloseableHttpResponse;
17+
import org.apache.http.client.methods.HttpPut;
18+
import org.apache.http.client.protocol.HttpClientContext;
19+
import org.apache.http.entity.InputStreamEntity;
20+
import org.apache.http.impl.client.CloseableHttpClient;
21+
import org.apache.http.util.EntityUtils;
22+
import static org.sead.uploader.AbstractUploader.println;
23+
import org.sead.uploader.util.Resource;
24+
25+
/**
26+
*
27+
* @author Jim
28+
*/
29+
public class MD5Job implements Runnable {
30+
31+
Resource file;
32+
Map infoMap;
33+
34+
public MD5Job(Resource file, Map infoMap) throws IllegalStateException {
35+
this.file = file;
36+
this.infoMap = infoMap;
37+
}
38+
39+
/*
40+
* (non-Javadoc)
41+
*
42+
* @see java.lang.Runnable#run()
43+
*/
44+
public void run() {
45+
try {
46+
MessageDigest messageDigest = MessageDigest.getInstance("MD5");
47+
48+
try (InputStream inStream = file.getInputStream(); DigestInputStream digestInputStream = new DigestInputStream(inStream, messageDigest)) {
49+
byte[] bytes = new byte[64*1024];
50+
while(digestInputStream.read(bytes) >= 0) {
51+
}
52+
String checksum = Hex.encodeHexString(digestInputStream.getMessageDigest().digest());
53+
infoMap.put("md5", checksum);
54+
} catch (IOException e) {
55+
e.printStackTrace(System.out);
56+
println("Error calculating digest for: " + file.getAbsolutePath() + " : " + e.getMessage());
57+
}
58+
} catch (NoSuchAlgorithmException nsae) {
59+
println("MD5 algorithm not found: " + nsae.getMessage());
60+
}
61+
}
62+
}

‎src/main/java/org/sead/uploader/util/FileResource.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,13 @@
3131

3232
import org.apache.commons.codec.binary.Hex;
3333
import org.apache.commons.io.IOUtils;
34+
import org.apache.commons.io.input.BoundedInputStream;
3435
import org.apache.http.entity.ContentType;
3536
import org.apache.http.entity.mime.content.ContentBody;
3637
import org.apache.http.entity.mime.content.FileBody;
3738
import org.json.JSONObject;
3839

39-
public class FileResource implements Resource {
40+
public class FileResource extends Resource {
4041

4142
private File f;
4243

@@ -163,5 +164,4 @@ public JSONObject getMetadata() {
163164
// No extra metadata by default for files
164165
return new JSONObject();
165166
}
166-
167167
}

‎src/main/java/org/sead/uploader/util/PublishedFolderProxyResource.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
* @author Jim
4141
*
4242
*/
43-
public class PublishedFolderProxyResource extends PublishedResource implements Resource {
43+
public class PublishedFolderProxyResource extends PublishedResource {
4444

4545
private PublishedResource resource;
4646
private String message;

‎src/main/java/org/sead/uploader/util/PublishedResource.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
import org.json.JSONException;
3434
import org.json.JSONObject;
3535

36-
public class PublishedResource implements Resource {
36+
public class PublishedResource extends Resource {
3737

3838
protected JSONObject resource;
3939
private String path;

‎src/main/java/org/sead/uploader/util/Resource.java

Lines changed: 34 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,33 +15,55 @@
1515
***************************************************************************** */
1616
package org.sead.uploader.util;
1717

18+
import java.io.FileNotFoundException;
19+
import java.io.IOException;
1820
import java.io.InputStream;
21+
import org.apache.commons.io.input.BoundedInputStream;
1922

2023
import org.apache.http.entity.mime.content.ContentBody;
2124
import org.json.JSONObject;
2225

23-
public interface Resource extends Iterable<Resource> {
26+
public abstract class Resource implements Iterable<Resource> {
2427

25-
String getName();
28+
public abstract String getName();
2629

27-
boolean isDirectory();
30+
public abstract boolean isDirectory();
2831

29-
String getPath();
32+
public abstract String getPath();
3033

31-
long length();
34+
public abstract long length();
3235

33-
String getAbsolutePath();
36+
public abstract String getAbsolutePath();
3437

35-
ContentBody getContentBody();
38+
public abstract ContentBody getContentBody();
3639

37-
InputStream getInputStream();
40+
public abstract InputStream getInputStream();
3841

39-
Iterable<Resource> listResources();
42+
public abstract Iterable<Resource> listResources();
4043

41-
String getHash(String algorithm);
44+
public abstract String getHash(String algorithm);
4245

43-
JSONObject getMetadata();
46+
public abstract JSONObject getMetadata();
4447

45-
String getMimeType();
48+
public abstract String getMimeType();
49+
50+
/**
51+
*
52+
* @param l
53+
* @param partSize
54+
* @return
55+
*/
56+
public InputStream getInputStream(long l, long partSize) {
57+
try {
58+
InputStream is = getInputStream();
59+
is.skip(l);
60+
return new BoundedInputStream(is, partSize);
61+
} catch (FileNotFoundException e) {
62+
e.printStackTrace();
63+
} catch (IOException e) {
64+
e.printStackTrace();
65+
}
66+
return null;
67+
}
4668

4769
}

0 commit comments

Comments
 (0)