Fix ram consumption and big files uploads

README.md

@@ -28,9 +28,3 @@ And after, you can launch script with
 
 This script will open a connection on your Ubuntu One and after on your hubiC account. To open a hubiC connection, you had to login in webpage printed by script. You will get a code, and after copy it inside your console, copy of all files begin.
 
-# Todo
-
-This script has two limitations : 
- * Usage of python-oauth require to use urllib2 to communicate with Ubuntu One and I didn't found how to stream response to reduce memory consumption. So, if you try to migrate a file of 2Gb, you will have 2gb taken by script in ram.
- * hubiC segmente big files (> 100 Mb) and this script does not (yet).
-

hubic.py

@@ -110,6 +110,13 @@ def create_folder(self,name):
     def upload_object(self,name,data):
         return self._os_call('put','default'+self.prefix_destination +name,data);
 
+    def create_manifest_big_file(self,manifest_link,name):
+        headers = { 'X-Object-Manifest': 'default_segments/' + manifest_link}
+        return self._os_call('put','default'+self.prefix_destination +name,data=None,headers=headers);
+
+    def upload_segment_big_file(self,manifest_link,number,content):
+        return self._os_call('put','default_segments/'+manifest_link + '/' + number,data=content);
+
     def delete_object(self,name):
         return self._os_call('delete','default'+name);
 

importUbuntuOneToHubic.py

@@ -1,39 +1,73 @@
 from ubuntuOne import UbuntuOne
 from hubic import Hubic
+import uuid
+import time
 
-def for_each_current_folder(content_path,name):
+def for_each_current_folder(content_path,chunk_size,name):
 	for item in ubuntu.get_list(content_path,name):
+		# if this is a directory, we create the directory in hubic 
+		# and we explore this directory
 		if item['kind'] == 'directory':
 			print "Create directory : " + item['path']
 			hubic.create_folder(item['path'])
 
 			print "Discover directory : " + item['path']
 			for_each_current_folder(content_path,item['path'])
 
+		#if this is a file, we verify his size
+		# if size > chunk, we had to create a segmented file
+		# if not, we get it in memory and upload directly in hubic
 		elif item['kind'] == 'file':
-			print "Copy file : " + item['path']
-			content = ubuntu.get_file(item['content_path'])
-			hubic.upload_object(item['path'],content)
+			if item['size'] > chunk_size:
+				print "Copy file : " + item['path']
+				stream = ubuntu.get_stream_file(item['content_path'])
+				if stream.status_code == 200: 
+					i = 1
+
+					# get infos
+					the_uuid = str(uuid.uuid4())
+					timestamp =  str(time.time())
+					size = str(item['size'])
+					manifest_link = "/".join([the_uuid,timestamp,size])
+					# print manifest_link
+
+					for chunk in stream.iter_content(chunk_size):	
+						number = str('{:08}'.format(i))
+						# print number
+						hubic.upload_segment_big_file(manifest_link,number,chunk)
+						i=i+1
 
-#ubuntu one informations
+					#Create manifest
+					hubic.create_manifest_big_file(manifest_link,item['path'])
+
+				print "End Copy file : " + item['path']
+			else:
+				print "Copy file : " + item['path']
+				content = ubuntu.get_file(item['content_path'])
+				hubic.upload_object(item['path'],content)
+
+# ubuntu one informations
 uo_login = "xxx"
 uo_pass = "xxx"
 content_path = '~/Ubuntu One'
 
-#hubic informations
+# hubic informations
 AK = 'xxxx'
 AS = 'xxxx'
 redirect = 'https://beveloper.fr/showHubicCode/'
 #refresh_token = 'xxxx'
 destination_folder = '/ubuntu_one'
 
+# general informations
+chunk_size=1048576 #1M
+
 #Open connections
 ubuntu = UbuntuOne(uo_login,uo_pass,'Ubuntu One @ python [migrate-to-hubic]')
 print "Connection to Ubuntu One : ok"
 hubic = Hubic(AK,AS, redirect, refresh_token=None, prefix_destination=destination_folder)
 print "Connection to Hubic : ok"
 
 # foreach element in ubuntu one
-for_each_current_folder(content_path,'')
+for_each_current_folder(content_path,chunk_size,'')
 
 

ubuntuOne.py

@@ -1,13 +1,11 @@
 import json
 import oauth2
 import requests
-import urllib
-
+from oauth2 import Request as Requ, SignatureMethod_HMAC_SHA1
 
 class BadRequest(Exception):
     """An invalid request was submitted."""
 
-
 class Unauthorized(Exception):
     """The provided email address and password were incorrect."""
 
@@ -49,3 +47,17 @@ def get_file(self, content_path):
         request_token_url = "https://files.one.ubuntu.com" +  content_path.replace(' ','%20') 
         resp,content = self.client.request(request_token_url, "GET")
         return content
+
+    def get_stream_file(self,content_path):
+        url = "https://files.one.ubuntu.com" +  content_path.replace(' ','%20')
+
+        # Get Oauth1 signature
+        req = Requ.from_consumer_and_token(self.consumer,
+            token=self.token, http_method='GET', http_url=url,
+            parameters=None, body='')
+
+        req.sign_request(SignatureMethod_HMAC_SHA1(), self.consumer, self.token)
+        realm = "https://files.one.ubuntu.com"
+        header = req.to_header(realm=realm)
+
+        return requests.get(url, stream=True, headers=header)