diff --git a/README.md b/README.md index 538e45f..93cbc84 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,3 @@ And after, you can launch script with This script will open a connection on your Ubuntu One and after on your hubiC account. To open a hubiC connection, you had to login in webpage printed by script. You will get a code, and after copy it inside your console, copy of all files begin. -# Todo - -This script has two limitations : - * Usage of python-oauth require to use urllib2 to communicate with Ubuntu One and I didn't found how to stream response to reduce memory consumption. So, if you try to migrate a file of 2Gb, you will have 2gb taken by script in ram. - * hubiC segmente big files (> 100 Mb) and this script does not (yet). - diff --git a/hubic.py b/hubic.py index 66be381..022ae9a 100644 --- a/hubic.py +++ b/hubic.py @@ -110,6 +110,13 @@ def create_folder(self,name): def upload_object(self,name,data): return self._os_call('put','default'+self.prefix_destination +name,data); + def create_manifest_big_file(self,manifest_link,name): + headers = { 'X-Object-Manifest': 'default_segments/' + manifest_link} + return self._os_call('put','default'+self.prefix_destination +name,data=None,headers=headers); + + def upload_segment_big_file(self,manifest_link,number,content): + return self._os_call('put','default_segments/'+manifest_link + '/' + number,data=content); + def delete_object(self,name): return self._os_call('delete','default'+name); diff --git a/importUbuntuOneToHubic.py b/importUbuntuOneToHubic.py index b781c05..c789388 100644 --- a/importUbuntuOneToHubic.py +++ b/importUbuntuOneToHubic.py @@ -1,8 +1,12 @@ from ubuntuOne import UbuntuOne from hubic import Hubic +import uuid +import time -def for_each_current_folder(content_path,name): +def for_each_current_folder(content_path,chunk_size,name): for item in ubuntu.get_list(content_path,name): + # if this is a directory, we create the directory in hubic + # and we explore this directory if item['kind'] == 'directory': print "Create directory : " + item['path'] hubic.create_folder(item['path']) @@ -10,23 +14,53 @@ def for_each_current_folder(content_path,name): print "Discover directory : " + item['path'] for_each_current_folder(content_path,item['path']) + #if this is a file, we verify his size + # if size > chunk, we had to create a segmented file + # if not, we get it in memory and upload directly in hubic elif item['kind'] == 'file': - print "Copy file : " + item['path'] - content = ubuntu.get_file(item['content_path']) - hubic.upload_object(item['path'],content) + if item['size'] > chunk_size: + print "Copy file : " + item['path'] + stream = ubuntu.get_stream_file(item['content_path']) + if stream.status_code == 200: + i = 1 + + # get infos + the_uuid = str(uuid.uuid4()) + timestamp = str(time.time()) + size = str(item['size']) + manifest_link = "/".join([the_uuid,timestamp,size]) + # print manifest_link + + for chunk in stream.iter_content(chunk_size): + number = str('{:08}'.format(i)) + # print number + hubic.upload_segment_big_file(manifest_link,number,chunk) + i=i+1 -#ubuntu one informations + #Create manifest + hubic.create_manifest_big_file(manifest_link,item['path']) + + print "End Copy file : " + item['path'] + else: + print "Copy file : " + item['path'] + content = ubuntu.get_file(item['content_path']) + hubic.upload_object(item['path'],content) + +# ubuntu one informations uo_login = "xxx" uo_pass = "xxx" content_path = '~/Ubuntu One' -#hubic informations +# hubic informations AK = 'xxxx' AS = 'xxxx' redirect = 'https://beveloper.fr/showHubicCode/' #refresh_token = 'xxxx' destination_folder = '/ubuntu_one' +# general informations +chunk_size=1048576 #1M + #Open connections ubuntu = UbuntuOne(uo_login,uo_pass,'Ubuntu One @ python [migrate-to-hubic]') print "Connection to Ubuntu One : ok" @@ -34,6 +68,6 @@ def for_each_current_folder(content_path,name): print "Connection to Hubic : ok" # foreach element in ubuntu one -for_each_current_folder(content_path,'') +for_each_current_folder(content_path,chunk_size,'') diff --git a/ubuntuOne.py b/ubuntuOne.py index 307b4e4..8e80729 100644 --- a/ubuntuOne.py +++ b/ubuntuOne.py @@ -1,13 +1,11 @@ import json import oauth2 import requests -import urllib - +from oauth2 import Request as Requ, SignatureMethod_HMAC_SHA1 class BadRequest(Exception): """An invalid request was submitted.""" - class Unauthorized(Exception): """The provided email address and password were incorrect.""" @@ -49,3 +47,17 @@ def get_file(self, content_path): request_token_url = "https://files.one.ubuntu.com" + content_path.replace(' ','%20') resp,content = self.client.request(request_token_url, "GET") return content + + def get_stream_file(self,content_path): + url = "https://files.one.ubuntu.com" + content_path.replace(' ','%20') + + # Get Oauth1 signature + req = Requ.from_consumer_and_token(self.consumer, + token=self.token, http_method='GET', http_url=url, + parameters=None, body='') + + req.sign_request(SignatureMethod_HMAC_SHA1(), self.consumer, self.token) + realm = "https://files.one.ubuntu.com" + header = req.to_header(realm=realm) + + return requests.get(url, stream=True, headers=header) \ No newline at end of file