Merge pull request #85 from developmentseed/features

resolves #81 #84

Author: Scisco

landsat/downloader.py

@@ -45,10 +45,12 @@ def download(self, scenes, bands=None):
             List
 
         :returns:
-            Boolean
+            (List) includes downloaded scenes as key and source as value (aws or google)
         """
 
         if isinstance(scenes, list):
+            output = {}
+
             for scene in scenes:
                 # If bands are provided the image is from 2015 or later use Amazon
                 if (bands and int(scene[12]) > 4):
@@ -61,14 +63,18 @@ def download(self, scenes, bands=None):
                             bands_plus.append('MTL')
                             for band in bands_plus:
                                 self.amazon_s3(scene, band, path)
+                                output[scene] = 'aws'
                         except RemoteFileDoesntExist:
                             self.google_storage(scene, self.download_dir)
+                            output[scene] = 'google'
+
                     else:
                         raise Exception('Expected bands list')
                 else:
                     self.google_storage(scene, self.download_dir)
+                    output[scene] = 'google'
 
-            return True
+            return output
 
         else:
             raise Exception('Expected sceneIDs list')

landsat/image.py

@@ -4,7 +4,7 @@
 
 import warnings
 import sys
-from os.path import join
+from os.path import join, isdir
 import tarfile
 import glob
 import subprocess
@@ -49,10 +49,14 @@ class Process(VerbosityMixin):
         Whether the output should be verbose. Default is False.
     :type verbose:
         boolean
+    :param force_unzip:
+        Whether to force unzip the tar file. Default is False
+    :type force_unzip:
+        boolean
 
     """
 
-    def __init__(self, path, bands=None, dst_path=None, verbose=False):
+    def __init__(self, path, bands=None, dst_path=None, verbose=False, force_unzip=False):
 
         self.projection = {'init': 'epsg:3857'}
         self.dst_crs = {'init': u'epsg:3857'}
@@ -71,7 +75,7 @@ def __init__(self, path, bands=None, dst_path=None, verbose=False):
         self.scene_path = join(self.src_path, self.scene)
 
         if self._check_if_zipped(path):
-            self._unzip(join(self.src_path, get_file(path)), join(self.src_path, self.scene), self.scene)
+            self._unzip(join(self.src_path, get_file(path)), join(self.src_path, self.scene), self.scene, force_unzip)
 
         self.bands_path = []
         for band in self.bands:
@@ -287,14 +291,19 @@ def _get_boundaries(self, src):
     def _percent_cut(self, color, low, high):
         return numpy.percentile(color[numpy.logical_and(color > 0, color < 65535)], (low, high))
 
-    def _unzip(self, src, dst, scene):
+    def _unzip(self, src, dst, scene, force_unzip=False):
         """ Unzip tar files """
         self.output("Unzipping %s - It might take some time" % scene, normal=True, arrow=True)
 
         try:
-            tar = tarfile.open(src, 'r')
-            tar.extractall(path=dst)
-            tar.close()
+            # check if file is already unzipped, skip
+            if isdir(dst) and not force_unzip:
+                self.output("%s is already unzipped." % scene, normal=True, arrow=True)
+                return
+            else:
+                tar = tarfile.open(src, 'r')
+                tar.extractall(path=dst)
+                tar.close()
         except tarfile.ReadError:
             check_create_folder(dst)
             subprocess.check_call(['tar', '-xf', src, '-C', dst])

landsat/landsat.py

@@ -86,6 +86,8 @@
 
                 --region            URL to S3 region e.g. s3-us-west-2.amazonaws.com
 
+                --force-unzip       Force unzip tar file
+
         Process:
             landsat.py process path [-h] [-b --bands] [-p --pansharpen]
 
@@ -115,6 +117,8 @@
                 --bucket            Bucket name (required if uploading to s3)
 
                 --region            URL to S3 region e.g. s3-us-west-2.amazonaws.com
+
+                --force-unzip       Force unzip tar file
 """
 
 
@@ -179,6 +183,7 @@ def args_options():
                                  'as Environment Variables)')
     parser_download.add_argument('--bucket', help='Bucket name (required if uploading to s3)')
     parser_download.add_argument('--region', help='URL to S3 region e.g. s3-us-west-2.amazonaws.com')
+    parser_download.add_argument('--force-unzip', help='Force unzip tar file', action='store_true')
 
     parser_process = subparsers.add_parser('process', help='Process Landsat imagery')
     parser_process.add_argument('path',
@@ -198,6 +203,7 @@ def args_options():
                                 'as Environment Variables)')
     parser_process.add_argument('--bucket', help='Bucket name (required if uploading to s3)')
     parser_process.add_argument('--region', help='URL to S3 region e.g. s3-us-west-2.amazonaws.com')
+    parser_process.add_argument('--force-unzip', help='Force unzip tar file', action='store_true')
 
     return parser
 
@@ -221,9 +227,11 @@ def main(args):
     v = VerbosityMixin()
 
     if args:
+
         if args.subs == 'process':
             verbose = True if args.verbose else False
-            stored = process_image(args.path, args.bands, verbose, args.pansharpen)
+            force_unzip = True if args.force_unzip else False
+            stored = process_image(args.path, args.bands, verbose, args.pansharpen, force_unzip)
 
             if args.upload:
                 u = Uploader(args.key, args.secret, args.region)
@@ -269,18 +277,21 @@ def main(args):
         elif args.subs == 'download':
             d = Downloader(download_dir=args.dest)
             try:
-                if d.download(args.scenes, convert_to_integer_list(args.bands)):
-                    if args.process:
+                downloaded = d.download(args.scenes, convert_to_integer_list(args.bands))
+
+                if args.process:
+                    force_unzip = True if args.force_unzip else False
+                    for scene, src in downloaded.iteritems():
                         if args.dest:
-                            path = join(args.dest, args.scenes[0])
+                            path = join(args.dest, scene)
                         else:
-                            path = join(settings.DOWNLOAD_DIR, args.scenes[0])
+                            path = join(settings.DOWNLOAD_DIR, scene)
 
                         # Keep using Google if the image is before 2015
-                        if (int(args.scenes[0][12]) < 5 or not args.bands):
+                        if src == 'google':
                             path = path + '.tar.bz'
 
-                        stored = process_image(path, args.bands, False, args.pansharpen)
+                        stored = process_image(path, args.bands, False, args.pansharpen, force_unzip)
 
                         if args.upload:
                             try:
@@ -291,14 +302,16 @@ def main(args):
                                 return ["Connection timeout. Probably the region parameter is incorrect", 1]
                             u.run(args.bucket, get_file(stored), stored)
 
-                        return ["The output is stored at %s" % stored]
-                    else:
-                        return ['Download Completed', 0]
+                        v.output("The output is stored at %s" % stored, normal=True, arrow=True)
+
+                    return ['Image Processing Completed', 0]
+                else:
+                    return ['Download Completed', 0]
             except IncorrectSceneId:
                 return ['The SceneID provided was incorrect', 1]
 
 
-def process_image(path, bands=None, verbose=False, pansharpen=False):
+def process_image(path, bands=None, verbose=False, pansharpen=False, force_unzip=None):
     """ Handles constructing and image process.
 
     :param path:
@@ -323,7 +336,7 @@ def process_image(path, bands=None, verbose=False, pansharpen=False):
     """
     try:
         bands = convert_to_integer_list(bands)
-        p = Process(path, bands=bands, verbose=verbose)
+        p = Process(path, bands=bands, verbose=verbose, force_unzip=force_unzip)
     except IOError:
         exit("Zip file corrupted", 1)
     except FileDoesNotExist as e:

landsat/tests/test_download.py

@@ -28,7 +28,9 @@ def setUpClass(cls):
         cls.d = Downloader()
         cls.temp_folder = mkdtemp()
         cls.scene = 'LT81360082013127LGN01'
+        cls.scene_2 = 'LC82050312014229LGN00'
         cls.scene_s3 = 'LC80010092015051LGN00'
+        cls.scene_s3_2 = 'LC82050312015136LGN00'
         cls.scene_size = 59204484
 
     @classmethod
@@ -49,21 +51,19 @@ def assertSize(self, url, path):
     def test_download(self, mock_fetch):
         mock_fetch.return_value = True
 
-        # download one list
+        # download one scene
         self.d.download([self.scene])
-        self.assertTrue(self.d.download([self.scene]))
+        self.assertEqual({self.scene: 'google'}, self.d.download([self.scene]))
+
+        # download multiple scenes
+        self.assertEqual({self.scene: 'google', self.scene_2: 'google'}, self.d.download([self.scene, self.scene_2]))
 
         # Test if error is raised when passing scene as string instead of list
         self.assertRaises(Exception, self.d.download, self.scene)
 
-        # Test if download works when passing scenes as list
-        self.d.download([self.scene, self.scene])
-        self.assertTrue(self.d.download([self.scene]))
-
         # Test when passing band list along with sceneID
-        self.d.download([self.scene_s3], bands=[11])
-
-        self.assertTrue(self.d.download([self.scene]))
+        self.assertEqual({self.scene_s3: 'aws', self.scene_s3_2: 'aws'},
+                         self.d.download([self.scene_s3, self.scene_s3_2], bands=[11]))
 
         # Test whether passing band as string raises an exception
         self.assertRaises(Exception, self.d.download, self.scene, 4)

landsat/tests/test_landsat.py

@@ -94,45 +94,54 @@ def test_download_incorrect(self):
     @mock.patch('landsat.landsat.Downloader.download')
     def test_download_process_continuous(self, mock_downloader, mock_process):
         """Test download and process commands together"""
-        mock_downloader.return_value = True
+        mock_downloader.return_value = {'LC80010092015051LGN00': 'aws',
+                                        'LC80010092014051LGN00': 'aws'}
         mock_process.return_value = 'image.TIF'
 
-        args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p']
+        args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d', self.mock_path, '-p']
         output = landsat.main(self.parser.parse_args(args))
-        mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2'])
-        mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False)
-        self.assertEquals(output, ["The output is stored at image.TIF"])
+        mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], ['4', '3', '2'])
+        mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, False)
+        self.assertEquals(output, ["Image Processing Completed", 0])
+
+        # Call with force unzip flag
+        args = ['download', 'LC80010092015051LGN00', 'LC80010092014051LGN00', '-b', '432', '-d',
+                self.mock_path, '-p', '--force-unzip']
+        output = landsat.main(self.parser.parse_args(args))
+        mock_downloader.assert_called_with(['LC80010092015051LGN00', 'LC80010092014051LGN00'], ['4', '3', '2'])
+        mock_process.assert_called_with('path/to/folder/LC80010092014051LGN00', '432', False, False, True)
+        self.assertEquals(output, ["Image Processing Completed", 0])
 
     @mock.patch('landsat.landsat.Uploader')
     @mock.patch('landsat.landsat.process_image')
     @mock.patch('landsat.landsat.Downloader.download')
     def test_download_process_continuous_with_upload(self, mock_downloader, mock_process, mock_upload):
         """Test download and process commands together"""
-        mock_downloader.return_value = True
+        mock_downloader.return_value = {'LC80010092015051LGN00': 'aws'}
         mock_process.return_value = 'image.TIF'
         mock_upload.run.return_value = True
 
         args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p',
                 '-u', '--key', 'somekey', '--secret', 'somesecret', '--bucket', 'mybucket', '--region', 'this']
         output = landsat.main(self.parser.parse_args(args))
         mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2'])
-        mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False)
+        mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False)
         mock_upload.assert_called_with('somekey', 'somesecret', 'this')
         mock_upload.return_value.run.assert_called_with('mybucket', 'image.TIF', 'image.TIF')
-        self.assertEquals(output, ["The output is stored at image.TIF"])
+        self.assertEquals(output, ["Image Processing Completed", 0])
 
     @mock.patch('landsat.landsat.process_image')
     @mock.patch('landsat.landsat.Downloader.download')
     def test_download_process_continuous_with_wrong_args(self, mock_downloader, mock_process):
         """Test download and process commands together"""
-        mock_downloader.return_value = True
+        mock_downloader.return_value = {'LC80010092015051LGN00': 'aws'}
         mock_process.return_value = 'image.TIF'
 
         args = ['download', 'LC80010092015051LGN00', '-b', '432', '-d', self.mock_path, '-p',
                 '-u', '--region', 'whatever']
         output = landsat.main(self.parser.parse_args(args))
         mock_downloader.assert_called_with(['LC80010092015051LGN00'], ['4', '3', '2'])
-        mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False)
+        mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', '432', False, False, False)
         self.assertEquals(output, ['Could not authenticate with AWS', 1])
 
     @mock.patch('landsat.landsat.process_image')
@@ -143,7 +152,7 @@ def test_process_correct(self, mock_process):
         args = ['process', 'path/to/folder/LC80010092015051LGN00']
         output = landsat.main(self.parser.parse_args(args))
 
-        mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, False)
+        mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, False, False)
         self.assertEquals(output, ["The output is stored at image.TIF"])
 
     @mock.patch('landsat.landsat.process_image')
@@ -154,7 +163,7 @@ def test_process_correct_pansharpen(self, mock_process):
         args = ['process', '--pansharpen', 'path/to/folder/LC80010092015051LGN00']
         output = landsat.main(self.parser.parse_args(args))
 
-        mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, True)
+        mock_process.assert_called_with('path/to/folder/LC80010092015051LGN00', None, False, True, False)
         self.assertEquals(output, ["The output is stored at image.TIF"])
 
     def test_process_incorrect(self):