initial checkin

.gitignore

@@ -0,0 +1 @@
+__pycache__

LICENSE-mit.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2022 Saul Pwanson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+include requirements.txt
+include LICENSE-mit.txt

README.md

@@ -0,0 +1,31 @@
+# unzip-http
+
+Extract files from .zip files over http without downloading the entire archive.
+
+## Install
+
+    pip install unzip-http
+
+## Usage
+
+    unzip-http <url.zip> <filenames..>
+
+Extract <filenames> from a remote .zip at <url> to stdout.
+
+If no filenames given, displays .zip contents (filenames and sizes).
+
+Each filename can be a wildcard glob; all matching files are concatenated and sent to stdout in zipfile order.
+
+Note: HTTP server must send `Accept-Ranges: bytes` and `Content-Length` in headers.
+
+# Python module `unzip_http`
+
+    import unzip_http
+
+    rzf = unzip_http.RemoteZipFile('https://example.com/foo.zip')
+    binfp = rzf.open('bar.bin')
+    txtfp = rzf.open_text('baz.txt')
+
+# Credits
+
+`unzip-http` was written by [Saul Pwanson](https://saul.pw) and made available for use under the MIT License.

requirements.txt

@@ -0,0 +1 @@
+urllib3

setup.py

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier:
+
+from setuptools import setup
+
+
+def readme():
+    with open("README.md") as f:
+        return f.read()
+
+def requirements():
+    with open("requirements.txt") as f:
+        return f.read().split("\n")
+
+
+setup(
+        name="unzip-http",
+        version="0.1",
+        description="extract files from .zip files over http without downloading entire archive",
+        long_description=readme(),
+        long_description_content_type="text/markdown",
+        classifiers=[
+            "Development Status :: 4 - Beta",
+            "Programming Language :: Python ::3",
+        ],
+        keywords="http zip unzip",
+        author="Saul Pwanson",
+        url="https://github.com/saulpw/unzip-http",
+        python_requires=">=3.8",
+        py_modules=["unzip_http"],
+        packages=["unzip-http"],
+        scripts=["unzip-http"],
+        install_requires=requirements(),
+)

unzip-http

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+'''
+Usage:
+    unzip-http <url.zip> <filenames..>
+
+Extract <filenames> from a remote .zip at <url> to stdout.
+If no filenames given, displays .zip contents (filenames and sizes).
+Each filename can be a wildcard glob; all matching files are concatenated and sent to stdout in zipfile order.
+
+HTTP server must send `Accept-Ranges: bytes` and `Content-Length` in headers.
+'''
+
+import sys
+import io
+import time
+import fnmatch
+
+import unzip_http
+
+
+class StreamProgress:
+    def __init__(self, fp, name='', total=0):
+        self.name = name
+        self.fp = fp
+        self.total = total
+        self.start_time = time.time()
+        self.last_update = 0
+        self.amtread = 0
+
+    def read(self, n):
+        r = self.fp.read(n)
+        self.amtread += len(r)
+        now = time.time()
+        if now - self.last_update > 0.1:
+            self.last_update = now
+
+            elapsed_s = now - self.start_time
+            sys.stderr.write(f'\r{elapsed_s:.0f}s  {self.amtread/10**6:.02f}/{self.total/10**6:.02f}MB  ({self.amtread/10**6/elapsed_s:.02f} MB/s)  {self.name}')
+
+        if not r:
+            sys.stderr.write('\n')
+
+        return r
+
+
+def main(url, *globs):
+    rzf = unzip_http.RemoteZipFile(url)
+    for f in rzf.infolist():
+        if not globs:
+            print(f'{f.compress_size/2**20:.02f}MB -> {f.file_size/2**20:.02f}MB  {f.filename}')
+        elif any(fnmatch.fnmatch(f.filename, g) for g in globs):
+            fp = StreamProgress(rzf.open(f), name=f.filename, total=f.compress_size)
+            while r := fp.read(2**18):
+                sys.stdout.buffer.write(r)
+
+
+args = sys.argv[1:]
+if not args:
+    print(__doc__, file=sys.stderr)
+else:
+    main(*args)

unzip_http.py

@@ -0,0 +1,125 @@
+from dataclasses import dataclass
+
+import sys
+import io
+import time
+import zlib
+import struct
+import fnmatch
+
+import urllib3
+
+
+def error(s):
+    raise Exception(s)
+
+
+@dataclass
+class RemoteZipInfo:
+    filename:str = ''
+    date_time:int = 0
+    header_offset:int = 0
+    compress_type:int = 0
+    compress_size:int = 0
+    file_size:int = 0
+
+
+class RemoteZipFile:
+    fmt_endcdir = 'IHHHHIIH'
+    fmt_cdirentry = '<IHHHHIIIIHHHHHII'
+    fmt_localhdr = '<IHHHIIIIHH'
+
+    def __init__(self, url):
+        self.url = url
+        self.http = urllib3.PoolManager()
+
+    @property
+    def files(self):
+        return {r.filename:r for r in self.infolist()}
+
+    def infolist(self):
+        resp = self.http.request('HEAD', self.url)
+        r = resp.headers.get('Accept-Ranges', '')
+        if r != 'bytes':
+            error(f"Accept-Ranges header must be 'bytes' ('{r}')")
+
+        sz = int(resp.headers['Content-Length'])
+        resp = self.get_range(sz-65536, 65536)
+        i = resp.data.rfind(b'\x50\x4b\x05\x06')
+
+        magic, disk_num, disk_start, disk_num_records, total_num_records, cdir_bytes, cdir_start, comment_len = struct.unpack_from(self.fmt_endcdir, resp.data, offset=i)
+
+        filehdr_index = 65536 - (sz - cdir_start)
+        cdir_end = filehdr_index + cdir_bytes
+        while filehdr_index < cdir_end:
+            sizeof_cdirentry = struct.calcsize(self.fmt_cdirentry)
+
+            magic, ver, ver_needed, flags, method, date_time, crc, \
+                complen, uncomplen, fnlen, extralen, commentlen, \
+                disknum_start, internal_attr, external_attr, local_header_ofs = \
+                    struct.unpack_from(self.fmt_cdirentry, resp.data, offset=filehdr_index)
+
+            filename = resp.data[filehdr_index+sizeof_cdirentry:filehdr_index+sizeof_cdirentry+fnlen]
+
+            filehdr_index += sizeof_cdirentry + fnlen + extralen + commentlen
+
+            yield RemoteZipInfo(filename.decode(), date_time, local_header_ofs, method, complen, uncomplen)
+
+    def get_range(self, start, n):
+        return self.http.request('GET', self.url, headers={'Range': f'bytes={start}-{start+n}'}, preload_content=False)
+
+    def matching_files(self, *globs):
+        for f in self.files.values():
+            if any(fnmatch.fnmatch(f.filename, g) for g in globs):
+                yield f
+
+    def open(self, fn):
+        if isinstance(fn, str):
+            f = list(self.matching_files(fn))
+            if not f:
+                error(f'no files matching {fn}')
+            f = f[0]
+        else:
+            f = fn
+
+        sizeof_localhdr = struct.calcsize(self.fmt_localhdr)
+        r = self.get_range(f.header_offset, sizeof_localhdr)
+        localhdr = struct.unpack_from(self.fmt_localhdr, r.data)
+        magic, ver, flags, method, dos_datetime, _, _, uncomplen, fnlen, extralen = localhdr
+        if method == 0: # none
+            return self.get_range(f.header_offset + sizeof_localhdr + fnlen + extralen, f.compress_size)
+        elif method == 8: # DEFLATE
+            resp = self.get_range(f.header_offset + sizeof_localhdr + fnlen + extralen, f.compress_size)
+            return RemoteZipStream(resp, f)
+        else:
+            error(f'unknown compression method {method}')
+
+    def open_text(self, fn):
+        return io.TextIOWrapper(io.BufferedReader(self.open(fn)))
+
+
+class RemoteZipStream(io.RawIOBase):
+    def __init__(self, fp, info):
+        self.raw = fp
+        self._decompressor = zlib.decompressobj(-15)
+        self._buffer = bytes()
+
+    def readable(self):
+        return True
+
+    def readinto(self, b, /):
+        r = self.read(len(b))
+        b[:len(r)] = r
+        return len(r)
+
+    def read(self, n):
+        while n > len(self._buffer):
+            r = self.raw.read(2**18)
+            if not r:
+                break
+            self._buffer += self._decompressor.decompress(r)
+
+        ret = self._buffer[:n]
+        self._buffer = self._buffer[n:]
+
+        return ret