Squashed commit of the following:

commit 7e40d00183cc9f8cb9610f558ff56c09667df204 Author: Michael Hirsch, Ph.D <[email protected]> Date: Mon Dec 31 23:53:16 2018 -0500 vers commit 15b469782d274e97c1bdba88acb76b3bf232390b Author: Michael Hirsch, Ph.D <[email protected]> Date: Mon Dec 31 23:52:26 2018 -0500 stringio API commit 3fac0a5e100923fe67e37feed90dadbcd48eec38 Author: Michael Hirsch, Ph.D <[email protected]> Date: Mon Dec 31 23:46:39 2018 -0500 check type stringio commit e57d5d6c8f14d2f7768d0c37c5113888fc7a03fd Author: Michael Hirsch, Ph.D <[email protected]> Date: Mon Dec 31 23:43:32 2018 -0500 gettime stringio commit 7f3f026a4fc15f193527ad79a22f2a02de1e4ce3 Author: Michael Hirsch, Ph.D <[email protected]> Date: Mon Dec 31 23:20:29 2018 -0500 OBS3 stringio commit 731c541bac29df634394747b6091a6459fae4dce Author: Michael Hirsch, Ph.D <[email protected]> Date: Mon Dec 31 23:12:33 2018 -0500 OBS2 stringio commit b8687aa669bcbd83716bbab482a4b18c3a2c7eaf Author: Michael Hirsch, Ph.D <[email protected]> Date: Mon Dec 31 22:50:27 2018 -0500 NAV2 stringio commit 01e78e96eb05a3d6f48cf6e03dcee7942532ee47 Author: Michael Hirsch, Ph.D <[email protected]> Date: Mon Dec 31 22:45:49 2018 -0500 NAV3 stringio
geospace-code · Jan 1, 2019 · 1a85c13 · 1a85c13
1 parent 64f0450
commit 1a85c13
Show file tree

Hide file tree

Showing 10 changed files with 284 additions and 106 deletions.
diff --git a/README.md b/README.md
@@ -9,7 +9,7 @@
 # GeoRinex
 
 RINEX 3 and RINEX 2 reader and batch conversion to NetCDF4 / HDF5 in Python or Matlab.
-Batch converts NAV and OBS GPS RINEX data into
+Batch converts NAV and OBS GPS RINEX (including Hatanaka compressed OBS) data into
 [xarray.Dataset](http://xarray.pydata.org/en/stable/api.html#dataset)
 for easy use in analysis and plotting.
 This gives remarkable speed vs. legacy iterative methods, and allows for HPC / out-of-core operations on massive amounts of GNSS data.
@@ -24,14 +24,15 @@ where ease of cross-platform install and correctness are primary goals.
 
 ## Inputs
 
-* RINEX 3 or RINEX 2
+* RINEX 3.x or RINEX 2.x
   * NAV
   * OBS
 * Plain ASCII or seamlessly read compressed ASCII in:
   * `.gz` GZIP
   * `.Z` LZW
   * `.zip`
 * Hatanaka compressed RINEX (plain `.crx` or `.crx.gz` etc.)
+* Python `io.StringIO` text stream RINEX
 
 ## Output
 

diff --git a/georinex/base.py b/georinex/base.py
@@ -1,6 +1,7 @@
 from pathlib import Path
 import xarray
 from typing import Union, Tuple, Dict, Sequence
+from typing.io import TextIO
 from datetime import datetime
 import logging
 from .io import rinexinfo
@@ -14,7 +15,7 @@
 ENC = {'zlib': True, 'complevel': 1, 'fletcher32': True}
 
 
-def load(rinexfn: Path,
+def load(rinexfn: Union[TextIO, str, Path],
          out: Path = None,
          use: Sequence[str] = None,
          tlim: Tuple[datetime, datetime] = None,
@@ -23,13 +24,15 @@ def load(rinexfn: Path,
          verbose: bool = False,
          fast: bool = True) -> Union[xarray.Dataset, Dict[str, xarray.Dataset]]:
     """
-    Reads OBS, NAV in RINEX 2,3.
-    Plain ASCII text or compressed (including Hatanaka)
+    Reads OBS, NAV in RINEX 2.x and 3.x
+
+    Files / StringIO input may be plain ASCII text or compressed (including Hatanaka)
     """
     if verbose:
         logging.basicConfig(level=logging.INFO)
 
-    rinexfn = Path(rinexfn).expanduser()
+    if isinstance(rinexfn, (str, Path)):
+        rinexfn = Path(rinexfn).expanduser()
 # %% detect type of Rinex file
     rtype = rinextype(rinexfn)
 # %% determine if/where to write NetCDF4/HDF5 output
@@ -97,18 +100,21 @@ def batch_convert(path: Path, glob: str, out: Path,
             logging.error(f'{fn.name}: {e}')
 
 
-def rinexnav(fn: Path,
+def rinexnav(fn: Union[TextIO, str, Path],
              outfn: Path = None,
              use: Sequence[str] = None,
              group: str = 'NAV',
              tlim: Tuple[datetime, datetime] = None) -> xarray.Dataset:
     """ Read RINEX 2 or 3  NAV files"""
-    fn = Path(fn).expanduser()
-    if fn.suffix == '.nc':
-        try:
-            return xarray.open_dataset(fn, group=group)
-        except OSError as e:
-            raise LookupError(f'Group {group} not found in {fn}    {e}')
+
+    if isinstance(fn, (str, Path)):
+        fn = Path(fn).expanduser()
+
+        if fn.suffix == '.nc':
+            try:
+                return xarray.open_dataset(fn, group=group)
+            except OSError as e:
+                raise LookupError(f'Group {group} not found in {fn}    {e}')
 
     tlim = _tlim(tlim)
 
@@ -135,7 +141,7 @@ def rinexnav(fn: Path,
 # %% Observation File
 
 
-def rinexobs(fn: Path,
+def rinexobs(fn: Union[TextIO, str, Path],
              outfn: Path = None,
              use: Sequence[str] = None,
              group: str = 'OBS',
@@ -145,16 +151,17 @@ def rinexobs(fn: Path,
              verbose: bool = False,
              fast: bool = True) -> xarray.Dataset:
     """
-    Read RINEX 2,3 OBS files in ASCII or GZIP
+    Read RINEX 2.x and 3.x OBS files in ASCII or GZIP (or Hatanaka)
     """
 
-    fn = Path(fn).expanduser()
+    if isinstance(fn, (str, Path)):
+        fn = Path(fn).expanduser()
 # %% NetCDF4
-    if fn.suffix == '.nc':
-        try:
-            return xarray.open_dataset(fn, group=group)
-        except OSError as e:
-            raise LookupError(f'Group {group} not found in {fn}   {e}')
+        if fn.suffix == '.nc':
+            try:
+                return xarray.open_dataset(fn, group=group)
+            except OSError as e:
+                raise LookupError(f'Group {group} not found in {fn}   {e}')
 
     tlim = _tlim(tlim)
 # %% version selection

diff --git a/georinex/io.py b/georinex/io.py
@@ -17,51 +17,57 @@
 
 
 @contextmanager
-def opener(fn: Path, header: bool = False, verbose: bool = False) -> TextIO:
+def opener(fn: Union[TextIO, Path],
+           header: bool = False,
+           verbose: bool = False) -> TextIO:
     """provides file handle for regular ASCII or gzip files transparently"""
-    if fn.is_dir():
-        raise FileNotFoundError(f'{fn} is a directory; I need a file')
 
-    if verbose:
-        if fn.stat().st_size > 100e6:
-            print(f'opening {fn.stat().st_size/1e6} MByte {fn.name}')
+    if isinstance(fn, io.StringIO):
+        yield fn
+    else:
+        if fn.is_dir():
+            raise FileNotFoundError(f'{fn} is a directory; I need a file')
+
+        if verbose:
+            if fn.stat().st_size > 100e6:
+                print(f'opening {fn.stat().st_size/1e6} MByte {fn.name}')
 
-    if fn.suffixes == ['.crx', '.gz']:
-        if header:
+        if fn.suffixes == ['.crx', '.gz']:
+            if header:
+                with gzip.open(fn, 'rt') as f:
+                    yield f
+            else:
+                with gzip.open(fn, 'rt') as g:
+                    f = io.StringIO(_opencrx(g))
+                    yield f
+        elif fn.suffix == '.crx':
+            if header:
+                with fn.open('r') as f:
+                    yield f
+            else:
+                with fn.open('r') as g:
+                    f = io.StringIO(_opencrx(g))
+                    yield f
+        elif fn.suffix == '.gz':
             with gzip.open(fn, 'rt') as f:
                 yield f
+        elif fn.suffix == '.zip':
+            with zipfile.ZipFile(fn, 'r') as z:
+                flist = z.namelist()
+                for rinexfn in flist:
+                    with z.open(rinexfn, 'r') as bf:
+                        f = io.TextIOWrapper(bf, newline=None)
+                        yield f
+        elif fn.suffix == '.Z':
+            if unlzw is None:
+                raise ImportError('pip install unlzw')
+            with fn.open('rb') as zu:
+                with io.StringIO(unlzw.unlzw(zu.read()).decode('ascii')) as f:
+                    yield f
+
         else:
-            with gzip.open(fn, 'rt') as g:
-                f = io.StringIO(_opencrx(g))
-                yield f
-    elif fn.suffix == '.crx':
-        if header:
             with fn.open('r') as f:
                 yield f
-        else:
-            with fn.open('r') as g:
-                f = io.StringIO(_opencrx(g))
-                yield f
-    elif fn.suffix == '.gz':
-        with gzip.open(fn, 'rt') as f:
-            yield f
-    elif fn.suffix == '.zip':
-        with zipfile.ZipFile(fn, 'r') as z:
-            flist = z.namelist()
-            for rinexfn in flist:
-                with z.open(rinexfn, 'r') as bf:
-                    f = io.TextIOWrapper(bf, newline=None)
-                    yield f
-    elif fn.suffix == '.Z':
-        if unlzw is None:
-            raise ImportError('pip install unlzw')
-        with fn.open('rb') as zu:
-            with io.StringIO(unlzw.unlzw(zu.read()).decode('ascii')) as f:
-                yield f
-
-    else:
-        with fn.open('r') as f:
-            yield f
 
 
 def _opencrx(f: TextIO) -> str:
@@ -101,6 +107,8 @@ def rinexinfo(f: Union[Path, TextIO]) -> Dict[str, Any]:
         else:
             with opener(fn) as f:
                 return rinexinfo(f)
+    elif isinstance(f, io.StringIO):
+        f.seek(0)
 
     try:
         line = f.readline(80)  # don't choke on binary files

diff --git a/georinex/nav2.py b/georinex/nav2.py
@@ -1,19 +1,20 @@
 #!/usr/bin/env python
 from pathlib import Path
 from datetime import datetime
-from typing import Dict, Any, Sequence, Optional
+from typing import Dict, Union, Any, Sequence, Optional
 from typing.io import TextIO
 import xarray
 import numpy as np
 import logging
+import io
 from .io import opener, rinexinfo
 from .common import rinex_string_to_float
 #
 STARTCOL2 = 3  # column where numerical data starts for RINEX 2
 Nl = {'G': 7, 'R': 3, 'E': 7}   # number of additional SV lines
 
 
-def rinexnav2(fn: Path,
+def rinexnav2(fn: Union[TextIO, str, Path],
               tlim: Sequence[datetime] = None) -> xarray.Dataset:
     """
     Reads RINEX 2.x NAV files
@@ -23,7 +24,8 @@ def rinexnav2(fn: Path,
     http://gage14.upc.es/gLAB/HTML/GPS_Navigation_Rinex_v2.11.html
     ftp://igs.org/pub/data/format/rinex211.txt
     """
-    fn = Path(fn).expanduser()
+    if isinstance(fn, (str, Path)):
+        fn = Path(fn).expanduser()
 
     Lf = 19  # string length per field
 
@@ -133,9 +135,10 @@ def rinexnav2(fn: Path,
 
 # %% other attributes
     nav.attrs['version'] = header['version']
-    nav.attrs['filename'] = fn.name
     nav.attrs['svtype'] = [svtype]  # Use list for consistency with NAV3.
     nav.attrs['rinextype'] = 'nav'
+    if isinstance(fn, Path):
+        nav.attrs['filename'] = fn.name
 
     if 'ION ALPHA' in header and 'ION BETA' in header:
         alpha = header['ION ALPHA']
@@ -154,7 +157,12 @@ def navheader2(f: TextIO) -> Dict[str, Any]:
         fn = f
         with fn.open('r') as f:
             return navheader2(f)
-
+    elif isinstance(f, io.StringIO):
+        f.seek(0)
+    elif isinstance(f, io.TextIOWrapper):
+        pass
+    else:
+        raise TypeError(f'unknown filetype {type(f)}')
 # %%verify RINEX version, and that it's NAV
     hdr = rinexinfo(f)
     if int(hdr['version']) != 2:
@@ -200,7 +208,7 @@ def _skip(f: TextIO, Nl: int):
         pass
 
 
-def navtime2(fn: Path) -> xarray.DataArray:
+def navtime2(fn: Union[TextIO, Path]) -> xarray.DataArray:
     """
     read all times in RINEX 2 NAV file
     """
@@ -227,7 +235,9 @@ def navtime2(fn: Path) -> xarray.DataArray:
     times = np.unique(times)
 
     timedat = xarray.DataArray(times,
-                               dims=['time'],
-                               attrs={'filename': fn})
+                               dims=['time'])
+
+    if isinstance(fn, Path):
+        timedat.attrs['filename'] = fn.name
 
     return timedat
diff --git a/georinex/nav3.py b/georinex/nav3.py
@@ -4,19 +4,19 @@
 import logging
 import numpy as np
 import math
-from io import BytesIO
+import io
 from datetime import datetime
 from .io import opener, rinexinfo
 from .common import rinex_string_to_float
-from typing import Dict, List, Any, Sequence, Optional
+from typing import Dict, Union, List, Any, Sequence, Optional
 from typing.io import TextIO
 # constants
 STARTCOL3 = 4  # column where numerical data starts for RINEX 3
 Nl = {'C': 7, 'E': 7, 'G': 7, 'J': 7, 'R': 3, 'S': 3}   # number of additional SV lines
 Lf = 19  # string length per field
 
 
-def rinexnav3(fn: Path,
+def rinexnav3(fn: Union[TextIO, str, Path],
               use: Sequence[str] = None,
               tlim: Sequence[datetime] = None) -> xarray.Dataset:
     """
@@ -27,8 +27,8 @@ def rinexnav3(fn: Path,
 
     The "eof" stuff is over detection of files that may or may not have a trailing newline at EOF.
     """
-
-    fn = Path(fn).expanduser()
+    if isinstance(fn, (str, Path)):
+        fn = Path(fn).expanduser()
 
     svs = []
     raws = []
@@ -104,7 +104,8 @@ def rinexnav3(fn: Path,
         darr = np.empty((svi.size, len(cf)))
 
         for j, i in enumerate(svi):
-            darr[j, :] = np.genfromtxt(BytesIO(raws[i].encode('ascii')), delimiter=Lf)
+            darr[j, :] = np.genfromtxt(io.BytesIO(raws[i].encode('ascii')),
+                                       delimiter=Lf)
 
 # %% discard duplicated times
 
@@ -147,9 +148,10 @@ def rinexnav3(fn: Path,
                                                            corr['IRNB']))
 
     nav.attrs['version'] = header['version']
-    nav.attrs['filename'] = fn.name
     nav.attrs['svtype'] = svtypes
     nav.attrs['rinextype'] = 'nav'
+    if isinstance(fn, Path):
+        nav.attrs['filename'] = fn.name
 
     return nav
 
@@ -307,6 +309,12 @@ def navheader3(f: TextIO) -> Dict[str, Any]:
         fn = f
         with fn.open('r') as f:
             return navheader3(f)
+    elif isinstance(f, io.StringIO):
+        f.seek(0)
+    elif isinstance(f, io.TextIOWrapper):
+        pass
+    else:
+        raise TypeError(f'unsure of input data type {type(f)}')
 
     hdr = rinexinfo(f)
     assert int(hdr['version']) == 3, 'see rinexnav2() for RINEX 2.x files'
@@ -332,7 +340,7 @@ def navheader3(f: TextIO) -> Dict[str, Any]:
     return hdr
 
 
-def navtime3(fn: Path) -> xarray.DataArray:
+def navtime3(fn: Union[TextIO, Path]) -> xarray.DataArray:
     """
     return all times in RINEX file
     """
@@ -355,7 +363,9 @@ def navtime3(fn: Path) -> xarray.DataArray:
     times = np.unique(times)
 
     timedat = xarray.DataArray(times,
-                               dims=['time'],
-                               attrs={'filename': fn})
+                               dims=['time'])
+
+    if isinstance(fn, Path):
+        timedat.attrs['filename'] = fn.name
 
     return timedat