From 1a85c13bc21fb0b483a35546ce923fe220914b3d Mon Sep 17 00:00:00 2001 From: "Michael Hirsch, Ph.D" Date: Mon, 31 Dec 2018 23:53:25 -0500 Subject: [PATCH] Squashed commit of the following: commit 7e40d00183cc9f8cb9610f558ff56c09667df204 Author: Michael Hirsch, Ph.D Date: Mon Dec 31 23:53:16 2018 -0500 vers commit 15b469782d274e97c1bdba88acb76b3bf232390b Author: Michael Hirsch, Ph.D Date: Mon Dec 31 23:52:26 2018 -0500 stringio API commit 3fac0a5e100923fe67e37feed90dadbcd48eec38 Author: Michael Hirsch, Ph.D Date: Mon Dec 31 23:46:39 2018 -0500 check type stringio commit e57d5d6c8f14d2f7768d0c37c5113888fc7a03fd Author: Michael Hirsch, Ph.D Date: Mon Dec 31 23:43:32 2018 -0500 gettime stringio commit 7f3f026a4fc15f193527ad79a22f2a02de1e4ce3 Author: Michael Hirsch, Ph.D Date: Mon Dec 31 23:20:29 2018 -0500 OBS3 stringio commit 731c541bac29df634394747b6091a6459fae4dce Author: Michael Hirsch, Ph.D Date: Mon Dec 31 23:12:33 2018 -0500 OBS2 stringio commit b8687aa669bcbd83716bbab482a4b18c3a2c7eaf Author: Michael Hirsch, Ph.D Date: Mon Dec 31 22:50:27 2018 -0500 NAV2 stringio commit 01e78e96eb05a3d6f48cf6e03dcee7942532ee47 Author: Michael Hirsch, Ph.D Date: Mon Dec 31 22:45:49 2018 -0500 NAV3 stringio --- README.md | 5 ++- georinex/base.py | 45 +++++++++++--------- georinex/io.py | 82 ++++++++++++++++++++---------------- georinex/nav2.py | 26 ++++++++---- georinex/nav3.py | 30 ++++++++----- georinex/obs2.py | 48 ++++++++++++++++----- georinex/obs3.py | 34 +++++++++------ georinex/utils.py | 18 +++++--- setup.cfg | 7 ++-- tests/test_stringio.py | 95 ++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 284 insertions(+), 106 deletions(-) create mode 100644 tests/test_stringio.py diff --git a/README.md b/README.md index d01ea2a..7733a5f 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ # GeoRinex RINEX 3 and RINEX 2 reader and batch conversion to NetCDF4 / HDF5 in Python or Matlab. -Batch converts NAV and OBS GPS RINEX data into +Batch converts NAV and OBS GPS RINEX (including Hatanaka compressed OBS) data into [xarray.Dataset](http://xarray.pydata.org/en/stable/api.html#dataset) for easy use in analysis and plotting. This gives remarkable speed vs. legacy iterative methods, and allows for HPC / out-of-core operations on massive amounts of GNSS data. @@ -24,7 +24,7 @@ where ease of cross-platform install and correctness are primary goals. ## Inputs -* RINEX 3 or RINEX 2 +* RINEX 3.x or RINEX 2.x * NAV * OBS * Plain ASCII or seamlessly read compressed ASCII in: @@ -32,6 +32,7 @@ where ease of cross-platform install and correctness are primary goals. * `.Z` LZW * `.zip` * Hatanaka compressed RINEX (plain `.crx` or `.crx.gz` etc.) +* Python `io.StringIO` text stream RINEX ## Output diff --git a/georinex/base.py b/georinex/base.py index 5904d3e..db0ccf0 100644 --- a/georinex/base.py +++ b/georinex/base.py @@ -1,6 +1,7 @@ from pathlib import Path import xarray from typing import Union, Tuple, Dict, Sequence +from typing.io import TextIO from datetime import datetime import logging from .io import rinexinfo @@ -14,7 +15,7 @@ ENC = {'zlib': True, 'complevel': 1, 'fletcher32': True} -def load(rinexfn: Path, +def load(rinexfn: Union[TextIO, str, Path], out: Path = None, use: Sequence[str] = None, tlim: Tuple[datetime, datetime] = None, @@ -23,13 +24,15 @@ def load(rinexfn: Path, verbose: bool = False, fast: bool = True) -> Union[xarray.Dataset, Dict[str, xarray.Dataset]]: """ - Reads OBS, NAV in RINEX 2,3. - Plain ASCII text or compressed (including Hatanaka) + Reads OBS, NAV in RINEX 2.x and 3.x + + Files / StringIO input may be plain ASCII text or compressed (including Hatanaka) """ if verbose: logging.basicConfig(level=logging.INFO) - rinexfn = Path(rinexfn).expanduser() + if isinstance(rinexfn, (str, Path)): + rinexfn = Path(rinexfn).expanduser() # %% detect type of Rinex file rtype = rinextype(rinexfn) # %% determine if/where to write NetCDF4/HDF5 output @@ -97,18 +100,21 @@ def batch_convert(path: Path, glob: str, out: Path, logging.error(f'{fn.name}: {e}') -def rinexnav(fn: Path, +def rinexnav(fn: Union[TextIO, str, Path], outfn: Path = None, use: Sequence[str] = None, group: str = 'NAV', tlim: Tuple[datetime, datetime] = None) -> xarray.Dataset: """ Read RINEX 2 or 3 NAV files""" - fn = Path(fn).expanduser() - if fn.suffix == '.nc': - try: - return xarray.open_dataset(fn, group=group) - except OSError as e: - raise LookupError(f'Group {group} not found in {fn} {e}') + + if isinstance(fn, (str, Path)): + fn = Path(fn).expanduser() + + if fn.suffix == '.nc': + try: + return xarray.open_dataset(fn, group=group) + except OSError as e: + raise LookupError(f'Group {group} not found in {fn} {e}') tlim = _tlim(tlim) @@ -135,7 +141,7 @@ def rinexnav(fn: Path, # %% Observation File -def rinexobs(fn: Path, +def rinexobs(fn: Union[TextIO, str, Path], outfn: Path = None, use: Sequence[str] = None, group: str = 'OBS', @@ -145,16 +151,17 @@ def rinexobs(fn: Path, verbose: bool = False, fast: bool = True) -> xarray.Dataset: """ - Read RINEX 2,3 OBS files in ASCII or GZIP + Read RINEX 2.x and 3.x OBS files in ASCII or GZIP (or Hatanaka) """ - fn = Path(fn).expanduser() + if isinstance(fn, (str, Path)): + fn = Path(fn).expanduser() # %% NetCDF4 - if fn.suffix == '.nc': - try: - return xarray.open_dataset(fn, group=group) - except OSError as e: - raise LookupError(f'Group {group} not found in {fn} {e}') + if fn.suffix == '.nc': + try: + return xarray.open_dataset(fn, group=group) + except OSError as e: + raise LookupError(f'Group {group} not found in {fn} {e}') tlim = _tlim(tlim) # %% version selection diff --git a/georinex/io.py b/georinex/io.py index d387286..1c8c80b 100644 --- a/georinex/io.py +++ b/georinex/io.py @@ -17,51 +17,57 @@ @contextmanager -def opener(fn: Path, header: bool = False, verbose: bool = False) -> TextIO: +def opener(fn: Union[TextIO, Path], + header: bool = False, + verbose: bool = False) -> TextIO: """provides file handle for regular ASCII or gzip files transparently""" - if fn.is_dir(): - raise FileNotFoundError(f'{fn} is a directory; I need a file') - if verbose: - if fn.stat().st_size > 100e6: - print(f'opening {fn.stat().st_size/1e6} MByte {fn.name}') + if isinstance(fn, io.StringIO): + yield fn + else: + if fn.is_dir(): + raise FileNotFoundError(f'{fn} is a directory; I need a file') + + if verbose: + if fn.stat().st_size > 100e6: + print(f'opening {fn.stat().st_size/1e6} MByte {fn.name}') - if fn.suffixes == ['.crx', '.gz']: - if header: + if fn.suffixes == ['.crx', '.gz']: + if header: + with gzip.open(fn, 'rt') as f: + yield f + else: + with gzip.open(fn, 'rt') as g: + f = io.StringIO(_opencrx(g)) + yield f + elif fn.suffix == '.crx': + if header: + with fn.open('r') as f: + yield f + else: + with fn.open('r') as g: + f = io.StringIO(_opencrx(g)) + yield f + elif fn.suffix == '.gz': with gzip.open(fn, 'rt') as f: yield f + elif fn.suffix == '.zip': + with zipfile.ZipFile(fn, 'r') as z: + flist = z.namelist() + for rinexfn in flist: + with z.open(rinexfn, 'r') as bf: + f = io.TextIOWrapper(bf, newline=None) + yield f + elif fn.suffix == '.Z': + if unlzw is None: + raise ImportError('pip install unlzw') + with fn.open('rb') as zu: + with io.StringIO(unlzw.unlzw(zu.read()).decode('ascii')) as f: + yield f + else: - with gzip.open(fn, 'rt') as g: - f = io.StringIO(_opencrx(g)) - yield f - elif fn.suffix == '.crx': - if header: with fn.open('r') as f: yield f - else: - with fn.open('r') as g: - f = io.StringIO(_opencrx(g)) - yield f - elif fn.suffix == '.gz': - with gzip.open(fn, 'rt') as f: - yield f - elif fn.suffix == '.zip': - with zipfile.ZipFile(fn, 'r') as z: - flist = z.namelist() - for rinexfn in flist: - with z.open(rinexfn, 'r') as bf: - f = io.TextIOWrapper(bf, newline=None) - yield f - elif fn.suffix == '.Z': - if unlzw is None: - raise ImportError('pip install unlzw') - with fn.open('rb') as zu: - with io.StringIO(unlzw.unlzw(zu.read()).decode('ascii')) as f: - yield f - - else: - with fn.open('r') as f: - yield f def _opencrx(f: TextIO) -> str: @@ -101,6 +107,8 @@ def rinexinfo(f: Union[Path, TextIO]) -> Dict[str, Any]: else: with opener(fn) as f: return rinexinfo(f) + elif isinstance(f, io.StringIO): + f.seek(0) try: line = f.readline(80) # don't choke on binary files diff --git a/georinex/nav2.py b/georinex/nav2.py index 798de32..bc3c5a8 100644 --- a/georinex/nav2.py +++ b/georinex/nav2.py @@ -1,11 +1,12 @@ #!/usr/bin/env python from pathlib import Path from datetime import datetime -from typing import Dict, Any, Sequence, Optional +from typing import Dict, Union, Any, Sequence, Optional from typing.io import TextIO import xarray import numpy as np import logging +import io from .io import opener, rinexinfo from .common import rinex_string_to_float # @@ -13,7 +14,7 @@ Nl = {'G': 7, 'R': 3, 'E': 7} # number of additional SV lines -def rinexnav2(fn: Path, +def rinexnav2(fn: Union[TextIO, str, Path], tlim: Sequence[datetime] = None) -> xarray.Dataset: """ Reads RINEX 2.x NAV files @@ -23,7 +24,8 @@ def rinexnav2(fn: Path, http://gage14.upc.es/gLAB/HTML/GPS_Navigation_Rinex_v2.11.html ftp://igs.org/pub/data/format/rinex211.txt """ - fn = Path(fn).expanduser() + if isinstance(fn, (str, Path)): + fn = Path(fn).expanduser() Lf = 19 # string length per field @@ -133,9 +135,10 @@ def rinexnav2(fn: Path, # %% other attributes nav.attrs['version'] = header['version'] - nav.attrs['filename'] = fn.name nav.attrs['svtype'] = [svtype] # Use list for consistency with NAV3. nav.attrs['rinextype'] = 'nav' + if isinstance(fn, Path): + nav.attrs['filename'] = fn.name if 'ION ALPHA' in header and 'ION BETA' in header: alpha = header['ION ALPHA'] @@ -154,7 +157,12 @@ def navheader2(f: TextIO) -> Dict[str, Any]: fn = f with fn.open('r') as f: return navheader2(f) - + elif isinstance(f, io.StringIO): + f.seek(0) + elif isinstance(f, io.TextIOWrapper): + pass + else: + raise TypeError(f'unknown filetype {type(f)}') # %%verify RINEX version, and that it's NAV hdr = rinexinfo(f) if int(hdr['version']) != 2: @@ -200,7 +208,7 @@ def _skip(f: TextIO, Nl: int): pass -def navtime2(fn: Path) -> xarray.DataArray: +def navtime2(fn: Union[TextIO, Path]) -> xarray.DataArray: """ read all times in RINEX 2 NAV file """ @@ -227,7 +235,9 @@ def navtime2(fn: Path) -> xarray.DataArray: times = np.unique(times) timedat = xarray.DataArray(times, - dims=['time'], - attrs={'filename': fn}) + dims=['time']) + + if isinstance(fn, Path): + timedat.attrs['filename'] = fn.name return timedat diff --git a/georinex/nav3.py b/georinex/nav3.py index 81fd5fc..5654f1b 100644 --- a/georinex/nav3.py +++ b/georinex/nav3.py @@ -4,11 +4,11 @@ import logging import numpy as np import math -from io import BytesIO +import io from datetime import datetime from .io import opener, rinexinfo from .common import rinex_string_to_float -from typing import Dict, List, Any, Sequence, Optional +from typing import Dict, Union, List, Any, Sequence, Optional from typing.io import TextIO # constants STARTCOL3 = 4 # column where numerical data starts for RINEX 3 @@ -16,7 +16,7 @@ Lf = 19 # string length per field -def rinexnav3(fn: Path, +def rinexnav3(fn: Union[TextIO, str, Path], use: Sequence[str] = None, tlim: Sequence[datetime] = None) -> xarray.Dataset: """ @@ -27,8 +27,8 @@ def rinexnav3(fn: Path, The "eof" stuff is over detection of files that may or may not have a trailing newline at EOF. """ - - fn = Path(fn).expanduser() + if isinstance(fn, (str, Path)): + fn = Path(fn).expanduser() svs = [] raws = [] @@ -104,7 +104,8 @@ def rinexnav3(fn: Path, darr = np.empty((svi.size, len(cf))) for j, i in enumerate(svi): - darr[j, :] = np.genfromtxt(BytesIO(raws[i].encode('ascii')), delimiter=Lf) + darr[j, :] = np.genfromtxt(io.BytesIO(raws[i].encode('ascii')), + delimiter=Lf) # %% discard duplicated times @@ -147,9 +148,10 @@ def rinexnav3(fn: Path, corr['IRNB'])) nav.attrs['version'] = header['version'] - nav.attrs['filename'] = fn.name nav.attrs['svtype'] = svtypes nav.attrs['rinextype'] = 'nav' + if isinstance(fn, Path): + nav.attrs['filename'] = fn.name return nav @@ -307,6 +309,12 @@ def navheader3(f: TextIO) -> Dict[str, Any]: fn = f with fn.open('r') as f: return navheader3(f) + elif isinstance(f, io.StringIO): + f.seek(0) + elif isinstance(f, io.TextIOWrapper): + pass + else: + raise TypeError(f'unsure of input data type {type(f)}') hdr = rinexinfo(f) assert int(hdr['version']) == 3, 'see rinexnav2() for RINEX 2.x files' @@ -332,7 +340,7 @@ def navheader3(f: TextIO) -> Dict[str, Any]: return hdr -def navtime3(fn: Path) -> xarray.DataArray: +def navtime3(fn: Union[TextIO, Path]) -> xarray.DataArray: """ return all times in RINEX file """ @@ -355,7 +363,9 @@ def navtime3(fn: Path) -> xarray.DataArray: times = np.unique(times) timedat = xarray.DataArray(times, - dims=['time'], - attrs={'filename': fn}) + dims=['time']) + + if isinstance(fn, Path): + timedat.attrs['filename'] = fn.name return timedat diff --git a/georinex/obs2.py b/georinex/obs2.py index 2381c7b..97cfcf7 100644 --- a/georinex/obs2.py +++ b/georinex/obs2.py @@ -2,10 +2,11 @@ from pathlib import Path import numpy as np import logging +import io from math import ceil from datetime import datetime import xarray -from typing import List, Any, Dict, Tuple, Sequence, Optional +from typing import List, Union, Any, Dict, Tuple, Sequence, Optional from typing.io import TextIO try: from pymap3d import ecef2geodetic @@ -46,7 +47,7 @@ def rinexobs2(fn: Path, return obs -def rinexsystem2(fn: Path, +def rinexsystem2(fn: Union[TextIO, Path], system: str, tlim: Tuple[datetime, datetime] = None, useindicators: bool = False, @@ -88,7 +89,17 @@ def rinexsystem2(fn: Path, * RINEX OBS2 files have at least one 80-byte line per time: Nsvmin* ceil(Nobs / 5) """ assert isinstance(Nextra, int) - Nt = ceil(fn.stat().st_size / 80 / (Nsvmin * Nextra)) + + if isinstance(fn, Path): + filesize = fn.stat().st_size + elif isinstance(fn, io.StringIO): + fn.seek(0, io.SEEK_END) + filesize = fn.tell() + fn.seek(0, io.SEEK_SET) + else: + raise TypeError(f'Unknown input data file type {type(fn)}') + + Nt = ceil(filesize / 80 / (Nsvmin * Nextra)) else: # strict preallocation by double-reading file, OK for < 100 MB files times = obstime2(fn, verbose=verbose) # < 10 ms for 24 hour 15 second cadence if times is None: @@ -249,12 +260,13 @@ def rinexsystem2(fn: Path, obs = obs.dropna(dim='sv', how='all') obs = obs.dropna(dim='time', how='all') # when tlim specified # %% attributes - obs.attrs['filename'] = fn.name obs.attrs['version'] = hdr['version'] obs.attrs['rinextype'] = 'obs' obs.attrs['toffset'] = toffset obs.attrs['fast_processing'] = int(fast) # bool is not allowed in NetCDF4 obs.attrs['time_system'] = determine_time_system(hdr) + if isinstance(fn, Path): + obs.attrs['filename'] = fn.name try: obs.attrs['position'] = hdr['position'] @@ -273,6 +285,12 @@ def obsheader2(f: TextIO, fn = f with opener(fn, header=True) as f: return obsheader2(f, useindicators, meas) + elif isinstance(f, io.StringIO): + f.seek(0) + elif isinstance(f, io.TextIOWrapper): + pass + else: + raise TypeError(f'Unknown input filetype {type(f)}') # %% selection if isinstance(meas, str): @@ -386,7 +404,8 @@ def _getSVlist(ln: str, N: int, return sv -def obstime2(fn: Path, verbose: bool = False) -> xarray.DataArray: +def obstime2(fn: Union[TextIO, Path], + verbose: bool = False) -> xarray.DataArray: """ read all times in RINEX2 OBS file """ @@ -409,8 +428,10 @@ def obstime2(fn: Path, verbose: bool = False) -> xarray.DataArray: timedat = xarray.DataArray(times, dims=['time'], - attrs={'filename': fn.name, - 'interval': hdr['interval']}) + attrs={'interval': hdr['interval']}) + + if isinstance(fn, Path): + timedat.attrs['filename'] = fn.name return timedat @@ -493,12 +514,19 @@ def _skip_header(f: TextIO): break -def _fast_alloc(fn: Path, Nl_sv: int) -> Optional[int]: +def _fast_alloc(fn: Union[TextIO, Path], Nl_sv: int) -> Optional[int]: """ prescan first N lines of file to see if it truncates to less than 80 bytes - Picking N: N > Nobs+4 or so. 100 seemed a good start. + + Picking N: N > Nobs+4 or so. + 100 seemed a good start. """ - assert fn.is_file(), 'need freshly opend file' + if isinstance(fn, Path): + assert fn.is_file(), 'need freshly opend file' + elif isinstance(fn, io.StringIO): + fn.seek(0) + else: + raise TypeError(f'Unknown filetype {type(fn)}') with opener(fn) as f: _skip_header(f) diff --git a/georinex/obs3.py b/georinex/obs3.py index 745e50e..9bd86ba 100644 --- a/georinex/obs3.py +++ b/georinex/obs3.py @@ -3,9 +3,9 @@ import numpy as np import logging from datetime import datetime -from io import BytesIO +import io import xarray -from typing import Dict, List, Tuple, Any, Sequence +from typing import Dict, Union, List, Tuple, Any, Sequence from typing.io import TextIO try: from pymap3d import ecef2geodetic @@ -20,7 +20,7 @@ BEIDOU = 0 -def rinexobs3(fn: Path, +def rinexobs3(fn: Union[TextIO, str, Path], use: Sequence[str] = None, tlim: Tuple[datetime, datetime] = None, useindicators: bool = False, @@ -57,7 +57,7 @@ def rinexobs3(fn: Path, break try: - time = _timeobs(ln, fn) + time = _timeobs(ln) except ValueError: # garbage between header and RINEX data logging.error(f'garbage detected in {fn}, trying to parse at next time step') continue @@ -87,10 +87,11 @@ def rinexobs3(fn: Path, # %% patch SV names in case of "G 7" => "G07" data = data.assign_coords(sv=[s.replace(' ', '0') for s in data.sv.values.tolist()]) # %% other attributes - data.attrs['filename'] = fn.name data.attrs['version'] = hdr['version'] data.attrs['rinextype'] = 'obs' data.attrs['time_system'] = determine_time_system(hdr) + if isinstance(fn, Path): + data.attrs['filename'] = fn.name try: data.attrs['position'] = hdr['position'] @@ -104,12 +105,12 @@ def rinexobs3(fn: Path, return data -def _timeobs(ln: str, fn: Path) -> datetime: +def _timeobs(ln: str) -> datetime: """ convert time from RINEX 3 OBS text to datetime """ if not ln.startswith('>'): # pg. A13 - raise ValueError(f'RINEX 3 line beginning > is not present in {fn}') + raise ValueError(f'RINEX 3 line beginning > is not present') return datetime(int(ln[2:6]), int(ln[7:9]), int(ln[10:12]), hour=int(ln[13:15]), minute=int(ln[16:18]), @@ -117,7 +118,8 @@ def _timeobs(ln: str, fn: Path) -> datetime: microsecond=int(float(ln[19:29]) % 1 * 1000000)) -def obstime3(fn: Path, verbose: bool = False) -> xarray.DataArray: +def obstime3(fn: Union[TextIO, Path], + verbose: bool = False) -> xarray.DataArray: """ return all times in RINEX file """ @@ -127,15 +129,17 @@ def obstime3(fn: Path, verbose: bool = False) -> xarray.DataArray: with opener(fn, verbose=verbose) as f: for ln in f: if ln.startswith('>'): - times.append(_timeobs(ln, fn)) + times.append(_timeobs(ln)) if not times: return None timedat = xarray.DataArray(times, dims=['time'], - attrs={'filename': fn, - 'interval': hdr['interval']}) + attrs={'interval': hdr['interval']}) + + if isinstance(fn, Path): + timedat.attrs['filename'] = fn.name return timedat @@ -149,7 +153,7 @@ def _epoch(data: xarray.Dataset, raw: str, """ block processing of each epoch (time step) """ - darr = np.atleast_2d(np.genfromtxt(BytesIO(raw.encode('ascii')), + darr = np.atleast_2d(np.genfromtxt(io.BytesIO(raw.encode('ascii')), delimiter=(14, 1, 1) * hdr['Fmax'])) # %% assign data for each time step for sk in hdr['fields']: # for each satellite system type (G,R,S, etc.) @@ -217,6 +221,12 @@ def obsheader3(f: TextIO, fn = f with opener(fn, header=True) as f: return obsheader3(f) + elif isinstance(f, io.StringIO): + f.seek(0) + elif isinstance(f, io.TextIOWrapper): + pass + else: + raise TypeError(f'Unknown input filetype {type(f)}') # %% first line ln = f.readline() hdr = {'version': float(ln[:9]), # yes :9 diff --git a/georinex/utils.py b/georinex/utils.py index 2c90972..9c8e37d 100644 --- a/georinex/utils.py +++ b/georinex/utils.py @@ -2,6 +2,9 @@ from typing import Tuple, Dict, Any, Optional, Sequence, List from datetime import datetime from dateutil.parser import parse +from typing import Union +from typing.io import TextIO +import io import xarray import pandas from .io import rinexinfo @@ -27,14 +30,15 @@ def globber(path: Path, glob: Sequence[str]) -> List[Path]: return flist -def gettime(fn: Path) -> xarray.DataArray: +def gettime(fn: Union[TextIO, str, Path]) -> xarray.DataArray: """ get times in RINEX 2/3 file Note: in header, * TIME OF FIRST OBS is mandatory * TIME OF LAST OBS is optional """ - fn = Path(fn).expanduser() + if isinstance(fn, (str, Path)): + fn = Path(fn).expanduser() info = rinexinfo(fn) assert int(info['version']) in (2, 3) @@ -95,15 +99,19 @@ def getlocations(flist: Sequence[Path]) -> pandas.DataFrame: return locs -def rinextype(fn: Path) -> str: +def rinextype(fn: Union[TextIO, Path]) -> str: """ determine if input file is NetCDF, OBS or NAV """ - if fn.suffix.endswith('.nc'): + if isinstance(fn, Path) and fn.suffix.endswith('.nc'): return 'nc' else: - return rinexinfo(fn)['rinextype'] + info = rinexinfo(fn)['rinextype'] + if isinstance(fn, io.StringIO): + fn.seek(0) + + return info def rinexheader(fn: Path) -> Dict[str, Any]: diff --git a/setup.cfg b/setup.cfg index 9f4da47..d282550 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = georinex -version = 1.6.9 +version = 1.7.0 author = Michael Hirsch, Ph.D. author_email = scivision@users.noreply.github.com description = Python RINEX 2/3 NAV/OBS reader with speed and simplicity. @@ -10,7 +10,7 @@ keywords = HDF5 NetCDF classifiers = - Development Status :: 4 - Beta + Development Status :: 5 - Production/Stable Environment :: Console Intended Audience :: Science/Research Operating System :: OS Independent @@ -60,11 +60,12 @@ io = netcdf4 unlzw psutil - + [tool:pytest] filterwarnings = ignore::DeprecationWarning + [flake8] max-line-length = 132 exclude = .git,__pycache__,.eggs/,doc/,docs/,build/,dist/,archive/ diff --git a/tests/test_stringio.py b/tests/test_stringio.py new file mode 100644 index 0000000..d649cdd --- /dev/null +++ b/tests/test_stringio.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +import pytest +from pytest import approx +from pathlib import Path +import georinex as gr +import io +from datetime import datetime + +R = Path(__file__).parent / 'data' + + +def test_nav3(): + fn = R / 'minimal3.10n' + with fn.open('r') as f: + txt = f.read() + + with io.StringIO(txt) as f: + rtype = gr.rinextype(f) + assert rtype == 'nav' + + times = gr.gettime(f).values.astype('datetime64[us]').astype(datetime).item() + nav = gr.load(f) + + nav3 = gr.rinexnav3(f) + + assert times == datetime(2010, 10, 18, 0, 1, 4) + assert nav.equals(nav3), 'NAV3 StringIO failure' + assert nav.equals(gr.load(fn)), 'NAV3 StringIO failure' + + +def test_nav2(): + fn = R / 'minimal.10n' + with fn.open('r') as f: + txt = f.read() + + with io.StringIO(txt) as f: + rtype = gr.rinextype(f) + assert rtype == 'nav' + + times = gr.gettime(f).values.astype('datetime64[us]').astype(datetime).item() + nav = gr.load(f) + + nav2 = gr.rinexnav2(f) + + assert times == datetime(1999, 9, 2, 19) + assert nav.equals(nav2), 'NAV2 StringIO failure' + assert nav.equals(gr.load(fn)), 'NAV2 StringIO failure' + + +def test_obs2(): + fn = R / 'minimal.10o' + with fn.open('r') as f: + txt = f.read() + + with io.StringIO(txt) as f: + rtype = gr.rinextype(f) + assert rtype == 'obs' + + times = gr.gettime(f).values.astype('datetime64[us]').astype(datetime).item() + obs = gr.load(f) + + obs2 = gr.rinexobs2(f) + + assert times == datetime(2010, 3, 5, 0, 0, 30) + assert obs.equals(obs2), 'OBS2 StringIO failure' + assert obs.equals(gr.load(fn)), 'OBS2 StringIO failure' + + +def test_obs3(): + fn = R / 'minimal3.10o' + with fn.open('r') as f: + txt = f.read() + + with io.StringIO(txt) as f: + rtype = gr.rinextype(f) + assert rtype == 'obs' + + times = gr.gettime(f).values.astype('datetime64[us]').astype(datetime).item() + obs = gr.load(f) + + obs3 = gr.rinexobs3(f) + + assert times == datetime(2010, 3, 5, 0, 0, 30) + assert obs.equals(obs3), 'OBS3 StringIO failure' + assert obs.equals(gr.load(fn)), 'OBS3 StringIO failure' + + +def test_locs(): + locs = gr.getlocations(R / 'demo.10o') + + assert locs.loc['demo.10o'].values == approx([41.3887, 2.112, 30]) + + +if __name__ == '__main__': + pytest.main(['-x', __file__])