Skip to content

Commit

Permalink
Fix filepaths for windows
Browse files Browse the repository at this point in the history
  • Loading branch information
AWehrhahn committed Oct 11, 2021
1 parent 0e36d69 commit 0ff06ad
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 25 deletions.
16 changes: 14 additions & 2 deletions src/pysme/atmosphere/savfile.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import os
from os.path import basename
from tempfile import NamedTemporaryFile

Expand Down Expand Up @@ -147,9 +148,20 @@ def __new__(cls, filename, source=None, lfs=None):
# Store in cache
cls._cache = self
# And also replace the IDL file with a numpy file in the cache
# We have to use a try except block, as this will crash with
# permissions denied on windows, when trying to copy an open file
# here the temporary file
# Therefore we close the file, after copying and then delete it manually
if lfs is not None:
with NamedTemporaryFile() as named:
self.save(named)
try:
with NamedTemporaryFile(delete=False) as named:
self.save(named)
named.flush()
lfs.move_to_cache(named.name, key=lfs.get_url(self.source))
finally:
try:
os.remove(named.name)
except:
pass

return self
54 changes: 32 additions & 22 deletions src/pysme/large_file_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import json
import logging
import os
from os.path import basename, join
from os.path import basename
from pathlib import Path
from tempfile import NamedTemporaryFile

Expand Down Expand Up @@ -102,24 +102,34 @@ def get(self, key):
def _unpack(self, fname, key, url):
logger.debug("Unpacking data file %s", key)

with gzip.open(fname, "rb") as f_in:
with NamedTemporaryFile("wb") as f_out:
with tqdm(
# total=f_in.size,
desc="Unpack",
unit="B",
unit_scale=True,
unit_divisor=1024,
) as t:
fobj = CallbackIOWrapper(t.update, f_in, "read")
while True:
chunk = fobj.read(1024)
if not chunk:
break
f_out.write(chunk)
f_out.flush()
t.reset()
import_file_to_cache(url, f_out.name, pkgname="pysme")
# We have to use a try except block, as this will crash with
# permissions denied on windows, when trying to copy an open file
# here the temporary file
# Therefore we close the file, after copying and then delete it manually
try:
with gzip.open(fname, "rb") as f_in:
with NamedTemporaryFile("wb", delete=False) as f_out:
with tqdm(
# total=f_in.size,
desc="Unpack",
unit="B",
unit_scale=True,
unit_divisor=1024,
) as t:
fobj = CallbackIOWrapper(t.update, f_in, "read")
while True:
chunk = fobj.read(1024)
if not chunk:
break
f_out.write(chunk)
f_out.flush()
t.reset()
import_file_to_cache(url, f_out.name, pkgname="pysme")
finally:
try:
os.remove(f_out.name)
except:
pass

def get_url(self, key):
key = str(key)
Expand All @@ -132,13 +142,13 @@ def get_url(self, key):
f"File {key} does not exist and is not tracked by the Large File system"
)
else:
return str(key)
return Path(key).as_uri()
else:
return str(self.current / key)
return (self.current / key).as_uri()

# Otherwise get it from the cache or online if necessary
newest = self.pointers[key]
url = join(self.server, newest)
url = self.server + "/" + newest
return url

def clean_cache(self):
Expand Down
2 changes: 1 addition & 1 deletion src/pysme/libtools.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,6 @@ def load_library(libfile=None):


def get_full_datadir():
localdir = realpath(dirname(__file__))
localdir = dirname(__file__)
datadir = join(localdir, "share/libsme/")
return datadir
2 changes: 2 additions & 0 deletions src/pysme/sme_synth.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
""" Wrapper for sme_synth.so C library """
import logging
import os
from os.path import normpath

import numpy as np

Expand Down Expand Up @@ -181,6 +182,7 @@ def SetLibraryPath(self, libpath=None):
"""Set the path to the library"""
if libpath is None:
libpath = get_full_datadir()
libpath = normpath(libpath) + os.sep
self.lib.SetLibraryPath(libpath, type="string", state=self.state)

def InputWaveRange(self, wfirst, wlast):
Expand Down

0 comments on commit 0ff06ad

Please sign in to comment.