Fixed broken proxy requests URLs and csv file loading was skipping

the first two data lines

Signed-off-by: Alexis Jeandet <[email protected]>

spwc/amda/amda.py

@@ -24,14 +24,15 @@ def load_csv(filename: str):
     with urlopen(filename) as csv:
         line = csv.readline().decode()
         meta = {}
-        columns = []
         y = None
         while line[0] == '#':
             if ':' in line:
                 key, value = line[1:].split(':', 1)
                 meta[key.strip()] = value.strip()
             line = csv.readline().decode()
-        data = pds.read_csv(csv, comment='#', delim_whitespace=True).values.transpose()
+        columns = [col.strip() for col in meta['DATA_COLUMNS'].split(',')[:]]
+        with urlopen(filename) as f:
+            data = pds.read_csv(f, comment='#', delim_whitespace=True, header=None, names=columns).values.transpose()
         time, data = data[0], data[1:].transpose()
         if "PARAMETER_TABLE_MIN_VALUES[1]" in meta:
             min_v = np.array([float(v) for v in meta["PARAMETER_TABLE_MIN_VALUES[1]"].split(',')])
@@ -41,9 +42,7 @@ def load_csv(filename: str):
             min_v = np.array([float(v) for v in meta["PARAMETER_TABLE_MIN_VALUES[0]"].split(',')])
             max_v = np.array([float(v) for v in meta["PARAMETER_TABLE_MAX_VALUES[0]"].split(',')])
             y = (max_v + min_v) / 2.
-        if 'DATA_COLUMNS' in meta:
-            columns = [col.strip() for col in meta['DATA_COLUMNS'].split(',')[1:]]
-        return SpwcVariable(time=time, data=data, meta=meta, columns=columns, y=y)
+        return SpwcVariable(time=time, data=data, meta=meta, columns=columns[1:], y=y)
 
 
 def get_parameter_args(start_time: datetime, stop_time: datetime, product: str, **kwargs):
@@ -119,7 +118,7 @@ def get_token(self, **kwargs: dict) -> str:
         return self.METHODS["REST"].get_token
 
     def _dl_parameter(self, start_time: datetime, stop_time: datetime, parameter_id: str,
-                      method: str = "SOAP", **kwargs) -> Optional[SpwcVariable]:
+                      method: str = "REST", **kwargs) -> Optional[SpwcVariable]:
 
         start_time = start_time.timestamp()
         stop_time = stop_time.timestamp()
@@ -150,7 +149,7 @@ def get_data(self, product, start_time: datetime, stop_time: datetime):
         return self._dl_parameter(start_time=start_time, stop_time=stop_time, parameter_id=product)
 
     def get_parameter(self, start_time: datetime, stop_time: datetime, parameter_id: str,
-                      method: str = "SOAP", **kwargs) -> Optional[SpwcVariable]:
+                      method: str = "REST", **kwargs) -> Optional[SpwcVariable]:
         return self.get_data(product=parameter_id, start_time=start_time, stop_time=stop_time, **kwargs)
 
     def get_obs_data_tree(self, method="SOAP") -> dict:

spwc/amda/rest.py

@@ -1,4 +1,7 @@
 import requests
+import logging
+
+log = logging.getLogger(__name__)
 
 
 class AmdaRest:
@@ -12,13 +15,14 @@ def get_parameter(self, **kwargs: dict):
             base_url += key + "=" + str(val) + "&"
         for _ in [None] * 3:  # sometime fails with no reason...
             url = base_url + "token=" + self.get_token
+            log.debug(f"Send request on AMDA server {url}")
             r = requests.get(url)
             js = r.json()
             if 'success' in js and js['success'] is True and 'dataFileURLs' in js:
+                log.debug(f"success: {js['dataFileURLs']}")
                 return js['dataFileURLs']
             else:
-                print(url)
-                print(r.text)
+                log.debug(f"Failed: {r.text}")
         return None
 
     @property

spwc/proxy/__init__.py

@@ -9,9 +9,13 @@ def __init__(self):
         pass
 
     @staticmethod
-    def get(path:str, start_time:str, stop_time:str):
+    def get(path: str, start_time: str, stop_time: str):
         url = proxy_url.get()
-        resp=requests.get(f"{url}/get_data?path={path}&start_time={start_time}&stop_time={stop_time}")
+        resp = requests.get(f"{url}/get_data?", params={
+            'path': path,
+            'start_time': start_time,
+            'stop_time': stop_time
+        })
         if resp.status_code == 200:
             var = pickle.loads(resp.content)
             return var
@@ -31,4 +35,5 @@ def wrapped(*args, **kwargs):
                 return self.request.get(**self.arg_builder(**kwargs))
             else:
                 return func(*args, **kwargs)
+
         return wrapped