Merge pull request #172 from ayushanand18/fix-inconsistent-data

fix: fixes #171; brings more type checks for returned data

README.md

@@ -5,7 +5,7 @@
 [![docs](https://github.com/iobis/pyobis/actions/workflows/deploy-docs.yml/badge.svg)](https://iobis.github.ic/pyobis)
 [![tests](https://github.com/iobis/pyobis/actions/workflows/tests.yml/badge.svg)](https://github.com/iobis/pyobis/actions/workflows/tests.yml)
 
-Python client for the `OBIS API(https://api.obis.org/).
+Python client for the [OBIS API](https://api.obis.org/).
 
 [Source on GitHub at iobis/pyobis](https://github.com/iobis/pyobis)
 

pyobis/occurrences/occurrences.py

@@ -132,7 +132,7 @@ def execute(self, **kwargs):
                 )
                 outdf = pd.concat(
                     [
-                        outdf.infer_objects(),
+                        outdf if len(outdf) else None,
                         pd.DataFrame(res["results"]).infer_objects(),
                     ],
                     ignore_index=True,
@@ -159,7 +159,10 @@ def execute(self, **kwargs):
                 **kwargs,
             )
             outdf = pd.concat(
-                [outdf.infer_objects(), pd.DataFrame(res["results"]).infer_objects()],
+                [
+                    outdf if len(outdf) else None,
+                    pd.DataFrame(res["results"]).infer_objects(),
+                ],
                 ignore_index=True,
             )
             logger.info(f"Fetched {size} records.")
@@ -176,16 +179,31 @@ def execute(self, **kwargs):
                     on="id",
                     how="inner",
                 )
-                self.data = merged
-                return self.data
-            self.data = outdf
+                self.data = {"total": len(merged), "results": merged}
+                # set the data as [total, results] K-V pair
+                # but still return the DataFrame since changing this
+                # will impact existing usage, and be a breaking change
+                return self.data["results"]
+            self.data = {"total": len(outdf), "results": outdf}
+
+            # again for not MeasurementOrFacts results, (simple search queries)
+            # should also return the DataFrame directly for backward compatibility
+            return self.data["results"]
 
         return self.data
 
     def to_pandas(self):
         """
         Convert the results into a pandas DataFrame
         """
+        # if the data format of the query executed cannot be converted to a
+        # pandas.DataFrame which is true for other formats like .mvt or kml (not geojson)
+        # then we should be raising a not implemented rather relying around exceptions
+        # from pandas while converting
+        if self.__isKML:
+            raise NotImplementedError(
+                "to_pandas method is not yet available for these query types.",
+            )
         return pd.DataFrame(self.data["results"])
 
 
@@ -400,7 +418,7 @@ def grid(
         from pyobis import occurrences
 
         occurrences.grid(100, True) // returns in GeoJSON format
-        ococcurrences.grid(1000, False)   // returns in KML format
+        occurrences.grid(1000, False)   // returns in KML format
     """
     url = obis_baseurl + "occurrence/grid/" + str(precision)
     scientificname = handle_arrstr(scientificname)

pyobis/occurrences/test_occurrence.py

@@ -15,8 +15,10 @@ def test_occurrences_search():
     query = occurrences.search(scientificname="Mola mola", size=size)
     assert not query.data  # the data is none after query building but before executing
     query.execute()
-    assert size == len(query.data)
-    assert "Mola mola" == query.data.scientificName[0]
+    assert "dict" == query.data.__class__.__name__
+    assert 2 == len(query.data)
+    assert size == len(query.to_pandas())
+    assert "Mola mola" == query.to_pandas().scientificName[0]
 
 
 @pytest.mark.vcr()
@@ -32,7 +34,7 @@ def test_occurrence_search_mof():
     )
     assert not query.data
     query.execute()
-    assert "Abra alba" == query.data.scientificName[0]
+    assert "Abra alba" == query.to_pandas().scientificName[0]
     assert requests.get(query.api_url).status_code == 200
     assert requests.get(query.mapper_url).status_code == 200
 
@@ -83,6 +85,13 @@ def test_occurrences_grid():
     assert requests.get(query.api_url).status_code == 200
     assert not query.mapper_url
 
+    # check for KML formats that to_pandas function is not implemented
+    with pytest.raises(
+        NotImplementedError,
+        match="to_pandas method is not yet available for these query types.",
+    ):
+        query.to_pandas()
+
 
 @pytest.mark.vcr()
 def test_occurrences_getpoints():
@@ -128,6 +137,14 @@ def test_occurrences_tile():
     query = occurrences.tile(x=1.77, y=52.26, z=0.5, mvt=1, scientificname="Mola mola")
     query.execute()
     assert requests.get(query.api_url).status_code == 200
+
+    # check for MVT formats that to_pandas function is not implemented
+    with pytest.raises(
+        NotImplementedError,
+        match="to_pandas method is not yet available for these query types.",
+    ):
+        query.to_pandas()
+
     query = occurrences.tile(x=1.77, y=52.26, z=0.5, mvt=0, scientificname="Mola mola")
     query.execute()
     assert requests.get(query.api_url).status_code == 200