update test script

- update version - change metric comparison tolerance to 1e-2
azavea · Aug 23, 2023 · 02f2806 · 02f2806
1 parent b766667
commit 02f2806
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 7 deletions.
diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/README.md b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/README.md
@@ -37,6 +37,11 @@ s3://raster-vision/examples/0.20.1/processed-data/ \
 python "rastervision_pytorch_backend/rastervision/pytorch_backend/examples/test.py" \
 run "spacenet-rio-cc" \
 --remote
+
+python "rastervision_pytorch_backend/rastervision/pytorch_backend/examples/test.py" \
+run "isprs-potsdam-ss" \
+-o "remote.raw_uri" "s3://raster-vision-ahassan/potsdam/data/raw" \
+--remote
 ```
 
 **All examples**:
@@ -60,8 +65,8 @@ compare \
 ```sh
 python "rastervision_pytorch_backend/rastervision/pytorch_backend/examples/test.py" \
 compare \
---examples_root_old "s3://raster-vision/examples/0.20/output/" \
---examples_root_new "s3://raster-vision/examples/0.20.1/output/"
+--examples_root_old "s3://raster-vision/examples/0.20.1/output/" \
+--examples_root_new "s3://raster-vision/examples/0.21/output/"
 ```
 
 ### Collect

diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/test.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/test.py
@@ -11,9 +11,8 @@
     file_to_json, sync_from_dir, upload_or_copy, download_or_copy, file_exists,
     sync_to_dir, NotReadableError, download_if_needed)
 
-OLD_VERSION = '0.20'
-NEW_VERSION = '0.20.1'
-NEW_VERSION_MAJOR_MINOR = '0.20'
+NEW_VERSION = '0.21'
+NEW_VERSION_MAJOR_MINOR = '0.21'
 
 EXAMPLES_MODULE_ROOT = 'rastervision.pytorch_backend.examples'
 EXAMPLES_PATH_ROOT = '/opt/src/rastervision_pytorch_backend/rastervision/pytorch_backend/examples'  # noqa
@@ -467,7 +466,7 @@ def _compare_runs(root_uri_old: str,
 
 def _compare_evals(root_uri_old: str,
                    root_uri_new: str,
-                   float_tol: float = 1e-3,
+                   float_tol: float = 1e-2,
                    exclude_keys: list = ['conf_mat', 'count',
                                          'per_scene']) -> None:
     """Compare outputs of the eval command for two runs of an example."""
@@ -578,7 +577,7 @@ def flatten_dict(d: Union[dict, list], sep: str = '.') -> dict:
 
 def _compare_dicts(dict_old: dict,
                    dict_new: dict,
-                   float_tol: float = 1e-3,
+                   float_tol: float = 1e-2,
                    exclude_keys: list = []) -> None:
     """Compare the keys and values of the two dicts.