From 0e2b0c4bde3abb3eb20422d741dffc757b4dcfb0 Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Mon, 4 Sep 2023 22:56:23 -0400 Subject: [PATCH 01/14] add pure-retrieval baseline --- scripts/benchmark_wildcat.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/benchmark_wildcat.sh b/scripts/benchmark_wildcat.sh index ef8c07333..526e5e288 100755 --- a/scripts/benchmark_wildcat.sh +++ b/scripts/benchmark_wildcat.sh @@ -14,6 +14,7 @@ datasets=( ) max_frame_lookahead_sizes=( + 0 5 10 15 From 96bcc53a82fcdb41061f828762def25a276329a7 Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Tue, 5 Sep 2023 08:38:05 -0400 Subject: [PATCH 02/14] add nms option --- .../keypoint_aggregator_dedup.py | 16 +++++++++++----- .../keypoint_aggregator_unique.py | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py index d31ef54ff..71677d6f2 100644 --- a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py +++ b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py @@ -15,11 +15,16 @@ class KeypointAggregatorDedup(KeypointAggregatorBase): - """Keypoint aggregator with de-duplication.""" + """Keypoint aggregator with de-duplication of keypoints within each image.""" - def __init__(self) -> None: - """Initialize global variables""" + def __init__(self, nms_radius: float = 3) -> None: + """Initialize global variables. + + Args: + nms_radius: Radius (in pixels) to use when merging detections within the same view. + """ self.duplicates_found = 0 + self.nms_radius = nms_radius def append_unique_keypoints( self, i: int, keypoints: Keypoints, per_image_kpt_coordinates: Dict[Tuple[int, int], np.ndarray] @@ -46,8 +51,8 @@ def append_unique_keypoints( for k, uv in enumerate(keypoints.coordinates): diff_norms = np.linalg.norm(per_image_kpt_coordinates[i] - uv, axis=1) # TODO(johnwlambert,ayushbaid): test loosening threshold below to some epsilon. - is_identical = np.any(diff_norms == 0) - if len(per_image_kpt_coordinates[i]) > 0 and is_identical: + is_duplicate = np.any(diff_norms <= self.nms_radius) + if len(per_image_kpt_coordinates[i]) > 0 and is_duplicate: self.duplicates_found += 1 i_indices[k] = np.argmin(diff_norms) else: @@ -103,6 +108,7 @@ def aggregate( putative_corr_idxs_dict[(i1, i2)] = putative_corr_idxs logger.info(f"Merged {self.duplicates_found} duplicates during de-duplication.") + print(f"Merged {self.duplicates_found} duplicates during de-duplication.") # Reset global state. self.duplicates_found = 0 diff --git a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_unique.py b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_unique.py index b3d748510..5f5f5da13 100644 --- a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_unique.py +++ b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_unique.py @@ -12,7 +12,7 @@ class KeypointAggregatorUnique(KeypointAggregatorBase): - """Keypoint aggregator without de-duplication.""" + """Keypoint aggregator without de-duplication, allowing for potentially duplicate keypoints per image.""" def aggregate( self, keypoints_dict: Dict[Tuple[int, int], Tuple[Keypoints, Keypoints]] From 1212d6535ed3e3e85e832460ff717a1cbb652fa7 Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Mon, 11 Sep 2023 09:12:17 -0400 Subject: [PATCH 03/14] int32 to MatchIndicesMap instead of uint32 --- gtsfm/data_association/cpp_dsf_tracks_estimator.py | 2 +- .../keypoint_aggregator/keypoint_aggregator_dedup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gtsfm/data_association/cpp_dsf_tracks_estimator.py b/gtsfm/data_association/cpp_dsf_tracks_estimator.py index a34fc7de3..e27e973cc 100644 --- a/gtsfm/data_association/cpp_dsf_tracks_estimator.py +++ b/gtsfm/data_association/cpp_dsf_tracks_estimator.py @@ -55,7 +55,7 @@ def run(self, matches_dict: Dict[Tuple[int, int], np.ndarray], keypoints_list: L # (Converts python dict into gtsam.MatchIndicesMap.) matches_map = gtsam.MatchIndicesMap() for (i1, i2), corr_idxs in matches_dict.items(): - matches_map[gtsam.IndexPair(i1, i2)] = corr_idxs + matches_map[gtsam.IndexPair(i1, i2)] = corr_idxs.astype(np.int32) # Convert gtsfm Keypoints into gtsam Keypoints. keypoints_vector = gtsam.KeypointsVector() diff --git a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py index 71677d6f2..1b4447734 100644 --- a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py +++ b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py @@ -104,7 +104,7 @@ def aggregate( per_image_kpt_coordinates, i2_indices = self.append_unique_keypoints( i=i2, keypoints=keypoints_i2, per_image_kpt_coordinates=per_image_kpt_coordinates ) - putative_corr_idxs = np.stack([i1_indices, i2_indices], axis=-1).astype(np.uint16) + putative_corr_idxs = np.stack([i1_indices, i2_indices], axis=-1).astype(np.int32) putative_corr_idxs_dict[(i1, i2)] = putative_corr_idxs logger.info(f"Merged {self.duplicates_found} duplicates during de-duplication.") From a3c023d649baa7b4ef111c17f60ea25a4304829c Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Mon, 11 Sep 2023 20:17:06 -0400 Subject: [PATCH 04/14] make BA return correct shape --- gtsfm/two_view_estimator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gtsfm/two_view_estimator.py b/gtsfm/two_view_estimator.py index 4357733cd..8e8836302 100644 --- a/gtsfm/two_view_estimator.py +++ b/gtsfm/two_view_estimator.py @@ -184,7 +184,7 @@ def bundle_adjust( logger.debug("Triangulated %d correspondences out of %d.", len(triangulated_tracks), len(verified_corr_idxs)) if len(triangulated_tracks) == 0: - return i2Ti1_initial.rotation(), Unit3(i2Ti1_initial.translation()), np.array([], dtype=np.uint32) + return i2Ti1_initial.rotation(), Unit3(i2Ti1_initial.translation()), np.zeros(shape=(0,2), dtype=np.int32) # Build BA inputs. start_time = timeit.default_timer() From 83775bb187826b571f2b735615354d6b49ad73b3 Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Mon, 11 Sep 2023 23:30:23 -0400 Subject: [PATCH 05/14] fix bug in script flag elif cond --- scripts/benchmark_wildcat.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/benchmark_wildcat.sh b/scripts/benchmark_wildcat.sh index 526e5e288..3d543cc8f 100755 --- a/scripts/benchmark_wildcat.sh +++ b/scripts/benchmark_wildcat.sh @@ -100,7 +100,7 @@ for num_matched in ${num_matched_sizes[@]}; do --output_root $OUTPUT_ROOT \ --max_resolution 760 \ 2>&1 | tee $OUTPUT_ROOT/out.log - elif [[ $loader == *"olsson"* ]] + elif [[ $loader == *"colmap"* ]] then python gtsfm/runner/run_scene_optimizer_colmaploader.py \ --mvs_off \ From 9c4f8535c949d1a849e722b15f0b1b53e115539d Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Mon, 11 Sep 2023 23:31:27 -0400 Subject: [PATCH 06/14] merge conflict --- gtsfm/runner/gtsfm_runner_base.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/gtsfm/runner/gtsfm_runner_base.py b/gtsfm/runner/gtsfm_runner_base.py index c7c5e2983..01632b343 100644 --- a/gtsfm/runner/gtsfm_runner_base.py +++ b/gtsfm/runner/gtsfm_runner_base.py @@ -10,7 +10,8 @@ import dask import hydra import numpy as np -from dask import config as dask_config + +# from dask import config as dask_config from dask.distributed import Client, LocalCluster, SSHCluster, performance_report from gtsam import Rot3, Unit3 from hydra.utils import instantiate @@ -29,7 +30,8 @@ from gtsfm.two_view_estimator import TWO_VIEW_OUTPUT, TwoViewEstimationReport, run_two_view_estimator_as_futures from gtsfm.ui.process_graph_generator import ProcessGraphGenerator -dask_config.set({"distributed.scheduler.worker-ttl": None}) +# from dask import config as dask_config +# dask_config.set({"distributed.scheduler.worker-ttl": None}) logger = logger_utils.get_logger() @@ -350,12 +352,17 @@ def run(self) -> GtsfmData: assert isinstance(sfm_result, GtsfmData) all_metrics_groups.extend(mvo_metrics_groups) - save_metrics_reports(all_metrics_groups, os.path.join(self.scene_optimizer.output_root, "result_metrics")) end_time = time.time() duration_sec = end_time - start_time logger.info("GTSFM took %.2f minutes to compute sparse multi-view result.", duration_sec / 60) + total_summary_metrics = GtsfmMetricsGroup( + "total_summary_metrics", [GtsfmMetric("total_runtime_sec", duration_sec)] + ) + all_metrics_groups.append(total_summary_metrics) + + save_metrics_reports(all_metrics_groups, os.path.join(self.scene_optimizer.output_root, "result_metrics")) return sfm_result From aab676f7f5127f4a12cc0de1148e18087b39fed7 Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Tue, 12 Sep 2023 00:34:27 -0400 Subject: [PATCH 07/14] fix test --- .../image_correspondence_generator.py | 24 +++++++-------- .../test_keypoint_aggregator_dedup.py | 30 +++++++++++++++++-- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/gtsfm/frontend/correspondence_generator/image_correspondence_generator.py b/gtsfm/frontend/correspondence_generator/image_correspondence_generator.py index 81e7bbcab..af1ed50b4 100644 --- a/gtsfm/frontend/correspondence_generator/image_correspondence_generator.py +++ b/gtsfm/frontend/correspondence_generator/image_correspondence_generator.py @@ -29,8 +29,8 @@ class ImageCorrespondenceGenerator(CorrespondenceGeneratorBase): def __init__(self, matcher: ImageMatcherBase, deduplicate: bool = True) -> None: """ Args: - matcher: matcher to use. - deduplicate: whether to de-duplicate with a single image the detections received from each image pair. + matcher: Matcher to use. + deduplicate: Whether to de-duplicate with a single image the detections received from each image pair. """ self._matcher = matcher @@ -42,6 +42,7 @@ def __repr__(self) -> str: return f""" ImageCorrespondenceGenerator: {self._matcher} + {self._aggregator} """ def generate_correspondences( @@ -53,9 +54,9 @@ def generate_correspondences( """Apply the correspondence generator to generate putative correspondences. Args: - client: dask client, used to execute the front-end as futures. - images: list of all images, as futures. - image_pairs: indices of the pairs of images to estimate two-view pose and correspondences. + client: Dask client, used to execute the front-end as futures. + images: List of all images, as futures. + image_pairs: Indices of the pairs of images to estimate two-view pose and correspondences. Returns: List of keypoints, one entry for each input images. @@ -96,14 +97,14 @@ def generate_correspondences_and_estimate_two_view( two view estimator to complete the front-end. Args: - client: dask client, used to execute the front-end as futures. - images: list of all images. - image_pairs: indices of the pairs of images to estimate two-view pose and correspondences. - camera_intrinsics: list of all camera intrinsics. - relative_pose_priors: priors on relative pose between two cameras. + client: Dask client, used to execute the front-end as futures. + images: List of all images. + image_pairs: Indices of the pairs of images to estimate two-view pose and correspondences. + camera_intrinsics: List of all camera intrinsics. + relative_pose_priors: Priors on relative pose between two cameras. gt_cameras: GT cameras, used to evaluate metrics. gt_scene_mesh: GT mesh of the 3D scene, used to evaluate metrics. - two_view_estimator: two view estimator, which is used to verify correspondences and estimate pose. + two_view_estimator: Two view estimator, which is used to verify correspondences and estimate pose. Returns: List of keypoints, one entry for each input images. @@ -170,5 +171,4 @@ def apply_two_view_estimator( } two_view_output_dict = client.gather(two_view_output_futures) - return keypoints_list, two_view_output_dict diff --git a/tests/frontend/correspondence_generator/keypoint_aggregator/test_keypoint_aggregator_dedup.py b/tests/frontend/correspondence_generator/keypoint_aggregator/test_keypoint_aggregator_dedup.py index f817551fd..17a69ec73 100644 --- a/tests/frontend/correspondence_generator/keypoint_aggregator/test_keypoint_aggregator_dedup.py +++ b/tests/frontend/correspondence_generator/keypoint_aggregator/test_keypoint_aggregator_dedup.py @@ -2,17 +2,24 @@ Authors: John Lambert """ +import pathlib import unittest from typing import Dict, Tuple import numpy as np from gtsfm.common.keypoints import Keypoints +from gtsfm.frontend.cacher.image_matcher_cacher import ImageMatcherCacher +from gtsfm.frontend.matcher.loftr import LOFTR +from gtsfm.loader.olsson_loader import OlssonLoader from gtsfm.frontend.correspondence_generator.keypoint_aggregator.keypoint_aggregator_dedup import ( KeypointAggregatorDedup, ) from tests.frontend.correspondence_generator.keypoint_aggregator import test_keypoint_aggregator_base +DATA_ROOT_PATH = pathlib.Path(__file__).resolve().parent.parent.parent.parent / "data" +DEFAULT_FOLDER = DATA_ROOT_PATH / "set1_lund_door" + class TestKeypointAggregatorDedup(test_keypoint_aggregator_base.TestKeypointAggregatorBase): """Test class for DoG detector class in frontend. @@ -22,7 +29,7 @@ class TestKeypointAggregatorDedup(test_keypoint_aggregator_base.TestKeypointAggr def setUp(self): super().setUp() - self.aggregator = KeypointAggregatorDedup() + self.aggregator = KeypointAggregatorDedup(nms_radius=0.0) def test_keypoint_aggregator_repeated_keypoints(self) -> None: """Ensure aggregation works over 3 images, with duplicate keypoints in the same image from separate pairs. @@ -59,7 +66,7 @@ def test_keypoint_aggregator_repeated_keypoints(self) -> None: assert len(keypoints_list) == 3 assert all([isinstance(kps, Keypoints) for kps in keypoints_list]) - # removing duplicates + # Duplicates should have been removed. expected_putative_corr_idxs_dict = { (0, 1): np.array([[0, 0]]), (1, 2): np.array([[1, 0]]), @@ -84,6 +91,25 @@ def test_keypoint_aggregator_repeated_keypoints(self) -> None: expected_image2_kps = np.array([[2.0, 2.0], [3.0, 3.0], [4.0, 4.0]]) assert np.allclose(keypoints_list[2].coordinates, expected_image2_kps) + def test_dedup_nms_radius_3(self) -> None: + loader = OlssonLoader(str(DEFAULT_FOLDER), max_frame_lookahead=4) + image_matcher = ImageMatcherCacher(matcher_obj=LOFTR()) + + images = [loader.get_image(i) for i in range(4)] + image_pairs = [(0, 1), (1, 2), (2, 3), (0, 2), (1, 3)] + aggregator = KeypointAggregatorDedup(nms_radius=3.0) + + pairwise_correspondences = { + (i1, i2): image_matcher.match(image_i1=images[i1], image_i2=images[i2]) for i1, i2 in image_pairs + } + + keypoints_list, putative_corr_idxs_dict = aggregator.aggregate(keypoints_dict=pairwise_correspondences) + + for (i1, i2), corr_idxs in putative_corr_idxs_dict.items(): + assert corr_idxs.dtype == np.int32 + assert len(corr_idxs.shape) == 2 + assert corr_idxs.shape[-1] == 2 + if __name__ == "__main__": unittest.main() From 3bf9c303bb3a11e143bfc54b9c46aac40fdca8c4 Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Tue, 12 Sep 2023 21:59:11 -0400 Subject: [PATCH 08/14] update var name --- gtsfm/runner/gtsfm_runner_base.py | 6 ++---- scripts/collect_results.py | 12 ++++++++---- .../test_keypoint_aggregator_dedup.py | 6 +++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/gtsfm/runner/gtsfm_runner_base.py b/gtsfm/runner/gtsfm_runner_base.py index 01632b343..d35a00df3 100644 --- a/gtsfm/runner/gtsfm_runner_base.py +++ b/gtsfm/runner/gtsfm_runner_base.py @@ -10,8 +10,7 @@ import dask import hydra import numpy as np - -# from dask import config as dask_config +from dask import config as dask_config from dask.distributed import Client, LocalCluster, SSHCluster, performance_report from gtsam import Rot3, Unit3 from hydra.utils import instantiate @@ -30,8 +29,7 @@ from gtsfm.two_view_estimator import TWO_VIEW_OUTPUT, TwoViewEstimationReport, run_two_view_estimator_as_futures from gtsfm.ui.process_graph_generator import ProcessGraphGenerator -# from dask import config as dask_config -# dask_config.set({"distributed.scheduler.worker-ttl": None}) +dask_config.set({"distributed.scheduler.worker-ttl": None}) logger = logger_utils.get_logger() diff --git a/scripts/collect_results.py b/scripts/collect_results.py index 12da1be4f..9a9313d24 100644 --- a/scripts/collect_results.py +++ b/scripts/collect_results.py @@ -61,6 +61,9 @@ "total_run_duration_sec", ] +total_fname = "total_summary_metrics.json" +total_metrics = ["total_runtime_sec"] + # Metrics that **do not** have a median + mean value associated. SCALAR_METRIC_NAMES = [ "number_cameras", @@ -83,6 +86,7 @@ "num_input_measurements", "num_inlier_measurements", "num_outlier_measurements", + "total_runtime_sec" ] @@ -92,7 +96,7 @@ def main(user_root: Path, output_fpath: str) -> None: table = defaultdict(list) headers = ["method_name"] - experiment_roots = sorted(list(user_root.glob("*-*"))) + experiment_roots = sorted(list(user_root.glob("*__*"))) method_idx = 0 for experiment_root in experiment_roots: @@ -102,9 +106,9 @@ def main(user_root: Path, output_fpath: str) -> None: table["method_name"].append(frontend_name) for json_fname, metric_names, nickname in zip( - [retriever_fname, isp_fname, vg_fname, ra_fname, ta_fname, ba_result_fname], - [retriever_metrics, isp_metrics, vg_metrics, ra_metrics, ta_metrics, ba_result_metrics], - ["retriever", "isp", "vg", "ra", "ta", "ba"], + [retriever_fname, isp_fname, vg_fname, ra_fname, ta_fname, ba_result_fname, total_fname], + [retriever_metrics, isp_metrics, vg_metrics, ra_metrics, ta_metrics, ba_result_metrics, total_metrics], + ["retriever", "isp", "vg", "ra", "ta", "ba", "total"], ): section_name = Path(json_fname).stem print(f"{dirpath}/{json_fname}") diff --git a/tests/frontend/correspondence_generator/keypoint_aggregator/test_keypoint_aggregator_dedup.py b/tests/frontend/correspondence_generator/keypoint_aggregator/test_keypoint_aggregator_dedup.py index 17a69ec73..47812a860 100644 --- a/tests/frontend/correspondence_generator/keypoint_aggregator/test_keypoint_aggregator_dedup.py +++ b/tests/frontend/correspondence_generator/keypoint_aggregator/test_keypoint_aggregator_dedup.py @@ -29,7 +29,7 @@ class TestKeypointAggregatorDedup(test_keypoint_aggregator_base.TestKeypointAggr def setUp(self): super().setUp() - self.aggregator = KeypointAggregatorDedup(nms_radius=0.0) + self.aggregator = KeypointAggregatorDedup(nms_merge_radius=0.0) def test_keypoint_aggregator_repeated_keypoints(self) -> None: """Ensure aggregation works over 3 images, with duplicate keypoints in the same image from separate pairs. @@ -91,13 +91,13 @@ def test_keypoint_aggregator_repeated_keypoints(self) -> None: expected_image2_kps = np.array([[2.0, 2.0], [3.0, 3.0], [4.0, 4.0]]) assert np.allclose(keypoints_list[2].coordinates, expected_image2_kps) - def test_dedup_nms_radius_3(self) -> None: + def test_dedup_nms_merge_radius_3(self) -> None: loader = OlssonLoader(str(DEFAULT_FOLDER), max_frame_lookahead=4) image_matcher = ImageMatcherCacher(matcher_obj=LOFTR()) images = [loader.get_image(i) for i in range(4)] image_pairs = [(0, 1), (1, 2), (2, 3), (0, 2), (1, 3)] - aggregator = KeypointAggregatorDedup(nms_radius=3.0) + aggregator = KeypointAggregatorDedup(nms_merge_radius=3.0) pairwise_correspondences = { (i1, i2): image_matcher.match(image_i1=images[i1], image_i2=images[i2]) for i1, i2 in image_pairs From 7b0066d70b732d709a44abdad34ebe9f80bbc863 Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Tue, 12 Sep 2023 21:59:32 -0400 Subject: [PATCH 09/14] update var name --- .../keypoint_aggregator/keypoint_aggregator_dedup.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py index 1b4447734..ac07429c1 100644 --- a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py +++ b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py @@ -17,14 +17,15 @@ class KeypointAggregatorDedup(KeypointAggregatorBase): """Keypoint aggregator with de-duplication of keypoints within each image.""" - def __init__(self, nms_radius: float = 3) -> None: + def __init__(self, nms_merge_radius: float = 3) -> None: """Initialize global variables. Args: - nms_radius: Radius (in pixels) to use when merging detections within the same view. + nms_merge_radius: Radius (in pixels) to use when merging detections within the same view (image). + Note that tracks are merged, not suppressed. """ self.duplicates_found = 0 - self.nms_radius = nms_radius + self.nms_merge_radius = nms_merge_radius def append_unique_keypoints( self, i: int, keypoints: Keypoints, per_image_kpt_coordinates: Dict[Tuple[int, int], np.ndarray] @@ -50,8 +51,8 @@ def append_unique_keypoints( for k, uv in enumerate(keypoints.coordinates): diff_norms = np.linalg.norm(per_image_kpt_coordinates[i] - uv, axis=1) - # TODO(johnwlambert,ayushbaid): test loosening threshold below to some epsilon. - is_duplicate = np.any(diff_norms <= self.nms_radius) + # TODO(johnwlambert,travisdriver): Use the average coordinate instead of first coordinate. + is_duplicate = np.any(diff_norms <= self.nms_merge_radius) if len(per_image_kpt_coordinates[i]) > 0 and is_duplicate: self.duplicates_found += 1 i_indices[k] = np.argmin(diff_norms) From dd5c99c99eb26bc6a9639e3747df2f75d23d13ea Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Tue, 12 Sep 2023 22:00:25 -0400 Subject: [PATCH 10/14] fix format --- gtsfm/two_view_estimator.py | 2 +- tests/data_association/test_dsf_tracks_estimator.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gtsfm/two_view_estimator.py b/gtsfm/two_view_estimator.py index 8e8836302..6f3ecac6f 100644 --- a/gtsfm/two_view_estimator.py +++ b/gtsfm/two_view_estimator.py @@ -184,7 +184,7 @@ def bundle_adjust( logger.debug("Triangulated %d correspondences out of %d.", len(triangulated_tracks), len(verified_corr_idxs)) if len(triangulated_tracks) == 0: - return i2Ti1_initial.rotation(), Unit3(i2Ti1_initial.translation()), np.zeros(shape=(0,2), dtype=np.int32) + return i2Ti1_initial.rotation(), Unit3(i2Ti1_initial.translation()), np.zeros(shape=(0, 2), dtype=np.int32) # Build BA inputs. start_time = timeit.default_timer() diff --git a/tests/data_association/test_dsf_tracks_estimator.py b/tests/data_association/test_dsf_tracks_estimator.py index 18a9fffae..e2469c494 100644 --- a/tests/data_association/test_dsf_tracks_estimator.py +++ b/tests/data_association/test_dsf_tracks_estimator.py @@ -174,13 +174,13 @@ def get_dummy_matches() -> Dict[Tuple[int, int], np.ndarray]: def get_nontransitive_matches() -> Dict[Tuple[int, int], np.ndarray]: """Set up correspondences for each (i1,i2) pair that violates transitivity. - + (i=0, k=0) (i=0, k=1) | \\ | | \\ | (i=1, k=2)--(i=2,k=3)--(i=3, k=4) - Transitivity is violated due to the match between frames 0 and 3. + Transitivity is violated due to the match between frames 0 and 3. """ nontransitive_matches_dict = { (0, 1): np.array([[0, 2]]), From 69dc6c62d6f3eb0a7abcb96fe422d1b540ec26be Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Wed, 13 Sep 2023 23:03:46 -0400 Subject: [PATCH 11/14] use running mean keypoint loc --- .../keypoint_aggregator/keypoint_aggregator_dedup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py index ac07429c1..144d0bcd3 100644 --- a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py +++ b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py @@ -51,11 +51,13 @@ def append_unique_keypoints( for k, uv in enumerate(keypoints.coordinates): diff_norms = np.linalg.norm(per_image_kpt_coordinates[i] - uv, axis=1) - # TODO(johnwlambert,travisdriver): Use the average coordinate instead of first coordinate. is_duplicate = np.any(diff_norms <= self.nms_merge_radius) if len(per_image_kpt_coordinates[i]) > 0 and is_duplicate: self.duplicates_found += 1 - i_indices[k] = np.argmin(diff_norms) + img_global_kpt_idx = np.argmin(diff_norms) + i_indices[k] = img_global_kpt_idx + # Modify keypoint coordinate to be set to average value, instead of first coordinate. + per_image_kpt_coordinates[i] = np.mean([per_image_kpt_coordinates[img_global_kpt_idx], uv]) else: i_indices[k] = i_count i_count += 1 From 07918c83c8cf98a54f6dd2a7b0b1e35d024cb891 Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Wed, 13 Sep 2023 23:04:04 -0400 Subject: [PATCH 12/14] improve comment --- .../keypoint_aggregator/keypoint_aggregator_dedup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py index 144d0bcd3..014a02d8c 100644 --- a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py +++ b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py @@ -56,7 +56,8 @@ def append_unique_keypoints( self.duplicates_found += 1 img_global_kpt_idx = np.argmin(diff_norms) i_indices[k] = img_global_kpt_idx - # Modify keypoint coordinate to be set to average value, instead of first coordinate. + # Modify global keypoint coordinate to be set to average value of merged detections, instead of + # using the first identified coordinate. per_image_kpt_coordinates[i] = np.mean([per_image_kpt_coordinates[img_global_kpt_idx], uv]) else: i_indices[k] = i_count From ffaf257ff3c03673ea06206aa038b54a374671ef Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Thu, 14 Sep 2023 00:02:35 -0400 Subject: [PATCH 13/14] fix index error --- .../keypoint_aggregator/keypoint_aggregator_dedup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py index 014a02d8c..77ac50646 100644 --- a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py +++ b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py @@ -58,7 +58,9 @@ def append_unique_keypoints( i_indices[k] = img_global_kpt_idx # Modify global keypoint coordinate to be set to average value of merged detections, instead of # using the first identified coordinate. - per_image_kpt_coordinates[i] = np.mean([per_image_kpt_coordinates[img_global_kpt_idx], uv]) + updated_uv = np.mean([per_image_kpt_coordinates[i][img_global_kpt_idx], uv], axis=0) + per_image_kpt_coordinates[i][img_global_kpt_idx] = updated_uv + else: i_indices[k] = i_count i_count += 1 From ccdf83125a4849590459ed251c69171593f8a54c Mon Sep 17 00:00:00 2001 From: senselessdev1 Date: Thu, 28 Sep 2023 09:22:03 -0400 Subject: [PATCH 14/14] remove superfluous print statement --- .../keypoint_aggregator/keypoint_aggregator_dedup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py index 2ea8b4a85..4e851ad60 100644 --- a/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py +++ b/gtsfm/frontend/correspondence_generator/keypoint_aggregator/keypoint_aggregator_dedup.py @@ -115,7 +115,6 @@ def aggregate( putative_corr_idxs_dict[(i1, i2)] = putative_corr_idxs logger.info(f"Merged {self.duplicates_found} duplicates during de-duplication.") - print(f"Merged {self.duplicates_found} duplicates during de-duplication.") # Reset global state. self.duplicates_found = 0