Remove code which has migrated into Hoplite repository.

PiperOrigin-RevId: 714119621

.github/workflows/ci.yml

@@ -36,7 +36,3 @@ jobs:
         run: poetry run python -m unittest discover -s chirp/train_tests -p "*test.py"
       - name: Test inference with unittest
         run: poetry run python -m unittest discover -s chirp/inference/tests -p "*test.py"
-      - name: Test hoplite with unittest
-        run: poetry run python -m unittest discover -s chirp/projects/agile2/tests -p "*test.py"
-      - name: Test agile2 with unittest
-        run: poetry run python -m unittest discover -s chirp/projects/hoplite/tests -p "*test.py"

.github/workflows/ci_pip.yml

@@ -29,12 +29,9 @@ jobs:
           pip install absl-py
           pip install requests
           pip install tensorflow-cpu
+          pip install git+https://github.com/google-research/perch-hoplite.git
           pip install git+https://github.com/google-research/perch.git
       - name: Test with unittest
         run: python -m unittest discover -s chirp/tests -p "*test.py"
       - name: Test inference with unittest
         run: python -m unittest discover -s chirp/inference/tests -p "*test.py"
-      - name: Test hoplite with unittest
-        run: python -m unittest discover -s chirp/projects/agile2/tests -p "*test.py"
-      - name: Test agile2 with unittest
-        run: python -m unittest discover -s chirp/projects/hoplite/tests -p "*test.py"

agile_modeling.ipynb

@@ -61,14 +61,14 @@
     "from chirp.inference import baw_utils\n",
     "from chirp.inference import tf_examples\n",
     "from chirp.models import metrics\n",
-    "from chirp.taxonomy import namespace\n",
+    "from perch_hoplite.taxonomy import namespace\n",
     "from chirp.inference.search import bootstrap\n",
     "from chirp.inference.search import search\n",
     "from chirp.inference.search import display\n",
     "from chirp.inference.classify import classify\n",
     "from chirp.inference.classify import data_lib\n",
-    "from chirp.projects.zoo import model_configs\n",
-    "from chirp.projects.zoo import zoo_interface\n"
+    "from perch_hoplite.zoo import model_configs\n",
+    "from perch_hoplite.zoo import zoo_interface\n"
    ]
   },
   {

analysis.ipynb

@@ -56,7 +56,7 @@
     "from chirp.inference.search import search\n",
     "from chirp.inference.search import display\n",
     "from chirp.inference.classify import classify\n",
-    "from chirp.projects.zoo import zoo_interface\n"
+    "from perch_hoplite.zoo import zoo_interface\n"
    ]
   },
   {

chirp/configs/config_globals.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 """Define the globals that can be used in configuration files."""
+
 from typing import Any
 
 from chirp import audio_utils

chirp/data/bird_taxonomy/bird_taxonomy.py

@@ -26,12 +26,12 @@
 from chirp.data import filter_scrub_utils as fsu
 from chirp.data import tfds_features
 from chirp.data.bird_taxonomy import premade_queries
-from chirp.taxonomy import namespace_db
 from etils import epath
 import jax
 from jax import numpy as jnp
 import numpy as np
 import pandas as pd
+from perch_hoplite.taxonomy import namespace_db
 import tensorflow as tf
 import tensorflow_datasets as tfds
 

chirp/data/bird_taxonomy/premade_queries.py

@@ -16,7 +16,7 @@
 """A set of premade queries to generate stable data configs."""
 
 from chirp.data import filter_scrub_utils as fsu
-from chirp.taxonomy import namespace_db
+from perch_hoplite.taxonomy import namespace_db
 
 # SSW_STATS_PATH contains useful statistics for SSW species, used to guide our
 # DFS search in chirp.data.sampling_utils.sample_recordings_under_constraints.

chirp/data/filter_scrub_utils.py

@@ -14,14 +14,15 @@
 # limitations under the License.
 
 """Utilities to filter/scrub data."""
+
 import enum
 import functools
 from typing import Any, Dict, NamedTuple, Sequence, Union
 
 from chirp.data import sampling_utils as su
-from chirp.taxonomy import namespace_db
 import numpy as np
 import pandas as pd
+from perch_hoplite.taxonomy import namespace_db
 
 
 class MergeStrategy(enum.Enum):

chirp/data/soundevents/soundevents.py

@@ -24,11 +24,11 @@
 from chirp import audio_utils
 from chirp.data import filter_scrub_utils as fsu
 from chirp.data import tfds_features
-from chirp.taxonomy import namespace_db
 from etils import epath
 from jax import numpy as jnp
 import numpy as np
 import pandas as pd
+from perch_hoplite.taxonomy import namespace_db
 import tensorflow as tf
 import tensorflow_datasets as tfds
 

chirp/data/soundscapes/soundscapes.py

@@ -25,10 +25,10 @@
 from chirp.data import tfds_features
 from chirp.data.bird_taxonomy import bird_taxonomy
 from chirp.data.soundscapes import soundscapes_lib
-from chirp.taxonomy import annotations_fns
 from etils import epath
 import numpy as np
 import pandas as pd
+from perch_hoplite.taxonomy import annotations_fns
 import tensorflow as tf
 import tensorflow_datasets as tfds
 
@@ -47,6 +47,8 @@
   organization={CEUR}
 }
 """
+
+
 @dataclasses.dataclass
 class SoundscapesConfig(bird_taxonomy.BirdTaxonomyConfig):
   """Dataset configuration for Soundscape datasets.
@@ -93,6 +95,7 @@ class SoundscapesConfig(bird_taxonomy.BirdTaxonomyConfig):
   supervised: bool = True
   audio_dir = epath.Path('gs://chirp-public-bucket/soundscapes')
 
+
 class Soundscapes(bird_taxonomy.BirdTaxonomy):
   """DatasetBuilder for soundscapes data."""
 
@@ -485,7 +488,7 @@ def _generate_examples(self, segments: pd.DataFrame):
         segments = segments.drop(k, axis=1)
 
     def _process_group(
-        group: tuple[int, tuple[str, pd.DataFrame]]
+        group: tuple[int, tuple[str, pd.DataFrame]],
     ) -> Sequence[tuple[str, dict[str, Any]]]:
       # Each filename gets a unique ID
       recording_id, (filename, segment_group) = group

chirp/data/soundscapes/soundscapes_lib.py

@@ -20,12 +20,12 @@
 from typing import Any, Callable, Iterator, Set
 
 from absl import logging
-from chirp.taxonomy import namespace
-from chirp.taxonomy import namespace_db
 from etils import epath
 from jax import numpy as jnp
 import numpy as np
 import pandas as pd
+from perch_hoplite.taxonomy import namespace
+from perch_hoplite.taxonomy import namespace_db
 import tensorflow_datasets as tfds
 import tqdm
 

chirp/eval/callbacks.py

@@ -20,10 +20,6 @@
 
 from absl import logging
 from chirp.eval import eval_lib
-from chirp.projects.zoo import model_configs
-from chirp.projects.zoo import zoo_interface
-from chirp.taxonomy import namespace
-from chirp.taxonomy import namespace_db
 from chirp.train import classifier
 from chirp.train import hubert
 from chirp.train import separator
@@ -33,6 +29,9 @@
 from jax import numpy as jnp
 import ml_collections
 import numpy as np
+from perch_hoplite.taxonomy import namespace
+from perch_hoplite.zoo import model_configs
+from perch_hoplite.zoo import zoo_interface
 import tensorflow as tf
 
 ConfigDict = ml_collections.ConfigDict
@@ -142,13 +141,11 @@ def fprop(inputs):
 
       head_index = list(model_bundle.model.num_classes.keys()).index('label')
       output_weights = train_state.params[f'Dense_{head_index}']['kernel'].T
-      self.learned_representations.update(
-          {
-              n: w
-              for n, w in zip(class_list.classes, output_weights)
-              if n not in self.learned_representation_blocklist
-          }
-      )
+      self.learned_representations.update({
+          n: w
+          for n, w in zip(class_list.classes, output_weights)
+          if n not in self.learned_representation_blocklist
+      })
 
   def __call__(self, inputs: np.ndarray) -> np.ndarray:
     return np.asarray(self.model_callback(inputs))
@@ -200,13 +197,11 @@ def _load_learned_representations(self):
     else:
       output_weights = tf.train.load_variable(variables_path, candidates[0])
       output_weights = np.squeeze(output_weights)
-    self.learned_representations.update(
-        {
-            n: w
-            for n, w in zip(class_list.classes, output_weights)
-            if n not in self.learned_representation_blocklist
-        }
-    )
+    self.learned_representations.update({
+        n: w
+        for n, w in zip(class_list.classes, output_weights)
+        if n not in self.learned_representation_blocklist
+    })
 
   def __post_init__(self):
     logging.info('Loading separation model...')
@@ -273,7 +268,7 @@ class EmbeddingModelCallback:
 
   def __post_init__(self):
     logging.info('Loading separation model...')
-    model_class = model_configs.MODEL_CLASS_MAP[self.model_key]
+    model_class = model_configs.get_model_class(self.model_key)
     self.loaded_model = model_class(**self.model_config)
     # Set the object's call method as the model_callback.
     self.model_callback = self.__call__

chirp/eval/eval_lib.py

@@ -23,11 +23,11 @@
 from absl import logging
 from chirp.data import utils as data_utils
 from chirp.models import metrics
-from chirp.taxonomy import namespace_db
 import jax
 import ml_collections
 import numpy as np
 import pandas as pd
+from perch_hoplite.taxonomy import namespace_db
 import tensorflow as tf
 
 _EMBEDDING_KEY = 'embedding'
@@ -691,14 +691,12 @@ class representatives.
   eval_search_results = dict()
 
   # Gather all query vectors.
-  queries = np.stack(
-      [
-          learned_representations[ces.class_name]
-          if ces.class_name in learned_representations
-          else create_species_query(ces.class_representatives_df['embedding'])
-          for ces in eval_set.classwise_eval_sets
-      ]
-  )
+  queries = np.stack([
+      learned_representations[ces.class_name]
+      if ces.class_name in learned_representations
+      else create_species_query(ces.class_representatives_df['embedding'])
+      for ces in eval_set.classwise_eval_sets
+  ])
 
   # Perform a matrix-matrix scoring using stacked queries and search corpus
   # embeddings.

chirp/export_utils.py

@@ -19,8 +19,8 @@
 from typing import Sequence
 
 from absl import logging
-from chirp.taxonomy import namespace
 from jax.experimental import jax2tf
+from perch_hoplite.taxonomy import namespace
 import tensorflow as tf
 
 

chirp/inference/classify/classify.py

@@ -21,8 +21,8 @@
 from chirp.inference import tf_examples
 from chirp.inference.classify import data_lib
 from chirp.models import metrics
-from chirp.projects.zoo import zoo_interface
 import numpy as np
+from perch_hoplite.zoo import zoo_interface
 import tensorflow as tf
 import tqdm
 

chirp/inference/classify/data_lib.py

@@ -34,9 +34,9 @@
 
 from chirp import audio_utils
 from chirp.inference import tf_examples
-from chirp.projects.zoo import zoo_interface
 from etils import epath
 import numpy as np
+from perch_hoplite.zoo import zoo_interface
 import tensorflow as tf
 import tqdm
 

chirp/inference/embed_lib.py

@@ -27,11 +27,11 @@
 from chirp import audio_utils
 from chirp import path_utils
 from chirp.inference import tf_examples
-from chirp.projects.zoo import model_configs
-from chirp.projects.zoo import zoo_interface
 from etils import epath
 from ml_collections import config_dict
 import numpy as np
+from perch_hoplite.zoo import model_configs
+from perch_hoplite.zoo import zoo_interface
 import soundfile
 import tensorflow as tf
 
@@ -223,7 +223,7 @@ def __init__(
 
   def setup(self):
     if self.embedding_model is None:
-      model_class = model_configs.MODEL_CLASS_MAP[self.model_key]
+      model_class = model_configs.get_model_class(self.model_key)
       self.embedding_model = model_class.from_config(self.model_config)
     if hasattr(self, 'model_key'):
       del self.model_key

chirp/inference/scann_search_lib.py

@@ -22,10 +22,10 @@
 from chirp import audio_utils
 from chirp.inference import embed_lib
 from chirp.inference import tf_examples
-from chirp.projects.zoo import taxonomy_model_tf
 from etils import epath
 from ml_collections import config_dict
 import numpy as np
+from perch_hoplite.zoo import taxonomy_model_tf
 import tensorflow as tf
 
 from scann.scam_ops.py import scam_ops_pybind

chirp/inference/search/bootstrap.py

@@ -25,10 +25,10 @@
 from chirp.inference import embed_lib
 from chirp.inference import tf_examples
 from chirp.inference.search import search
-from chirp.projects.zoo import model_configs
-from chirp.projects.zoo import zoo_interface
 from etils import epath
 from ml_collections import config_dict
+from perch_hoplite.zoo import model_configs
+from perch_hoplite.zoo import zoo_interface
 import tensorflow as tf
 
 
@@ -52,9 +52,9 @@ class BootstrapState:
 
   def __post_init__(self):
     if self.embedding_model is None:
-      self.embedding_model = model_configs.MODEL_CLASS_MAP[
+      self.embedding_model = model_configs.get_model_class(
           self.config.model_key
-      ].from_config(self.config.model_config)
+      ).from_config(self.config.model_config)
     self.create_embeddings_dataset()
     if self.source_map is None:
       if self.baw_auth_token:

chirp/inference/search/search.py

@@ -22,9 +22,9 @@
 from typing import Any, Callable, List, Sequence
 
 from chirp.inference import tf_examples
-from chirp.projects.zoo import zoo_interface
 from etils import epath
 import numpy as np
+from perch_hoplite.zoo import zoo_interface
 from scipy.io import wavfile
 import tensorflow as tf
 import tqdm
@@ -33,6 +33,7 @@
 @dataclasses.dataclass
 class SearchResult:
   """Container for a single search result."""
+
   # Embedding vector.
   embedding: np.ndarray
   # Raw score for this result.

chirp/inference/tests/classify_test.py

@@ -19,9 +19,9 @@
 
 from chirp.inference.classify import classify
 from chirp.inference.classify import data_lib
-from chirp.projects.zoo import zoo_interface
-from chirp.taxonomy import namespace
 import numpy as np
+from perch_hoplite.taxonomy import namespace
+from perch_hoplite.zoo import zoo_interface
 
 from absl.testing import absltest
 from absl.testing import parameterized