Support optional imports in unit tests (#293)

Author: izellevy

tests/unit/tokenizer/test_cohere_hf_tokenizer.py

@@ -8,7 +8,14 @@ class TestCohereHFTokenizer(BaseTestTokenizer):
     @staticmethod
     @pytest.fixture(scope="class")
     def tokenizer():
-        return CohereHFTokenizer()
+        try:
+            tokenizer = CohereHFTokenizer()
+        except ImportError:
+            pytest.skip(
+                "`cohere` extra not installed. Skipping CohereHFTokenizer unit "
+                "tests"
+            )
+        return tokenizer
 
     @staticmethod
     @pytest.fixture

tests/unit/tokenizer/test_llama_tokenizer.py

@@ -8,7 +8,14 @@ class TestLlamaTokenizer(BaseTestTokenizer):
     @staticmethod
     @pytest.fixture(scope="class")
     def tokenizer():
-        return LlamaTokenizer(model_name="hf-internal-testing/llama-tokenizer")
+        try:
+            tokenizer = LlamaTokenizer(model_name="hf-internal-testing/llama-tokenizer")
+        except ImportError:
+            pytest.skip(
+                "`transformers` extra not installed. Skipping LLamaTokenizer unit "
+                "tests"
+            )
+        return tokenizer
 
     @staticmethod
     @pytest.fixture