Add primitives from nlp-primitives add-on (#7)

* add nlp * release notes * fix unit tests * fix unit tests * fix unit tests * fix unit tests * fix unit tests * remove tensorflow * fix data * fix data * fix data * fix data * fix data
alteryx · May 9, 2023 · 54ada4b · 54ada4b
1 parent 6f5f75a
commit 54ada4b
Show file tree

Hide file tree

Showing 202 changed files with 5,353,977 additions and 25 deletions.
diff --git a/.github/workflows/release_notes_updated.yaml b/.github/workflows/release_notes_updated.yaml
@@ -30,4 +30,4 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
       - if: ${{ steps.branch.outputs.is_dev == 'true' }}
         name: Check if release notes were updated
-        run: cat docs/release_notes.rst | grep ":pr:\`${{ github.event.number }}\`"
+        run: cat release_notes.rst | grep ":pr:\`${{ github.event.number }}\`"
diff --git a/.github/workflows/unit_tests_with_latest_deps.yaml b/.github/workflows/unit_tests_with_latest_deps.yaml
@@ -26,6 +26,7 @@ jobs:
           python-version: ${{ matrix.python_version }}
       - name: Build premium primitives package (whl, tar.gz)
         run: |
+          make clean
           make package
       - name: Install premium primitives via sdist
         run: |

diff --git a/.gitignore b/.gitignore
@@ -127,7 +127,6 @@ ENV/
 
 # pickle files
 *.p
-*.pickle
 
 .pytest_cache
 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,5 +1,6 @@
 exclude: |
   (?x)
+  ^premium_primitives/data/nltk_data/|
   .html$|.csv$|.svg$|.md$|.txt$|.json$|.xml$|.pickle$|^.github/|
   (LICENSE.*|README.*)
 repos:

diff --git a/Makefile b/Makefile
@@ -6,6 +6,10 @@ clean:
 	find . -name '.coverage.*' -delete
 	find . -name __pycache__ -delete
 	find . -type d -name ".pytest_cache" -exec rm -r {} +
+	rm -rf ./build
+	rm -rf ./dist
+	rm -rf ./premium_primitives.egg-info
+	rm -rf ./unpacked_sdist
 
 .PHONY: lint
 lint:

diff --git a/premium_primitives/__init__.py b/premium_primitives/__init__.py
@@ -1,39 +1,50 @@
 import inspect
 
+import nltk.data
+from importlib.util import find_spec
+
 import featuretools
 import pkg_resources
 from featuretools.primitives import AggregationPrimitive, TransformPrimitive
 
-from premium_primitives.country_code_to_continent import (  # noqa: F401
+from premium_primitives.diversity_score import DiversityScore
+from premium_primitives.lsa import LSA
+from premium_primitives.mean_characters_per_sentence import MeanCharactersPerSentence
+from premium_primitives.number_of_sentences import NumberOfSentences
+from premium_primitives.part_of_speech_count import PartOfSpeechCount
+from premium_primitives.polarity_score import PolarityScore
+from premium_primitives.stopword_count import StopwordCount
+
+from premium_primitives.country_code_to_continent import (
     CountryCodeToContinent,
 )
-from premium_primitives.country_code_to_income import CountryCodeToIncome  # noqa: F401
-from premium_primitives.country_code_to_population import (  # noqa: F401
+from premium_primitives.country_code_to_income import CountryCodeToIncome
+from premium_primitives.country_code_to_population import (
     CountryCodeToPopulation,
 )
-from premium_primitives.latlong_to_city import LatLongToCity  # noqa: F401
-from premium_primitives.latlong_to_countrycode import LatLongToCountryCode  # noqa: F401
-from premium_primitives.latlong_to_county import LatLongToCounty  # noqa: F401
-from premium_primitives.latlong_to_state import LatLongToState  # noqa: F401
-from premium_primitives.phone_number_to_area import PhoneNumberToArea  # noqa: F401
-from premium_primitives.phone_number_to_country import (  # noqa: F401
+from premium_primitives.latlong_to_city import LatLongToCity
+from premium_primitives.latlong_to_countrycode import LatLongToCountryCode
+from premium_primitives.latlong_to_county import LatLongToCounty
+from premium_primitives.latlong_to_state import LatLongToState
+from premium_primitives.phone_number_to_area import PhoneNumberToArea
+from premium_primitives.phone_number_to_country import (
     PhoneNumberToCountry,
 )
-from premium_primitives.postalcode_to_latlong import PostalCodeToLatLong  # noqa: F401
-from premium_primitives.postalcode_to_per_capita_income import (  # noqa: F401
+from premium_primitives.postalcode_to_latlong import PostalCodeToLatLong
+from premium_primitives.postalcode_to_per_capita_income import (
     PostalCodeToPerCapitaIncome,
 )
-from premium_primitives.postalcode_to_state import PostalCodeToState  # noqa: F401
-from premium_primitives.sub_region_code_to_median_household_income import (  # noqa: F401
+from premium_primitives.postalcode_to_state import PostalCodeToState
+from premium_primitives.sub_region_code_to_median_household_income import (
     SubRegionCodeToMedianHouseholdIncome,
 )
-from premium_primitives.sub_region_code_to_per_capita_income import (  # noqa: F401
+from premium_primitives.sub_region_code_to_per_capita_income import (
     SubRegionCodeToPerCapitaIncome,
 )
-from premium_primitives.sub_region_code_to_region import (  # noqa: F401
+from premium_primitives.sub_region_code_to_region import (
     SubRegionCodeToRegion,
 )
-from premium_primitives.version import __version__  # noqa: F401
+from premium_primitives.version import __version__
 
 PREMIUM_PRIMITIVES = [
     obj
@@ -46,6 +57,12 @@
     )
 ]
 
+nltk_data_path = pkg_resources.resource_filename(
+    "premium_primitives",
+    "data/nltk_data/",
+)
+nltk.data.path.insert(0, nltk_data_path)
+
 
 # set data primitives BEFORE we import them
 premium_primitives_data_folder = pkg_resources.resource_filename(