Skip to content

Commit

Permalink
update nlu demo to provide verbosity flag options explicitly
Browse files Browse the repository at this point in the history
  • Loading branch information
@fmigneault
fmigneault committed Dec 12, 2023
1 parent e8228c0 commit c885432
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 100 deletions.
148 changes: 51 additions & 97 deletions nlp/notebooks/NLU_demo.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
"source": [
"# initialize the demo class\n",
"from nlu_demo import NLU_demo\n",
"nlu = NLU_demo(verbose=True)"
"nlu = NLU_demo(verbose=False)"
]
},
{
Expand Down Expand Up @@ -84,7 +84,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "bf5d55d08da44a98bb3608c18c5f920d",
"model_id": "1b7ce44f1ada4f2c8fa9e88348490dfe",
"version_major": 2,
"version_minor": 0
},
Expand Down Expand Up @@ -138,7 +138,7 @@
}
],
"source": [
"# optionnally: view the gold annotations\n",
"# optionally: view the gold annotations\n",
"nlu.get_gold_annotations()"
]
},
Expand All @@ -156,13 +156,16 @@
"ExecuteTime": {
"end_time": "2023-12-11T19:15:17.518240803Z",
"start_time": "2023-12-11T19:15:17.480797364Z"
},
"jupyter": {
"source_hidden": true
}
},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "9552202fc8374fca8ed8c54cd429053c",
"model_id": "bc639df1ddb548fa8f2eb9d35a92d44e",
"version_major": 2,
"version_minor": 0
},
Expand Down Expand Up @@ -204,7 +207,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5a6b1deb341c47c5bda4ef0c89609aeb",
"model_id": "44f5fa3e2bb24eaf954bf8f7d51bf2c1",
"version_major": 2,
"version_minor": 0
},
Expand Down Expand Up @@ -263,57 +266,10 @@
"Reading config file: /home/francis/dev/daccs/pavics-jupyter-images/nlp/notebooks/nl2query/V1/v1_config.cfg\n",
"Reading config file: nl2query/V1/spacy_config.cfg\n",
"Reading config file: nl2query/V1/flair_config.cfg\n",
"2023-12-11 17:26:21,166 SequenceTagger predicts: Dictionary with 20 tags: <unk>, O, S-ORG, S-MISC, B-PER, E-PER, S-LOC, B-ORG, E-ORG, I-PER, S-PER, B-MISC, I-MISC, E-MISC, I-ORG, B-LOC, E-LOC, I-LOC, <START>, <STOP>\n",
"2023-12-12 10:38:34,066 SequenceTagger predicts: Dictionary with 20 tags: <unk>, O, S-ORG, S-MISC, B-PER, E-PER, S-LOC, B-ORG, E-ORG, I-PER, S-PER, B-MISC, I-MISC, E-MISC, I-ORG, B-LOC, E-LOC, I-LOC, <START>, <STOP>\n",
"Reading config file: /home/francis/dev/daccs/pavics-jupyter-images/nlp/notebooks/nl2query/V2/v2_config.cfg\n",
"Loading Chroma Vdb from... nl2query/V2/prop_vdb\n",
"Loading Chroma Vdb from... nl2query/V2/target_vdb\n",
"New query: yearly precipitation over Ottawa between 2000 and 2020\n",
"SPACY:\n",
" Ottawa 26 32 GPE\n",
"SPACY:\n",
" between 2000 and 2020 33 54 DATE\n",
"FLAIR:\n",
" Ottawa 26 32 LOC 0.9999957084655762\n",
"TEMPEX - V2:\n",
" {\n",
" \"text\": \"between 2000 and 2020\",\n",
" \"position\": [\n",
" 33,\n",
" 54\n",
" ],\n",
" \"type\": \"tempex\",\n",
" \"tempex_type\": \"range\",\n",
" \"target\": \"dataDate\",\n",
" \"value\": {\n",
" \"start\": \"2000-01-01T00:00:00Z\",\n",
" \"end\": \"2021-01-01T00:00:00Z\"\n",
" }\n",
"}\n",
"New query: yearly precipitation over Ottawa \n",
"LOCATION - V2:\n",
" {\n",
" \"text\": \"Ottawa\",\n",
" \"position\": [\n",
" 26,\n",
" 32\n",
" ],\n",
" \"type\": \"location\",\n",
" \"name\": \"Ottawa, Eastern Ontario, Ontario, Canada\",\n",
" \"value\": {\n",
" \"type\": \"Polygon\",\n",
" \"coordinates\": [\n",
" 45.5376502,\n",
" 44.9617738,\n",
" -75.2465783,\n",
" -76.3555857\n",
" ]\n",
" },\n",
" \"matchingType\": \"overlap\"\n",
"}\n",
"New query: yearly precipitation over \n",
"\n",
"Removing stopwords\n",
"New query: yearly precipitation\n"
"Loading Chroma Vdb from... nl2query/V2/target_vdb\n"
]
},
{
Expand All @@ -328,45 +284,6 @@
"name": "stdout",
"output_type": "stream",
"text": [
"TARGET - V2: {\n",
" \"text\": \"precipitation\",\n",
" \"position\": [\n",
" 7,\n",
" 20\n",
" ],\n",
" \"type\": \"target\",\n",
" \"name\": [\n",
" \"precipitation amount\",\n",
" \"stratiform precipitation amount\",\n",
" \"large scale precipitation amount\",\n",
" \"stratiform precipitation flux\",\n",
" \"large scale precipitation flux\",\n",
" \"stratiform rainfall amount\",\n",
" \"large scale rainfall amount\",\n",
" \"solid precipitation flux\",\n",
" \"stratiform rainfall rate\",\n",
" \"large scale rainfall rate\",\n",
" \"convective precipitation amount\",\n",
" \"lwe stratiform precipitation rate\",\n",
" \"lwe large scale precipitation rate\",\n",
" \"tendency of specific humidity due to stratiform precipitation\",\n",
" \"tendency of specific humidity due to large scale precipitation\",\n",
" \"rainfall amount\",\n",
" \"stratiform graupel fall amount\",\n",
" \"solid precipitation flux containing single 2H\",\n",
" \"prsn2h\",\n",
" \"Precipitation Flux of Snow and Ice Containing Deuterium (1H 2H O)\",\n",
" \"tendency of air temperature due to stratiform precipitation\",\n",
" \"tendency of air temperature due to large scale precipitation\",\n",
" \"convective precipitation flux\",\n",
" \"prc\",\n",
" \"Convective Precipitation\",\n",
" \"mass fraction of rainfall falling onto surface snow\",\n",
" \"prrsn\",\n",
" \"Fraction of Rainfall on Snow\"\n",
" ]\n",
"}\n",
"New query: yearly \n",
"\n",
"V3 structured query: \n",
"{'query': 'yearly precipitation over Ottawa between 2000 and 2020',\n",
Expand Down Expand Up @@ -436,6 +353,9 @@
}
],
"source": [
"# if desired, verbosity can be modified here\n",
"# if enabled, each pipeline step will output intermediate results\n",
"nlu.verbose = False\n",
"# run the NLU pipeline on the natural language query\n",
"nlu.nl2query()"
]
Expand Down Expand Up @@ -480,7 +400,7 @@
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0cb50e40a66841c2b1ab0c9bf0560397",
"model_id": "dc89304d9f1344ce9a45983870d59451",
"version_major": 2,
"version_minor": 0
},
Expand Down Expand Up @@ -511,11 +431,43 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Created STAC query with the parameters:\n",
" {'bbox': [45.5376502, 44.9617738, -75.2465783, -76.3555857], 'datetime': ['2000-01-01T00:00:00Z', '2021-01-01T00:00:00Z'], 'query': ['precipitation amount', 'stratiform precipitation amount', 'large scale precipitation amount', 'stratiform precipitation flux', 'large scale precipitation flux', 'stratiform rainfall amount', 'large scale rainfall amount', 'solid precipitation flux', 'stratiform rainfall rate', 'large scale rainfall rate', 'convective precipitation amount', 'lwe stratiform precipitation rate', 'lwe large scale precipitation rate', 'tendency of specific humidity due to stratiform precipitation', 'tendency of specific humidity due to large scale precipitation', 'rainfall amount', 'stratiform graupel fall amount', 'solid precipitation flux containing single 2H', 'prsn2h', 'Precipitation Flux of Snow and Ice Containing Deuterium (1H 2H O)', 'tendency of air temperature due to stratiform precipitation', 'tendency of air temperature due to large scale precipitation', 'convective precipitation flux', 'prc', 'Convective Precipitation', 'mass fraction of rainfall falling onto surface snow', 'prrsn', 'Fraction of Rainfall on Snow'], 'collections': []}\n",
"Created STAC query with parameters:\n",
"\n",
"{'bbox': (-76.3555857, 44.9617738, -75.2465783, 45.5376502),\n",
" 'datetime': ['2000-01-01T00:00:00Z', '2021-01-01T00:00:00Z'],\n",
" 'query': ['precipitation amount',\n",
" 'stratiform precipitation amount',\n",
" 'large scale precipitation amount',\n",
" 'stratiform precipitation flux',\n",
" 'large scale precipitation flux',\n",
" 'stratiform rainfall amount',\n",
" 'large scale rainfall amount',\n",
" 'solid precipitation flux',\n",
" 'stratiform rainfall rate',\n",
" 'large scale rainfall rate',\n",
" 'convective precipitation amount',\n",
" 'lwe stratiform precipitation rate',\n",
" 'lwe large scale precipitation rate',\n",
" 'tendency of specific humidity due to stratiform precipitation',\n",
" 'tendency of specific humidity due to large scale precipitation',\n",
" 'rainfall amount',\n",
" 'stratiform graupel fall amount',\n",
" 'solid precipitation flux containing single 2H',\n",
" 'prsn2h',\n",
" 'Precipitation Flux of Snow and Ice Containing Deuterium (1H 2H O)',\n",
" 'tendency of air temperature due to stratiform precipitation',\n",
" 'tendency of air temperature due to large scale precipitation',\n",
" 'convective precipitation flux',\n",
" 'prc',\n",
" 'Convective Precipitation',\n",
" 'mass fraction of rainfall falling onto surface snow',\n",
" 'prrsn',\n",
" 'Fraction of Rainfall on Snow'],\n",
" 'collections': []}\n",
"\n",
"Opening catalog: Earth Search by Element 84\n",
"Searching catalog: https://earth-search.aws.element84.com/v1\n",
"QUERY: {'url': 'https://earth-search.aws.element84.com/v1/search', 'client': <Client id=earth-search-aws>, '_stac_io': <pystac_client.stac_api_io.StacApiIO object at 0x7ff403cc7700>, '_max_items': 10, 'method': 'GET', 'modifier': None, '_parameters': {'bbox': (-76.3555857, 44.9617738, -75.2465783, 45.5376502), 'datetime': '2000-01-01T00:00:00Z/2021-01-01T00:00:00Z', 'collections': (), 'query': {}}}\n"
"QUERY: {'url': 'https://earth-search.aws.element84.com/v1/search', 'client': <Client id=earth-search-aws>, '_stac_io': <pystac_client.stac_api_io.StacApiIO object at 0x7fda3442eb80>, '_max_items': 10, 'method': 'GET', 'modifier': None, '_parameters': {'bbox': (-76.3555857, 44.9617738, -75.2465783, 45.5376502), 'datetime': '2000-01-01T00:00:00Z/2021-01-01T00:00:00Z', 'collections': (), 'query': {}}}\n"
]
},
{
Expand Down Expand Up @@ -11909,6 +11861,8 @@
}
],
"source": [
"# verbosity can be modified here to visualize the converted NL query to STAC query\n",
"nlu.verbose = True\n",
"nlu.run_stac_query()"
]
},
Expand Down
5 changes: 5 additions & 0 deletions nlp/notebooks/nlu_demo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,11 @@ def select_stac_catalog(self, catalog: Optional[str] = None):
def run_stac_query(self):
if self.struct_query:
return self.stac_handler.handle_query(self.struct_query, verbose=self.verbose)
else:
print(
"NL structured query is not set! Cannot run STAC query. "
"Make sure that nl2query operations were performed first."
)

def run_custom_stac_query(self, params):
return self.stac_handler.search_query(params=params)
9 changes: 6 additions & 3 deletions nlp/notebooks/stac_wrapper/query_handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import json
import os
from configparser import ConfigParser
from pprint import pprint
from typing import List, Optional

import ipywidgets as widgets
Expand Down Expand Up @@ -129,8 +130,12 @@ def query2stac(self, struct_query: JSON, verbose: bool = False) -> JSON:
params['query'] = [prop_string]
elif annotation['type'] == 'target':
params['query'] = annotation['name']
if params["bbox"]:
params["bbox"] = self.convert_bbox_nlquery2stac(params["bbox"])
if verbose:
print("Created STAC query with the parameters:\n", params)
print("Created STAC query with parameters:\n")
pprint(params, sort_dicts=False)
print("")
return params

@staticmethod
Expand Down Expand Up @@ -184,6 +189,4 @@ def handle_query(self, struct_query: JSON, verbose: bool = False) -> Optional[Vi
Returns result items or None.
"""
params = self.query2stac(struct_query, verbose)
if params["bbox"]:
params["bbox"] = self.convert_bbox_nlquery2stac(params["bbox"])
return self.search_query(params, verbose)

0 comments on commit c885432

Please sign in to comment.