Skip to content

Commit 6797366

Browse files
committed
Merge remote-tracking branch 'origin/develop' into merge-dev-to-main
2 parents f45c26b + 404c2b5 commit 6797366

File tree

19 files changed

+296
-616
lines changed

19 files changed

+296
-616
lines changed

.github/actions/cdk-deploy/action.yml

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -66,13 +66,6 @@ runs:
6666
working-directory: ${{ inputs.dir }}
6767
run: docker compose up --build -d
6868

69-
- name: Ingest Stac Items/Collection
70-
if: ${{ inputs.skip_tests == false }}
71-
shell: bash
72-
working-directory: ${{ inputs.dir }}
73-
run: |
74-
./scripts/load-data-container.sh
75-
7669
- name: Sleep for 10 seconds
7770
if: ${{ inputs.skip_tests == false }}
7871
shell: bash

.github/workflows/data/noaa-emergency-response.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
"interval": [
2121
[
2222
"2005-01-01T00:00:00Z",
23-
"null"
23+
null
2424
]
2525
]
2626
}

.github/workflows/pr.yml

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,6 @@ jobs:
6363
- name: Launch services
6464
run: AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY=${{secrets.AWS_SECRET_ACCESS_KEY}} docker compose up --build -d
6565

66-
- name: Ingest Stac Items/Collection
67-
run: |
68-
./scripts/load-data-container.sh
69-
7066
- name: Sleep for 10 seconds
7167
run: sleep 10s
7268
shell: bash
@@ -83,7 +79,7 @@ jobs:
8379
- name: Ingest unit tests
8480
run: NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest ingest_api/runtime/tests/ -vv -s
8581

86-
- name: Stac-api transactions unit tests
82+
- name: Stac-api extensions unit tests
8783
run: python -m pytest stac_api/runtime/tests/ --asyncio-mode=auto -vv -s
8884

8985
- name: Stop services

database/runtime/handler.py

Lines changed: 1 addition & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -145,96 +145,6 @@ def create_dashboard_schema(cursor, username: str) -> None:
145145
)
146146

147147

148-
def create_collection_search_functions(cursor) -> None:
149-
"""Create custom functions for collection-level search."""
150-
151-
search_collection_ids_sql = """
152-
CREATE OR REPLACE FUNCTION pgstac.collection_id_search(_search jsonb = '{}'::jsonb) RETURNS SETOF text AS $$
153-
DECLARE
154-
searches searches%ROWTYPE;
155-
_where text;
156-
token_where text;
157-
full_where text;
158-
orderby text;
159-
query text;
160-
token_type text := substr(_search->>'token',1,4);
161-
curs refcursor;
162-
iter_record items%ROWTYPE;
163-
prev_query text;
164-
next text;
165-
prev_id text;
166-
has_next boolean := false;
167-
has_prev boolean := false;
168-
prev text;
169-
total_count bigint;
170-
context jsonb;
171-
includes text[];
172-
excludes text[];
173-
exit_flag boolean := FALSE;
174-
batches int := 0;
175-
timer timestamptz := clock_timestamp();
176-
pstart timestamptz;
177-
pend timestamptz;
178-
pcurs refcursor;
179-
search_where search_wheres%ROWTYPE;
180-
id text;
181-
collections text[];
182-
BEGIN
183-
CREATE TEMP TABLE results (collection text, content jsonb) ON COMMIT DROP;
184-
-- if ids is set, short circuit and just use direct ids query for each id
185-
-- skip any paging or caching
186-
-- hard codes ordering in the same order as the array of ids
187-
searches := search_query(_search);
188-
_where := searches._where;
189-
orderby := searches.orderby;
190-
search_where := where_stats(_where);
191-
total_count := coalesce(search_where.total_count, search_where.estimated_count);
192-
193-
IF token_type='prev' THEN
194-
token_where := get_token_filter(_search, null::jsonb);
195-
orderby := sort_sqlorderby(_search, TRUE);
196-
END IF;
197-
IF token_type='next' THEN
198-
token_where := get_token_filter(_search, null::jsonb);
199-
END IF;
200-
201-
full_where := concat_ws(' AND ', _where, token_where);
202-
RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer;
203-
timer := clock_timestamp();
204-
205-
FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions)
206-
LOOP
207-
timer := clock_timestamp();
208-
RAISE NOTICE 'Partition Query: %', query;
209-
batches := batches + 1;
210-
-- curs = create_cursor(query);
211-
RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction');
212-
OPEN curs FOR EXECUTE query;
213-
LOOP
214-
FETCH curs into iter_record;
215-
EXIT WHEN NOT FOUND;
216-
217-
INSERT INTO results (collection, content) VALUES (iter_record.collection, iter_record.content);
218-
219-
END LOOP;
220-
CLOSE curs;
221-
RAISE NOTICE 'Query took %.', clock_timestamp()-timer;
222-
timer := clock_timestamp();
223-
EXIT WHEN exit_flag;
224-
END LOOP;
225-
RAISE NOTICE 'Scanned through % partitions.', batches;
226-
227-
RETURN QUERY SELECT DISTINCT collection FROM results WHERE content is not NULL;
228-
229-
DROP TABLE results;
230-
231-
RETURN;
232-
END;
233-
$$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1;
234-
"""
235-
cursor.execute(sql.SQL(search_collection_ids_sql))
236-
237-
238148
def create_collection_extents_functions(cursor) -> None:
239149
"""
240150
Functions to update spatial and temporal extents off all items in a collection
@@ -451,12 +361,7 @@ def handler(event, context):
451361
)
452362
)
453363

454-
# As admin, create custom search functions
455-
with psycopg.connect(stac_db_conninfo, autocommit=True) as conn:
456-
with conn.cursor() as cur:
457-
print("Creating custom STAC search functions...")
458-
create_collection_search_functions(cursor=cur)
459-
364+
# As admin, create custom dashboard functions
460365
with psycopg.connect(stac_db_conninfo, autocommit=True) as conn:
461366
with conn.cursor() as cur:
462367
print("Creating dashboard schema...")

docker-compose.yml

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
version: '3'
2-
31
services:
42
stac:
53
container_name: veda.stac
@@ -32,6 +30,7 @@ services:
3230
# https://github.com/developmentseed/eoAPI/issues/16
3331
# - TITILER_ENDPOINT=raster
3432
- TITILER_ENDPOINT=http://0.0.0.0:8082
33+
- VEDA_STAC_ENABLE_TRANSACTIONS=True
3534
depends_on:
3635
- database
3736
- raster
@@ -97,7 +96,7 @@ services:
9796
database:
9897
container_name: veda.db
9998
platform: linux/amd64
100-
image: ghcr.io/stac-utils/pgstac:v0.7.10
99+
image: ghcr.io/stac-utils/pgstac:v0.9.6
101100
environment:
102101
- POSTGRES_USER=username
103102
- POSTGRES_PASSWORD=password
@@ -106,14 +105,31 @@ services:
106105
- PGPASSWORD=password
107106
- PGDATABASE=postgis
108107
ports:
109-
- "5432:5432"
108+
- "5439:5432"
110109
command: postgres -N 500
111110
volumes:
112111
- ./scripts:/tmp/scripts
113112
- ./.github/workflows/data:/tmp/data
114113
# broken in github actions (definitely when run in act, possibly in tests involving ingest-api)? re-enable to persist local database
115114
# - ./.pgdata:/var/lib/postgresql/data
116115

116+
pypgstac:
117+
container_name: veda.loadtestdata
118+
platform: linux/amd64
119+
image: ghcr.io/stac-utils/pgstac-pypgstac:v0.9.6
120+
environment:
121+
- PGUSER=username
122+
- PGPASSWORD=password
123+
- PGDATABASE=postgis
124+
- PGHOST=database
125+
- PGPORT=5432
126+
depends_on:
127+
- database
128+
command: bash -c "sleep 10 && /tmp/scripts/bin/load-data.sh"
129+
volumes:
130+
- ./scripts:/tmp/scripts
131+
- ./.github/workflows/data:/tmp/data
132+
117133
ingestor:
118134
container_name: veda.ingestor
119135
platform: linux/amd64
@@ -128,7 +144,7 @@ services:
128144
- PGPASSWORD=password
129145
- PGDATABASE=postgis
130146
- DYNAMODB_ENDPOINT=http://localhost:8085
131-
- VEDA_DB_PGSTAC_VERSION=0.7.10
147+
- VEDA_DB_PGSTAC_VERSION=0.9.6
132148
- AWS_REGION=us-west-2
133149
- AWS_DEFAULT_REGION=us-west-2
134150
- DYNAMODB_TABLE=veda

scripts/bin/load-data.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,17 @@
33
set -e
44

55
# Wait for the database to start
6-
until pypgstac pgready --dsn postgresql://username:[email protected]:5432/postgis; do
6+
until pypgstac pgready; do
77
echo "Waiting for database to start..."
88
sleep 1
99
done
1010

1111
# Load collections
1212
echo "Loading collections..."
13-
pypgstac load collections /tmp/data/noaa-emergency-response.json --dsn postgresql://username:[email protected]:5432/postgis --method upsert
13+
pypgstac load collections /tmp/data/noaa-emergency-response.json --method upsert
1414

1515
# Load items
1616
echo "Loading items..."
17-
pypgstac load items /tmp/data/noaa-eri-nashville2020.json --dsn postgresql://username:[email protected]:5432/postgis --method upsert
17+
pypgstac load items /tmp/data/noaa-eri-nashville2020.json --method upsert
1818

1919
echo "Data loaded successfully!"

scripts/load-data-container.sh

Lines changed: 0 additions & 6 deletions
This file was deleted.

scripts/run-local-tests.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ set -e
55
pre-commit run --all-files
66

77
# Bring up stack for testing; ingestor not required
8-
docker compose up -d stac raster database dynamodb
8+
docker compose up -d --wait stac raster database dynamodb pypgstac
99

1010
# cleanup, logging in case of failure
1111
cleanup() {
@@ -26,7 +26,7 @@ cleanup() {
2626
trap cleanup EXIT
2727

2828
# Load data for tests
29-
docker exec veda.db /tmp/scripts/bin/load-data.sh
29+
docker exec veda.loadtestdata /tmp/scripts/bin/load-data.sh
3030

3131
# Run tests
3232
python -m pytest .github/workflows/tests/ -vv -s
@@ -35,4 +35,4 @@ python -m pytest .github/workflows/tests/ -vv -s
3535
NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest --cov=ingest_api/runtime/src ingest_api/runtime/tests/ -vv -s
3636

3737
# Transactions tests
38-
python -m pytest stac_api/runtime/tests/ --asyncio-mode=auto -vv -s
38+
python -m pytest stac_api/runtime/tests/ --asyncio-mode=auto -vv -s

stac_api/runtime/setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"stac-fastapi.api~=5.0",
1313
"stac-fastapi.types~=5.0",
1414
"stac-fastapi.extensions~=5.0",
15-
"stac-fastapi.pgstac~=5.0",
15+
"stac-fastapi.pgstac>=5.0.3,<6.0",
1616
"jinja2>=2.11.2,<4.0.0",
1717
"starlette-cramjam>=0.3.2,<0.4",
1818
"importlib_resources>=1.1.0;python_version<='3.11'", # https://github.com/cogeotiff/rio-tiler/pull/379

stac_api/runtime/src/api.py

Lines changed: 0 additions & 58 deletions
This file was deleted.

0 commit comments

Comments
 (0)