Skip to content

Commit

Permalink
fix initial ingest
Browse files Browse the repository at this point in the history
  • Loading branch information
lcouzens committed Jun 28, 2024
1 parent fea633c commit 86e0b93
Showing 1 changed file with 15 additions and 7 deletions.
22 changes: 15 additions & 7 deletions koku/masu/processor/orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,16 +78,24 @@ def check_currently_processing(schema, provider):
process_wait_delta = datetime.now(tz=settings.UTC) - timedelta(days=settings.PROCESSING_WAIT_TIMER)
if is_customer_large(schema):
process_wait_delta = datetime.now(tz=settings.UTC) - timedelta(days=settings.LARGE_PROCESSING_WAIT_TIMER)
# Fallback to creation timestamp if its a new provider
check_timestamp = (
provider.data_updated_timestamp if provider.data_updated_timestamp else provider.created_timestamp
)
# Check processing, if polling timestamp more recent than updated timestamp skip polling
if provider.polling_timestamp > check_timestamp:
if provider.data_updated_timestamp:
if provider.polling_timestamp > provider.data_updated_timestamp:
result = True
# Check failed processing, if updated timestamp not updated in x days we should polling again
if process_wait_delta > provider.data_updated_timestamp:
result = False
# Fallback to creation timestamp if its a new provider
else:
# Dont trigger provider that has no updated timestamp
result = True

Check warning on line 91 in koku/masu/processor/orchestrator.py

View check run for this annotation

Codecov / codecov/patch

koku/masu/processor/orchestrator.py#L91

Added line #L91 was not covered by tests
# Check failed processing, if updated timestamp not updated in x days we should polling again
if process_wait_delta > check_timestamp:
# Enable initial ingest for new providers
if datetime.now(tz=settings.UTC) - timedelta(days=1) < provider.created_timestamp:
result = False

Check warning on line 94 in koku/masu/processor/orchestrator.py

View check run for this annotation

Codecov / codecov/patch

koku/masu/processor/orchestrator.py#L94

Added line #L94 was not covered by tests
# Reprocess new providers that may have fialed to complete their first download
if process_wait_delta > provider.created_timestamp:
result = False

Check warning on line 97 in koku/masu/processor/orchestrator.py

View check run for this annotation

Codecov / codecov/patch

koku/masu/processor/orchestrator.py#L97

Added line #L97 was not covered by tests

return result


Expand Down

0 comments on commit 86e0b93

Please sign in to comment.