Skip to content

ref(storage-routing): make long term retention downsampling the default #8235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
phacops wants to merge 1 commit into
base: master
Choose a base branch
from
+6 −16
Open

ref(storage-routing): make long term retention downsampling the default #8235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions sentry-options/schemas/snuba/schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,11 +195,6 @@
"default": false,
"description": "When true, the storage routing strategy fetches ClickHouse cluster load info to inform routing decisions."
},
"enable_long_term_retention_downsampling": {
"type": "boolean",
"default": false,
"description": "When true, EAP queries older than 31 days (for non-full-retention item types) are routed to the most-downsampled tier."
},
"storage_routing_config_override": {
"type": "string",
"default": "{}",
Expand Down
8 changes: 2 additions & 6 deletions snuba/web/rpc/storage_routing/routing_strategies/outcomes_based.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
from snuba.query.logical import Query
from snuba.query.query_settings import OutcomesQuerySettings
from snuba.request import Request as SnubaRequest
from snuba.state.sentry_options import get_mapped_option, get_option
from snuba.state.sentry_options import get_mapped_option
from snuba.web.query import run_query
from snuba.web.rpc.common.common import (
timestamp_in_range_condition,
Expand Down Expand Up @@ -239,11 +239,7 @@ def _update_routing_decision(
thirty_one_days_ago_ts = int((datetime.now(tz=UTC) - timedelta(days=31)).timestamp())
older_than_thirty_days = thirty_one_days_ago_ts > in_msg_meta.start_timestamp.seconds

if (
get_option("enable_long_term_retention_downsampling", False)
and older_than_thirty_days
and in_msg_meta.trace_item_type not in ITEM_TYPE_FULL_RETENTION
):
if older_than_thirty_days and in_msg_meta.trace_item_type not in ITEM_TYPE_FULL_RETENTION:
routing_decision.tier = Tier.TIER_8

sentry_sdk.update_current_span(
Expand Down
9 changes: 4 additions & 5 deletions tests/web/rpc/v1/routing_strategies/test_outcomes_based.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,14 +91,15 @@ def test_outcomes_based_routing_queries_daily_table() -> None:

routing_decision = strategy.get_routing_decision(context)

assert routing_decision.tier == Tier.TIER_1
# the query window starts more than 31 days ago, so long term retention
# downsampling applies
assert routing_decision.tier == Tier.TIER_8
assert routing_decision.clickhouse_settings == {"max_threads": 10}
assert routing_decision.can_run


@pytest.mark.eap
@pytest.mark.redis_db
@override_options("snuba", {"enable_long_term_retention_downsampling": True})
def test_item_type_full_retention() -> None:
"""
Certain item types will not use the long term retention downsampling,
Expand Down Expand Up @@ -130,7 +131,6 @@ def test_item_type_full_retention() -> None:

@pytest.mark.eap
@pytest.mark.redis_db
@override_options("snuba", {"enable_long_term_retention_downsampling": True})
def test_item_type_full_retention_preprod() -> None:
"""
PREPROD item type should not use long term retention downsampling,
Expand Down Expand Up @@ -162,7 +162,6 @@ def test_item_type_full_retention_preprod() -> None:

@pytest.mark.eap
@pytest.mark.redis_db
@override_options("snuba", {"enable_long_term_retention_downsampling": True})
def test_outcomes_based_routing_sampled_data_past_thirty_days() -> None:
strategy = OutcomesBasedRoutingStrategy()
end_time = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)
Expand Down Expand Up @@ -198,7 +197,7 @@ def test_outcomes_based_routing_sampled_data_past_thirty_days() -> None:

# request(s) that query window of 30 minutes, but with timestamps 40 days ago
# one in MODE_NORMAL, one in MODE_HIGHEST_ACCURACY (which is ignored in favor of
# the enable_long_term_retention_downsampling)
# the long term retention downsampling)
start = datetime.now(tz=UTC) - timedelta(days=40, minutes=30)
end = datetime.now(tz=UTC) - timedelta(days=40)

Expand Down
Loading