Skip to content

Commit 7dad3bb

Browse files
lrafeeimergify[bot]hmstepanek
authored
Initial Hybrid Agent Trace implementation (#1587)
* Initial otel trace/span implementation * Add some hybrid cross agent tests * Tweak some formatting and merge issues * [MegaLinter] Apply linters fixes * Fix application initialization bug * [MegaLinter] Apply linters fixes * Reviewer suggestions, part 1 * [MegaLinter] Apply linters fixes * Fix hasattr syntax * More reviewer suggestions/syntax fixes * [MegaLinter] Apply linters fixes * Apply suggestions from code review Co-authored-by: Hannah Stepanek <[email protected]> * Reviewer suggestions, part 2 * Add fixture & enable/disable testing * Log potential error instead of raising error * Apply suggestions from code review Co-authored-by: Hannah Stepanek <[email protected]> * Update newrelic/api/opentelemetry.py Co-authored-by: Hannah Stepanek <[email protected]> --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Hannah Stepanek <[email protected]>
1 parent afbc9f2 commit 7dad3bb

File tree

10 files changed

+959
-2
lines changed

10 files changed

+959
-2
lines changed

‎newrelic/api/opentelemetry.py‎

Lines changed: 439 additions & 0 deletions
Large diffs are not rendered by default.

‎newrelic/config.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -691,6 +691,7 @@ def _process_configuration(section):
691691
_process_setting(section, "instrumentation.middleware.django.enabled", "getboolean", None)
692692
_process_setting(section, "instrumentation.middleware.django.exclude", "get", _map_inc_excl_middleware)
693693
_process_setting(section, "instrumentation.middleware.django.include", "get", _map_inc_excl_middleware)
694+
_process_setting(section, "otel_bridge.enabled", "getboolean", None)
694695

695696

696697
# Loading of configuration from specified file and for specified
@@ -4367,6 +4368,15 @@ def _process_module_builtin_defaults():
43674368
"pyzeebe.worker.job_executor", "newrelic.hooks.external_pyzeebe", "instrument_pyzeebe_worker_job_executor"
43684369
)
43694370

4371+
# Hybrid Agent Hooks
4372+
_process_module_definition(
4373+
"opentelemetry.trace", "newrelic.hooks.hybridagent_opentelemetry", "instrument_trace_api"
4374+
)
4375+
4376+
_process_module_definition(
4377+
"opentelemetry.instrumentation.utils", "newrelic.hooks.hybridagent_opentelemetry", "instrument_utils"
4378+
)
4379+
43704380

43714381
def _process_module_entry_points():
43724382
try:

‎newrelic/core/config.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,10 @@ class EventHarvestConfigHarvestLimitSettings(Settings):
534534
nested = True
535535

536536

537+
class OtelBridgeSettings(Settings):
538+
pass
539+
540+
537541
_settings = TopLevelSettings()
538542
_settings.agent_limits = AgentLimitsSettings()
539543
_settings.application_logging = ApplicationLoggingSettings()
@@ -620,6 +624,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings):
620624
_settings.instrumentation.middleware = InstrumentationMiddlewareSettings()
621625
_settings.instrumentation.middleware.django = InstrumentationDjangoMiddlewareSettings()
622626
_settings.message_tracer = MessageTracerSettings()
627+
_settings.otel_bridge = OtelBridgeSettings()
623628
_settings.process_host = ProcessHostSettings()
624629
_settings.rum = RumSettings()
625630
_settings.serverless_mode = ServerlessModeSettings()
@@ -1267,6 +1272,7 @@ def default_otlp_host(host):
12671272
_settings.azure_operator.enabled = _environ_as_bool("NEW_RELIC_AZURE_OPERATOR_ENABLED", default=False)
12681273
_settings.package_reporting.enabled = _environ_as_bool("NEW_RELIC_PACKAGE_REPORTING_ENABLED", default=True)
12691274
_settings.ml_insights_events.enabled = _environ_as_bool("NEW_RELIC_ML_INSIGHTS_EVENTS_ENABLED", default=False)
1275+
_settings.otel_bridge.enabled = _environ_as_bool("NEW_RELIC_OTEL_BRIDGE_ENABLED", default=False)
12701276

12711277

12721278
def global_settings():

‎newrelic/core/otlp_utils.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,9 @@ def create_key_values_from_iterable(iterable):
116116
return list(filter(lambda i: i is not None, (create_key_value(key, value) for key, value in iterable)))
117117

118118

119-
def create_resource(attributes=None, attach_apm_entity=True):
120-
attributes = attributes or {"instrumentation.provider": "newrelic-opentelemetry-python-ml"}
119+
def create_resource(attributes=None, attach_apm_entity=True, hybrid_bridge=False):
120+
instrumentation_provider = "newrelic-opentelemetry-bridge" if hybrid_bridge else "newrelic-opentelemetry-python-ml"
121+
attributes = attributes or {"instrumentation.provider": instrumentation_provider}
121122
if attach_apm_entity:
122123
metadata = get_service_linking_metadata()
123124
attributes.update(metadata)

‎newrelic/hooks/hybridagent_opentelemetry.py‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import logging
16+
17+
from newrelic.api.application import application_instance
18+
from newrelic.api.time_trace import add_custom_span_attribute, current_trace
19+
from newrelic.api.transaction import Sentinel, current_transaction
20+
from newrelic.common.object_wrapper import wrap_function_wrapper
21+
from newrelic.common.signature import bind_args
22+
from newrelic.core.config import global_settings
23+
24+
_logger = logging.getLogger(__name__)
25+
_TRACER_PROVIDER = None
26+
27+
###########################################
28+
# Trace Instrumentation
29+
###########################################
30+
31+
32+
def wrap_set_tracer_provider(wrapped, instance, args, kwargs):
33+
settings = global_settings()
34+
35+
if not settings.otel_bridge.enabled:
36+
return wrapped(*args, **kwargs)
37+
38+
global _TRACER_PROVIDER
39+
40+
if _TRACER_PROVIDER is None:
41+
bound_args = bind_args(wrapped, args, kwargs)
42+
tracer_provider = bound_args.get("tracer_provider")
43+
_TRACER_PROVIDER = tracer_provider
44+
else:
45+
_logger.warning("TracerProvider has already been set.")
46+
47+
48+
def wrap_get_tracer_provider(wrapped, instance, args, kwargs):
49+
settings = global_settings()
50+
51+
if not settings.otel_bridge.enabled:
52+
return wrapped(*args, **kwargs)
53+
54+
# This needs to act as a singleton, like the agent instance.
55+
# We should initialize the agent here as well, if there is
56+
# not an instance already.
57+
application = application_instance(activate=False)
58+
if not application or (application and not application.active):
59+
application_instance().activate()
60+
61+
global _TRACER_PROVIDER
62+
63+
if _TRACER_PROVIDER is None:
64+
from newrelic.api.opentelemetry import TracerProvider
65+
66+
hybrid_agent_tracer_provider = TracerProvider("hybrid_agent_tracer_provider")
67+
_TRACER_PROVIDER = hybrid_agent_tracer_provider
68+
return _TRACER_PROVIDER
69+
70+
71+
def wrap_get_current_span(wrapped, instance, args, kwargs):
72+
transaction = current_transaction()
73+
trace = current_trace()
74+
75+
# If a NR trace does not exist (aside from the Sentinel
76+
# trace), return the original function's result.
77+
if not transaction or isinstance(trace, Sentinel):
78+
return wrapped(*args, **kwargs)
79+
80+
# Do not allow the wrapper to continue if
81+
# the Hybrid Agent setting is not enabled
82+
settings = transaction.settings or global_settings()
83+
84+
if not settings.otel_bridge.enabled:
85+
return wrapped(*args, **kwargs)
86+
87+
# If a NR trace does exist, check to see if the current
88+
# OTel span corresponds to the current NR trace. If so,
89+
# return the original function's result.
90+
span = wrapped(*args, **kwargs)
91+
92+
if span.get_span_context().span_id == int(trace.guid, 16):
93+
return span
94+
95+
# If the current OTel span does not match the current NR
96+
# trace, this means that a NR trace was created either
97+
# manually or through the NR agent. Either way, the OTel
98+
# API was not used to create a span object. The Hybrid
99+
# Agent's Span object creates a NR trace but since the NR
100+
# trace has already been created, we just need a symbolic
101+
# OTel span to represent it the span object. A LazySpan
102+
# will be created. It will effectively be a NonRecordingSpan
103+
# with the ability to add custom attributes.
104+
105+
from opentelemetry import trace as otel_api_trace
106+
107+
class LazySpan(otel_api_trace.NonRecordingSpan):
108+
def set_attribute(self, key, value):
109+
add_custom_span_attribute(key, value)
110+
111+
def set_attributes(self, attributes):
112+
for key, value in attributes.items():
113+
add_custom_span_attribute(key, value)
114+
115+
otel_tracestate_headers = None
116+
117+
span_context = otel_api_trace.SpanContext(
118+
trace_id=int(transaction.trace_id, 16),
119+
span_id=int(trace.guid, 16),
120+
is_remote=span.get_span_context().is_remote,
121+
trace_flags=otel_api_trace.TraceFlags(span.get_span_context().trace_flags),
122+
trace_state=otel_api_trace.TraceState(otel_tracestate_headers),
123+
)
124+
125+
return LazySpan(span_context)
126+
127+
128+
def wrap_start_internal_or_server_span(wrapped, instance, args, kwargs):
129+
# We want to take the NR version of the context_carrier
130+
# and put that into the attributes. Keep the original
131+
# context_carrier intact.
132+
133+
# Do not allow the wrapper to continue if
134+
# the Hybrid Agent setting is not enabled
135+
settings = global_settings()
136+
137+
if not settings.otel_bridge.enabled:
138+
return wrapped(*args, **kwargs)
139+
140+
bound_args = bind_args(wrapped, args, kwargs)
141+
context_carrier = bound_args.get("context_carrier")
142+
attributes = bound_args.get("attributes", {})
143+
144+
if context_carrier:
145+
if ("HTTP_HOST" in context_carrier) or ("http_version" in context_carrier):
146+
# This is an HTTP request (WSGI, ASGI, or otherwise)
147+
nr_environ = context_carrier.copy()
148+
attributes["nr.http.headers"] = nr_environ
149+
150+
else:
151+
nr_headers = context_carrier.copy()
152+
attributes["nr.nonhttp.headers"] = nr_headers
153+
154+
bound_args["attributes"] = attributes
155+
156+
return wrapped(**bound_args)
157+
158+
159+
def instrument_trace_api(module):
160+
if hasattr(module, "set_tracer_provider"):
161+
wrap_function_wrapper(module, "set_tracer_provider", wrap_set_tracer_provider)
162+
163+
if hasattr(module, "get_tracer_provider"):
164+
wrap_function_wrapper(module, "get_tracer_provider", wrap_get_tracer_provider)
165+
166+
if hasattr(module, "get_current_span"):
167+
wrap_function_wrapper(module, "get_current_span", wrap_get_current_span)
168+
169+
170+
def instrument_utils(module):
171+
if hasattr(module, "_start_internal_or_server_span"):
172+
wrap_function_wrapper(module, "_start_internal_or_server_span", wrap_start_internal_or_server_span)

‎tests/hybridagent_opentelemetry/conftest.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import pytest
16+
from opentelemetry import trace
17+
from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture
18+
19+
from newrelic.api.opentelemetry import TracerProvider
20+
21+
_default_settings = {
22+
"package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs.
23+
"transaction_tracer.explain_threshold": 0.0,
24+
"transaction_tracer.transaction_threshold": 0.0,
25+
"transaction_tracer.stack_trace_threshold": 0.0,
26+
"debug.log_data_collector_payloads": True,
27+
"debug.record_transaction_failure": True,
28+
"otel_bridge.enabled": True,
29+
}
30+
31+
collector_agent_registration = collector_agent_registration_fixture(
32+
app_name="Python Agent Test (Hybrid Agent)", default_settings=_default_settings
33+
)
34+
35+
@pytest.fixture(scope="session")
36+
def tracer():
37+
trace_provider = TracerProvider()
38+
trace.set_tracer_provider(trace_provider)
39+
40+
return trace.get_tracer(__name__)

‎tests/hybridagent_opentelemetry/test_hybrid_cross_agent.py‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from opentelemetry import trace as otel_api_trace
16+
from testing_support.validators.validate_error_event_attributes import validate_error_event_attributes
17+
from testing_support.validators.validate_span_events import validate_span_events
18+
from testing_support.validators.validate_transaction_count import validate_transaction_count
19+
from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics
20+
21+
from newrelic.api.application import application_instance
22+
from newrelic.api.background_task import BackgroundTask
23+
from newrelic.api.function_trace import FunctionTrace
24+
from newrelic.api.time_trace import current_trace
25+
from newrelic.api.transaction import current_transaction
26+
27+
28+
# Does not create segment without a transaction
29+
@validate_transaction_count(0)
30+
def test_does_not_create_segment_without_a_transaction(tracer):
31+
with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.INTERNAL):
32+
# The OpenTelmetry span should not be created
33+
assert otel_api_trace.get_current_span() == otel_api_trace.INVALID_SPAN
34+
35+
# There should be no transaction
36+
assert not current_transaction()
37+
38+
39+
# Creates OpenTelemetry segment in a transaction
40+
@validate_transaction_metrics(name="Foo", background_task=True)
41+
@validate_span_events(
42+
exact_intrinsics={"name": "Function/Bar", "category": "generic"}, expected_intrinsics=("parentId",)
43+
)
44+
@validate_span_events(exact_intrinsics={"name": "Function/Foo", "category": "generic", "nr.entryPoint": True})
45+
def test_creates_opentelemetry_segment_in_a_transaction(tracer):
46+
application = application_instance(activate=False)
47+
48+
with BackgroundTask(application, name="Foo"):
49+
with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.INTERNAL):
50+
# OpenTelemetry API and New Relic API report the same traceId
51+
assert otel_api_trace.get_current_span().get_span_context().trace_id == int(
52+
current_transaction()._trace_id, 16
53+
)
54+
55+
# OpenTelemetry API and New Relic API report the same spanId
56+
assert otel_api_trace.get_current_span().get_span_context().span_id == int(current_trace().guid, 16)
57+
58+
59+
# Creates New Relic span as child of OpenTelemetry span
60+
@validate_transaction_metrics(name="Foo", background_task=True)
61+
@validate_span_events(
62+
exact_intrinsics={"name": "Function/Baz", "category": "generic"}, expected_intrinsics=("parentId",)
63+
)
64+
@validate_span_events(
65+
exact_intrinsics={"name": "Function/Bar", "category": "generic"}, expected_intrinsics=("parentId",)
66+
)
67+
@validate_span_events(exact_intrinsics={"name": "Function/Foo", "category": "generic"})
68+
def test_creates_new_relic_span_as_child_of_open_telemetry_span(tracer):
69+
application = application_instance(activate=False)
70+
71+
with BackgroundTask(application, name="Foo"):
72+
with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.INTERNAL):
73+
with FunctionTrace(name="Baz"):
74+
# OpenTelemetry API and New Relic API report the same traceId
75+
assert otel_api_trace.get_current_span().get_span_context().trace_id == int(
76+
current_transaction().trace_id, 16
77+
)
78+
79+
# OpenTelemetry API and New Relic API report the same spanId
80+
assert otel_api_trace.get_current_span().get_span_context().span_id == int(current_trace().guid, 16)
81+
82+
83+
# OpenTelemetry API can add custom attributes to spans
84+
@validate_transaction_metrics(name="Foo", background_task=True)
85+
@validate_span_events(exact_intrinsics={"name": "Function/Baz"}, exact_users={"spanNumber": 2})
86+
@validate_span_events(exact_intrinsics={"name": "Function/Bar"}, exact_users={"spanNumber": 1})
87+
def test_opentelemetry_api_can_add_custom_attributes_to_spans(tracer):
88+
application = application_instance(activate=False)
89+
90+
with BackgroundTask(application, name="Foo"):
91+
with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.INTERNAL):
92+
with FunctionTrace(name="Baz"):
93+
otel_api_trace.get_current_span().set_attribute("spanNumber", 2)
94+
95+
otel_api_trace.get_current_span().set_attribute("spanNumber", 1)
96+
97+
98+
# OpenTelemetry API can record errors
99+
@validate_transaction_metrics(name="Foo", background_task=True)
100+
@validate_error_event_attributes(
101+
exact_attrs={"agent": {}, "intrinsic": {"error.message": "Test exception message"}, "user": {}}
102+
)
103+
@validate_span_events(exact_intrinsics={"name": "Function/Bar"})
104+
def test_opentelemetry_api_can_record_errors(tracer):
105+
application = application_instance(activate=False)
106+
107+
with BackgroundTask(application, name="Foo"):
108+
with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.INTERNAL):
109+
try:
110+
raise Exception("Test exception message")
111+
except Exception as e:
112+
otel_api_trace.get_current_span().record_exception(e)

0 commit comments

Comments
 (0)