From 72f2363db5f0ff3b3b9cfbd4dad99dd3347d819a Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Mon, 3 Feb 2020 15:55:23 -0800 Subject: [PATCH] Release for v0.7.7 (#853) --- CHANGELOG.md | 8 + contrib/opencensus-ext-azure/CHANGELOG.md | 10 + contrib/opencensus-ext-azure/README.rst | 151 +++++++- .../examples/logs/properties.py | 12 +- .../opencensus/ext/azure/common/__init__.py | 1 + .../opencensus/ext/azure/common/processor.py | 63 ++++ .../opencensus/ext/azure/common/utils.py | 10 - .../opencensus/ext/azure/common/version.py | 2 +- .../ext/azure/log_exporter/__init__.py | 27 +- .../ext/azure/metrics_exporter/__init__.py | 220 ++++------- .../ext/azure/trace_exporter/__init__.py | 52 ++- .../tests/test_azure_log_exporter.py | 95 ++++- .../tests/test_azure_metrics_exporter.py | 341 ++---------------- .../tests/test_azure_trace_exporter.py | 300 ++++++++++++++- .../tests/test_azure_utils.py | 7 - .../tests/test_processor.py | 94 +++++ contrib/opencensus-ext-httplib/CHANGELOG.md | 5 + .../opencensus/ext/httplib/trace.py | 4 + .../tests/test_httplib_trace.py | 3 +- contrib/opencensus-ext-httplib/version.py | 2 +- contrib/opencensus-ext-requests/CHANGELOG.md | 5 + .../opencensus/ext/requests/trace.py | 23 +- .../tests/test_requests_trace.py | 13 +- contrib/opencensus-ext-requests/version.py | 2 +- noxfile.py | 19 - 25 files changed, 915 insertions(+), 554 deletions(-) create mode 100644 contrib/opencensus-ext-azure/opencensus/ext/azure/common/processor.py create mode 100644 contrib/opencensus-ext-azure/tests/test_processor.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3218a8ebb..f83a01243 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +## 0.7.7 +Released 2020-02-03 +- Update `azure` module + ([#837](https://github.com/census-instrumentation/opencensus-python/pull/837), + [#845](https://github.com/census-instrumentation/opencensus-python/pull/845), + [#848](https://github.com/census-instrumentation/opencensus-python/pull/848), + [#851](https://github.com/census-instrumentation/opencensus-python/pull/851)) + ## 0.7.6 Released 2019-11-26 diff --git a/contrib/opencensus-ext-azure/CHANGELOG.md b/contrib/opencensus-ext-azure/CHANGELOG.md index 148905dce..72cb95326 100644 --- a/contrib/opencensus-ext-azure/CHANGELOG.md +++ b/contrib/opencensus-ext-azure/CHANGELOG.md @@ -2,6 +2,16 @@ ## Unreleased +## 1.0.2 +Released 2020-02-03 + +- Add local storage and retry logic for Azure Metrics Exporter + ([#845](https://github.com/census-instrumentation/opencensus-python/pull/845)) +- Add Fixed-rate sampling logic for Azure Log Exporter + ([#848](https://github.com/census-instrumentation/opencensus-python/pull/848)) +- Implement TelemetryProcessors for Azure exporters + ([#851](https://github.com/census-instrumentation/opencensus-python/pull/851)) + ## 1.0.1 Released 2019-11-26 diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 8de7a8049..49ba625f3 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -37,6 +37,8 @@ This example shows how to send a warning level log to Azure Monitor. logger.addHandler(AzureLogHandler(connection_string='InstrumentationKey=')) logger.warning('Hello, World!') +Correlation +########### You can enrich the logs with trace IDs and span IDs by using the `logging integration <../opencensus-ext-logging>`_. @@ -73,9 +75,12 @@ You can enrich the logs with trace IDs and span IDs by using the `logging integr logger.warning('In the span') logger.warning('After the span') -You can also add custom properties to your log messages in the form of key-values. +Custom Properties +################# -WARNING: For this feature to work, you need to pass a dictionary as the argument. If you pass arguments of any other type, the logger will ignore them. The solution is to convert these arguments into a dictionary. +You can also add custom properties to your log messages in the *extra* keyword argument using the custom_dimensions field. + +WARNING: For this feature to work, you need to pass a dictionary to the custom_dimensions field. If you pass arguments of any other type, the logger will ignore them. .. code:: python @@ -85,7 +90,37 @@ WARNING: For this feature to work, you need to pass a dictionary as the argument logger = logging.getLogger(__name__) logger.addHandler(AzureLogHandler(connection_string='InstrumentationKey=')) - logger.warning('action', {'key-1': 'value-1', 'key-2': 'value2'}) + + properties = {'custom_dimensions': {'key_1': 'value_1', 'key_2': 'value_2'}} + logger.warning('action', extra=properties) + +Modifying Logs +############## + +* You can pass a callback function to the exporter to process telemetry before it is exported. +* Your callback function can return `False` if you do not want this envelope exported. +* Your callback function must accept an [envelope](https://github.com/census-instrumentation/opencensus-python/blob/master/contrib/opencensus-ext-azure/opencensus/ext/azure/common/protocol.py#L86) data type as its parameter. +* You can see the schema for Azure Monitor data types in the envelopes [here](https://github.com/census-instrumentation/opencensus-python/blob/master/contrib/opencensus-ext-azure/opencensus/ext/azure/common/protocol.py). +* The `AzureLogHandler` handles `ExceptionData` and `MessageData` data types. + +.. code:: python + + import logging + + from opencensus.ext.azure.log_exporter import AzureLogHandler + + logger = logging.getLogger(__name__) + + # Callback function to append '_hello' to each log message telemetry + def callback_function(envelope): + envelope.data.baseData.message += '_hello' + return True + + handler = AzureLogHandler(connection_string='InstrumentationKey=') + handler.add_telemetry_processor(callback_function) + logger.addHandler(handler) + logger.warning('Hello, World!') + Metrics ~~~~~~~ @@ -143,6 +178,9 @@ The **Azure Monitor Metrics Exporter** allows you to export metrics to `Azure Mo if __name__ == "__main__": main() +Standard Metrics +################ + The exporter also includes a set of standard metrics that are exported to Azure Monitor by default. .. code:: python @@ -177,6 +215,67 @@ Below is a list of standard metrics that are currently available: - Process CPU Usage (percentage) - Process Private Bytes (bytes) +Modifying Metrics +################# + +* You can pass a callback function to the exporter to process telemetry before it is exported. +* Your callback function can return `False` if you do not want this envelope exported. +* Your callback function must accept an [envelope](https://github.com/census-instrumentation/opencensus-python/blob/master/contrib/opencensus-ext-azure/opencensus/ext/azure/common/protocol.py#L86) data type as its parameter. +* You can see the schema for Azure Monitor data types in the envelopes [here](https://github.com/census-instrumentation/opencensus-python/blob/master/contrib/opencensus-ext-azure/opencensus/ext/azure/common/protocol.py). +* The `MetricsExporter` handles `MetricData` data types. + +.. code:: python + + import time + + from opencensus.ext.azure import metrics_exporter + from opencensus.stats import aggregation as aggregation_module + from opencensus.stats import measure as measure_module + from opencensus.stats import stats as stats_module + from opencensus.stats import view as view_module + from opencensus.tags import tag_map as tag_map_module + + stats = stats_module.stats + view_manager = stats.view_manager + stats_recorder = stats.stats_recorder + + CARROTS_MEASURE = measure_module.MeasureInt("carrots", + "number of carrots", + "carrots") + CARROTS_VIEW = view_module.View("carrots_view", + "number of carrots", + [], + CARROTS_MEASURE, + aggregation_module.CountAggregation()) + + # Callback function to only export the metric if value is greater than 0 + def callback_function(envelope): + return envelope.data.baseData.metrics[0].value > 0 + + def main(): + # Enable metrics + # Set the interval in seconds in which you want to send metrics + exporter = metrics_exporter.new_metrics_exporter(connection_string='InstrumentationKey=') + exporter.add_telemetry_processor(callback_function) + view_manager.register_exporter(exporter) + + view_manager.register_view(CARROTS_VIEW) + mmap = stats_recorder.new_measurement_map() + tmap = tag_map_module.TagMap() + + mmap.measure_int_put(CARROTS_MEASURE, 1000) + mmap.record(tmap) + # Default export interval is every 15.0s + # Your application should run for at least this amount + # of time so the exporter will meet this interval + # Sleep can fulfill this + time.sleep(60) + + print("Done recording metrics") + + if __name__ == "__main__": + main() + Trace ~~~~~ @@ -195,16 +294,21 @@ This example shows how to send a span "hello" to Azure Monitor. from opencensus.trace.tracer import Tracer tracer = Tracer( - exporter=AzureExporter(connection_string='InstrumentationKey='), + exporter=AzureExporter( + connection_string='InstrumentationKey=' + ), sampler=ProbabilitySampler(1.0) ) with tracer.span(name='hello'): print('Hello, World!') -OpenCensus also supports several [integrations](https://github.com/census-instrumentation/opencensus-python#integration) which allows OpenCensus to integrate with third party libraries. +Integrations +############ + +OpenCensus also supports several `integrations `_ which allows OpenCensus to integrate with third party libraries. -This example shows how to integrate with the [requests](https://2.python-requests.org/en/master/) library. +This example shows how to integrate with the `requests `_ library. * Create an Azure Monitor resource and get the instrumentation key, more information can be found `here `_. * Install the `requests integration package <../opencensus-ext-requests>`_ using ``pip install opencensus-ext-requests``. @@ -223,14 +327,45 @@ This example shows how to integrate with the [requests](https://2.python-request config_integration.trace_integrations(['requests']) tracer = Tracer( exporter=AzureExporter( - # TODO: replace this with your own instrumentation key. - instrumentation_key='00000000-0000-0000-0000-000000000000', + connection_string='InstrumentationKey=', ), sampler=ProbabilitySampler(1.0), ) with tracer.span(name='parent'): response = requests.get(url='https://www.wikipedia.org/wiki/Rabbit') +Modifying Traces +################ + +* You can pass a callback function to the exporter to process telemetry before it is exported. +* Your callback function can return `False` if you do not want this envelope exported. +* Your callback function must accept an [envelope](https://github.com/census-instrumentation/opencensus-python/blob/master/contrib/opencensus-ext-azure/opencensus/ext/azure/common/protocol.py#L86) data type as its parameter. +* You can see the schema for Azure Monitor data types in the envelopes [here](https://github.com/census-instrumentation/opencensus-python/blob/master/contrib/opencensus-ext-azure/opencensus/ext/azure/common/protocol.py). +* The `AzureExporter` handles `Data` data types. + +.. code:: python + + import requests + + from opencensus.ext.azure.trace_exporter import AzureExporter + from opencensus.trace import config_integration + from opencensus.trace.samplers import ProbabilitySampler + from opencensus.trace.tracer import Tracer + + config_integration.trace_integrations(['requests']) + + # Callback function to add os_type: linux to span properties + def callback_function(envelope): + envelope.data.baseData.properties['os_type'] = 'linux' + return True + + exporter = AzureExporter( + connection_string='InstrumentationKey=' + ) + exporter.add_telemetry_processor(callback_function) + tracer = Tracer(exporter=exporter, sampler=ProbabilitySampler(1.0)) + with tracer.span(name='parent'): + response = requests.get(url='https://www.wikipedia.org/wiki/Rabbit') References ---------- diff --git a/contrib/opencensus-ext-azure/examples/logs/properties.py b/contrib/opencensus-ext-azure/examples/logs/properties.py index e6f8c5b2a..5cfdd3568 100644 --- a/contrib/opencensus-ext-azure/examples/logs/properties.py +++ b/contrib/opencensus-ext-azure/examples/logs/properties.py @@ -21,4 +21,14 @@ # and place it in the APPLICATIONINSIGHTS_CONNECTION_STRING # environment variable. logger.addHandler(AzureLogHandler()) -logger.warning('action', {'key-1': 'value-1', 'key-2': 'value2'}) + +properties = {'custom_dimensions': {'key_1': 'value_1', 'key_2': 'value_2'}} + +# Use properties in logging statements +logger.warning('action', extra=properties) + +# Use properties in exception logs +try: + result = 1 / 0 # generate a ZeroDivisionError +except Exception: + logger.exception('Captured an exception.', extra=properties) diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py index 754e2a27c..6b763c037 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py @@ -92,6 +92,7 @@ def __init__(self, *args, **kwargs): export_interval=15.0, grace_period=5.0, instrumentation_key=None, + logging_sampling_rate=1.0, max_batch_size=100, minimum_retry_interval=60, # minimum retry interval in seconds proxy=None, diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/processor.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/processor.py new file mode 100644 index 000000000..4a98afa50 --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/processor.py @@ -0,0 +1,63 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +logger = logging.getLogger(__name__) + + +class ProcessorMixin(object): + """ProcessorMixin adds the ability to process telemetry processors + + Telemetry processors are functions that are called before exporting of + telemetry to possibly modify the envelope contents. + """ + + def add_telemetry_processor(self, processor): + """Adds telemetry processor to the collection. Telemetry processors + will be called one by one before telemetry item is pushed for sending + and in the order they were added. + + :param processor: The processor to add. + """ + self._telemetry_processors.append(processor) + + def clear_telemetry_processors(self): + """Removes all telemetry processors""" + self._telemetry_processors = [] + + def apply_telemetry_processors(self, envelopes): + """Applies all telemetry processors in the order they were added. + + This function will return the list of envelopes to be exported after + each processor has been run sequentially. Individual processors can + throw exceptions and fail, but the applying of all telemetry processors + will proceed (not fast fail). Processors also return True if envelope + should be included for exporting, False otherwise. + + :param envelopes: The envelopes to apply each processor to. + """ + filtered_envelopes = [] + for envelope in envelopes: + accepted = True + for processor in self._telemetry_processors: + try: + if processor(envelope) is False: + accepted = False + break + except Exception as ex: + logger.warning('Telemetry processor failed with: %s.', ex) + if accepted: + filtered_envelopes.append(envelope) + return filtered_envelopes diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/utils.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/utils.py index 6224b9561..4907f74af 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/utils.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/utils.py @@ -23,12 +23,6 @@ from opencensus.common.version import __version__ as opencensus_version from opencensus.ext.azure.common.version import __version__ as ext_version -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - - azure_monitor_context = { 'ai.cloud.role': os.path.basename(sys.argv[0]) or 'Python Application', 'ai.cloud.roleInstance': platform.node(), @@ -64,10 +58,6 @@ def timestamp_to_iso_str(timestamp): return to_iso_str(datetime.datetime.utcfromtimestamp(timestamp)) -def url_to_dependency_name(url): - return urlparse(url).netloc - - # Validate UUID format # Specs taken from https://tools.ietf.org/html/rfc4122 uuid_regex_pattern = re.compile('^[0-9a-f]{8}-' diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py index ad304edc9..4b19db0b6 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '1.0.1' +__version__ = '1.0.2' diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py index dc9a50db8..6bc4db53f 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py @@ -13,12 +13,14 @@ # limitations under the License. import logging +import random import threading import time import traceback from opencensus.common.schedule import Queue, QueueEvent, QueueExitEvent from opencensus.ext.azure.common import Options, utils +from opencensus.ext.azure.common.processor import ProcessorMixin from opencensus.ext.azure.common.protocol import ( Data, Envelope, @@ -107,7 +109,17 @@ def stop(self, timeout=None): # pragma: NO COVER return time.time() - start_time # time taken to stop -class AzureLogHandler(TransportMixin, BaseLogHandler): +class SamplingFilter(logging.Filter): + + def __init__(self, probability=1.0): + super(SamplingFilter, self).__init__() + self.probability = probability + + def filter(self, record): + return random.random() < self.probability + + +class AzureLogHandler(TransportMixin, ProcessorMixin, BaseLogHandler): """Handler for logging to Microsoft Azure Monitor. :param options: Options for the log handler. @@ -116,6 +128,8 @@ class AzureLogHandler(TransportMixin, BaseLogHandler): def __init__(self, **options): self.options = Options(**options) utils.validate_instrumentation_key(self.options.instrumentation_key) + if not 0 <= self.options.logging_sampling_rate <= 1: + raise ValueError('Sampling must be in the range: [0,1]') self.export_interval = self.options.export_interval self.max_batch_size = self.options.max_batch_size self.storage = LocalFileStorage( @@ -124,7 +138,9 @@ def __init__(self, **options): maintenance_period=self.options.storage_maintenance_period, retention_period=self.options.storage_retention_period, ) + self._telemetry_processors = [] super(AzureLogHandler, self).__init__() + self.addFilter(SamplingFilter(self.options.logging_sampling_rate)) def close(self): self.storage.close() @@ -134,6 +150,7 @@ def _export(self, batch, event=None): # pragma: NO COVER try: if batch: envelopes = [self.log_record_to_envelope(x) for x in batch] + envelopes = self.apply_telemetry_processors(envelopes) result = self._transmit(envelopes) if result > 0: self.storage.put(envelopes, result) @@ -153,6 +170,7 @@ def log_record_to_envelope(self, record): tags=dict(utils.azure_monitor_context), time=utils.timestamp_to_iso_str(record.created), ) + envelope.tags['ai.operation.id'] = getattr( record, 'traceId', @@ -169,6 +187,11 @@ def log_record_to_envelope(self, record): 'lineNumber': record.lineno, 'level': record.levelname, } + + if (hasattr(record, 'custom_dimensions') and + isinstance(record.custom_dimensions, dict)): + properties.update(record.custom_dimensions) + if record.exc_info: exctype, _value, tb = record.exc_info callstack = [] @@ -198,8 +221,6 @@ def log_record_to_envelope(self, record): ) envelope.data = Data(baseData=data, baseType='ExceptionData') else: - if isinstance(record.args, dict): - properties.update(record.args) envelope.name = 'Microsoft.ApplicationInsights.Message' data = Message( message=self.format(record), diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py index 9b531b808..a5ba2a4ec 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py @@ -12,19 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json +import atexit import logging -import requests - from opencensus.common import utils as common_utils from opencensus.ext.azure.common import Options, utils +from opencensus.ext.azure.common.processor import ProcessorMixin from opencensus.ext.azure.common.protocol import ( Data, DataPoint, Envelope, MetricData, ) +from opencensus.ext.azure.common.storage import LocalFileStorage +from opencensus.ext.azure.common.transport import TransportMixin from opencensus.ext.azure.metrics_exporter import standard_metrics from opencensus.metrics import transport from opencensus.metrics.export.metric_descriptor import MetricDescriptorType @@ -35,49 +36,67 @@ logger = logging.getLogger(__name__) -class MetricsExporter(object): +class MetricsExporter(TransportMixin, ProcessorMixin): """Metrics exporter for Microsoft Azure Monitor.""" - def __init__(self, options=None): - if options is None: - options = Options() - self.options = options + def __init__(self, **options): + self.options = Options(**options) utils.validate_instrumentation_key(self.options.instrumentation_key) if self.options.max_batch_size <= 0: raise ValueError('Max batch size must be at least 1.') + self.export_interval = self.options.export_interval self.max_batch_size = self.options.max_batch_size + self._telemetry_processors = [] + self.storage = LocalFileStorage( + path=self.options.storage_path, + max_size=self.options.storage_max_size, + maintenance_period=self.options.storage_maintenance_period, + retention_period=self.options.storage_retention_period, + ) + super(MetricsExporter, self).__init__() def export_metrics(self, metrics): - if metrics: - envelopes = [] - for metric in metrics: - # No support for histogram aggregations - type_ = metric.descriptor.type - if type_ != MetricDescriptorType.CUMULATIVE_DISTRIBUTION: - md = metric.descriptor - # Each time series will be uniquely identified by its - # label values - for time_series in metric.time_series: - # Using stats, time_series should only have one point - # which contains the aggregated value - data_point = self.create_data_points( - time_series, md)[0] - # The timestamp is when the metric was recorded - time_stamp = time_series.points[0].timestamp - # Get the properties using label keys from metric and - # label values of the time series - properties = self.create_properties(time_series, md) - envelopes.append(self.create_envelope(data_point, - time_stamp, - properties)) - # Send data in batches of max_batch_size - if envelopes: - batched_envelopes = list(common_utils.window( - envelopes, self.max_batch_size)) - for batch in batched_envelopes: - self._transmit_without_retry(batch) - - def create_data_points(self, time_series, metric_descriptor): + envelopes = [] + for metric in metrics: + envelopes.extend(self.metric_to_envelopes(metric)) + # Send data in batches of max_batch_size + batched_envelopes = list(common_utils.window( + envelopes, self.max_batch_size)) + for batch in batched_envelopes: + batch = self.apply_telemetry_processors(batch) + result = self._transmit(batch) + if result > 0: + self.storage.put(batch, result) + + # If there is still room to transmit envelopes, transmit from storage + # if available + if len(envelopes) < self.options.max_batch_size: + self._transmit_from_storage() + + def metric_to_envelopes(self, metric): + envelopes = [] + # No support for histogram aggregations + if (metric.descriptor.type != + MetricDescriptorType.CUMULATIVE_DISTRIBUTION): + md = metric.descriptor + # Each time series will be uniquely identified by its + # label values + for time_series in metric.time_series: + # Using stats, time_series should only have one + # point which contains the aggregated value + data_point = self._create_data_points( + time_series, md)[0] + # The timestamp is when the metric was recorded + timestamp = time_series.points[0].timestamp + # Get the properties using label keys from metric + # and label values of the time series + properties = self._create_properties(time_series, md) + envelopes.append(self._create_envelope(data_point, + timestamp, + properties)) + return envelopes + + def _create_data_points(self, time_series, metric_descriptor): """Convert a metric's OC time series to list of Azure data points.""" data_points = [] for point in time_series.points: @@ -88,7 +107,7 @@ def create_data_points(self, time_series, metric_descriptor): data_points.append(data_point) return data_points - def create_properties(self, time_series, metric_descriptor): + def _create_properties(self, time_series, metric_descriptor): properties = {} # We construct a properties map from the label keys and values. We # assume the ordering is already correct @@ -100,11 +119,11 @@ def create_properties(self, time_series, metric_descriptor): properties[metric_descriptor.label_keys[i].key] = value return properties - def create_envelope(self, data_point, time_stamp, properties): + def _create_envelope(self, data_point, timestamp, properties): envelope = Envelope( iKey=self.options.instrumentation_key, tags=dict(utils.azure_monitor_context), - time=time_stamp.isoformat(), + time=timestamp.isoformat(), ) envelope.name = "Microsoft.ApplicationInsights.Metric" data = MetricData( @@ -114,125 +133,14 @@ def create_envelope(self, data_point, time_stamp, properties): envelope.data = Data(baseData=data, baseType="MetricData") return envelope - def _transmit_without_retry(self, envelopes): - # Contains logic from transport._transmit - # TODO: Remove this function from exporter and consolidate with - # transport._transmit to cover all exporter use cases. Uses cases - # pertain to properly handling failures and implementing a retry - # policy for this exporter. - # TODO: implement retry policy - """ - Transmit the data envelopes to the ingestion service. - Does not perform retry logic. For partial success and - non-retryable failure, simply outputs result to logs. - This function should never throw exception. - """ - try: - response = requests.post( - url=self.options.endpoint, - data=json.dumps(envelopes), - headers={ - 'Accept': 'application/json', - 'Content-Type': 'application/json; charset=utf-8', - }, - timeout=self.options.timeout, - ) - except Exception as ex: - # No retry policy, log output - logger.warning('Transient client side error %s.', ex) - return - - text = 'N/A' - data = None - # Handle the possible results from the response - if response is None: - logger.warning('Error: cannot read response.') - return - try: - status_code = response.status_code - except Exception as ex: - logger.warning('Error while reading response status code %s.', ex) - return - try: - text = response.text - except Exception as ex: - logger.warning('Error while reading response body %s.', ex) - return - try: - data = json.loads(text) - except Exception as ex: - logger.warning('Error while loading ' + - 'json from response body %s.', ex) - return - if status_code == 200: - logger.info('Transmission succeeded: %s.', text) - return - # Check for retryable partial content - if status_code == 206: - if data: - try: - retryable_envelopes = [] - for error in data['errors']: - if error['statusCode'] in ( - 429, # Too Many Requests - 500, # Internal Server Error - 503, # Service Unavailable - ): - retryable_envelopes.append( - envelopes[error['index']]) - else: - logger.error( - 'Data drop %s: %s %s.', - error['statusCode'], - error['message'], - envelopes[error['index']], - ) - # show the envelopes that can be retried manually for - # visibility - if retryable_envelopes: - logger.warning( - 'Error while processing data. Data dropped. ' + - 'Consider manually retrying for envelopes: %s.', - retryable_envelopes - ) - return - except Exception: - logger.exception( - 'Error while processing %s: %s.', - status_code, - text - ) - return - # Check for non-retryable result - if status_code in ( - 206, # Partial Content - 429, # Too Many Requests - 500, # Internal Server Error - 503, # Service Unavailable - ): - # server side error (retryable) - logger.warning( - 'Transient server side error %s: %s. ' + - 'Consider manually trying.', - status_code, - text, - ) - else: - # server side error (non-retryable) - logger.error( - 'Non-retryable server side error %s: %s.', - status_code, - text, - ) - def new_metrics_exporter(**options): - options_ = Options(**options) - exporter = MetricsExporter(options=options_) + exporter = MetricsExporter(**options) producers = [stats_module.stats] - if options_.enable_standard_metrics: + if exporter.options.enable_standard_metrics: producers.append(standard_metrics.producer) transport.get_exporter_thread(producers, exporter, - interval=options_.export_interval) + interval=exporter.options.export_interval) + atexit.register(exporter.export_metrics, stats_module.stats.get_metrics()) return exporter diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py index 9daf58706..ce904e673 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py @@ -17,6 +17,7 @@ from opencensus.common.schedule import QueueExitEvent from opencensus.ext.azure.common import Options, utils from opencensus.ext.azure.common.exporter import BaseExporter +from opencensus.ext.azure.common.processor import ProcessorMixin from opencensus.ext.azure.common.protocol import ( Data, Envelope, @@ -27,12 +28,17 @@ from opencensus.ext.azure.common.transport import TransportMixin from opencensus.trace.span import SpanKind +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + logger = logging.getLogger(__name__) __all__ = ['AzureExporter'] -class AzureExporter(TransportMixin, BaseExporter): +class AzureExporter(BaseExporter, ProcessorMixin, TransportMixin): """An exporter that sends traces to Microsoft Azure Monitor. :param options: Options for the exporter. @@ -47,6 +53,7 @@ def __init__(self, **options): maintenance_period=self.options.storage_maintenance_period, retention_period=self.options.storage_retention_period, ) + self._telemetry_processors = [] super(AzureExporter, self).__init__(**options) def span_data_to_envelope(self, sd): @@ -55,50 +62,58 @@ def span_data_to_envelope(self, sd): tags=dict(utils.azure_monitor_context), time=sd.start_time, ) + envelope.tags['ai.operation.id'] = sd.context.trace_id if sd.parent_span_id: - envelope.tags['ai.operation.parentId'] = '|{}.{}.'.format( - sd.context.trace_id, + envelope.tags['ai.operation.parentId'] = '{}'.format( sd.parent_span_id, ) if sd.span_kind == SpanKind.SERVER: envelope.name = 'Microsoft.ApplicationInsights.Request' data = Request( - id='|{}.{}.'.format(sd.context.trace_id, sd.span_id), + id='{}'.format(sd.span_id), duration=utils.timestamp_to_duration( sd.start_time, sd.end_time, ), - responseCode='0', - success=False, + responseCode=str(sd.status.code), + success=False, # Modify based off attributes or status properties={}, ) envelope.data = Data(baseData=data, baseType='RequestData') + data.name = '' if 'http.method' in sd.attributes: data.name = sd.attributes['http.method'] if 'http.route' in sd.attributes: data.name = data.name + ' ' + sd.attributes['http.route'] envelope.tags['ai.operation.name'] = data.name + data.properties['request.name'] = data.name + elif 'http.path' in sd.attributes: + data.properties['request.name'] = data.name + \ + ' ' + sd.attributes['http.path'] if 'http.url' in sd.attributes: data.url = sd.attributes['http.url'] + data.properties['request.url'] = sd.attributes['http.url'] if 'http.status_code' in sd.attributes: status_code = sd.attributes['http.status_code'] data.responseCode = str(status_code) data.success = ( status_code >= 200 and status_code <= 399 ) + elif sd.status.code == 0: + data.success = True else: envelope.name = \ 'Microsoft.ApplicationInsights.RemoteDependency' data = RemoteDependency( name=sd.name, # TODO - id='|{}.{}.'.format(sd.context.trace_id, sd.span_id), - resultCode='0', # TODO + id='{}'.format(sd.span_id), + resultCode=str(sd.status.code), duration=utils.timestamp_to_duration( sd.start_time, sd.end_time, ), - success=True, # TODO + success=False, # Modify based off attributes or status properties={}, ) envelope.data = Data( @@ -106,15 +121,27 @@ def span_data_to_envelope(self, sd): baseType='RemoteDependencyData', ) if sd.span_kind == SpanKind.CLIENT: - data.type = 'HTTP' # TODO + data.type = sd.attributes.get('component') if 'http.url' in sd.attributes: url = sd.attributes['http.url'] # TODO: error handling, probably put scheme as well - data.name = utils.url_to_dependency_name(url) + data.data = url + parse_url = urlparse(url) + # target matches authority (host:port) + data.target = parse_url.netloc + if 'http.method' in sd.attributes: + # name is METHOD/path + data.name = sd.attributes['http.method'] \ + + ' ' + parse_url.path if 'http.status_code' in sd.attributes: - data.resultCode = str(sd.attributes['http.status_code']) + status_code = sd.attributes["http.status_code"] + data.resultCode = str(status_code) + data.success = 200 <= status_code < 400 + elif sd.status.code == 0: + data.success = True else: data.type = 'INPROC' + data.success = True # TODO: links, tracestate, tags for key in sd.attributes: # This removes redundant data from ApplicationInsights @@ -127,6 +154,7 @@ def emit(self, batch, event=None): try: if batch: envelopes = [self.span_data_to_envelope(sd) for sd in batch] + envelopes = self.apply_telemetry_processors(envelopes) result = self._transmit(envelopes) if result > 0: self.storage.put(envelopes, result) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py index 7ff06022a..8aa6baa8d 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py @@ -78,6 +78,13 @@ def test_ctor(self): self.assertRaises(ValueError, lambda: log_exporter.AzureLogHandler()) Options._default.instrumentation_key = instrumentation_key + def test_invalid_sampling_rate(self): + with self.assertRaises(ValueError): + log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + logging_sampling_rate=4.0, + ) + @mock.patch('requests.post', return_value=mock.Mock()) def test_exception(self, requests_mock): logger = logging.getLogger(self.id()) @@ -95,6 +102,32 @@ def test_exception(self, requests_mock): post_body = requests_mock.call_args_list[0][1]['data'] self.assertTrue('ZeroDivisionError' in post_body) + @mock.patch('requests.post', return_value=mock.Mock()) + def test_exception_with_custom_properties(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + logger.addHandler(handler) + try: + return 1 / 0 # generate a ZeroDivisionError + except Exception: + properties = { + 'custom_dimensions': + { + 'key_1': 'value_1', + 'key_2': 'value_2' + } + } + logger.exception('Captured an exception.', extra=properties) + handler.close() + self.assertEqual(len(requests_mock.call_args_list), 1) + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('ZeroDivisionError' in post_body) + self.assertTrue('key_1' in post_body) + self.assertTrue('key_2' in post_body) + @mock.patch('requests.post', return_value=mock.Mock()) def test_export_empty(self, request_mock): handler = log_exporter.AzureLogHandler( @@ -143,12 +176,18 @@ def test_log_record_with_custom_properties(self, requests_mock): storage_path=os.path.join(TEST_FOLDER, self.id()), ) logger.addHandler(handler) - logger.warning('action', {'key-1': 'value-1', 'key-2': 'value-2'}) + logger.warning('action', extra={ + 'custom_dimensions': + { + 'key_1': 'value_1', + 'key_2': 'value_2' + } + }) handler.close() post_body = requests_mock.call_args_list[0][1]['data'] self.assertTrue('action' in post_body) - self.assertTrue('key-1' in post_body) - self.assertTrue('key-2' in post_body) + self.assertTrue('key_1' in post_body) + self.assertTrue('key_2' in post_body) @mock.patch('requests.post', return_value=mock.Mock()) def test_log_with_invalid_custom_properties(self, requests_mock): @@ -159,9 +198,53 @@ def test_log_with_invalid_custom_properties(self, requests_mock): ) logger.addHandler(handler) logger.warning('action_1_%s', None) - logger.warning('action_2_%s', 'not_a_dict') + logger.warning('action_2_%s', 'arg', extra={ + 'custom_dimensions': 'not_a_dict' + }) + logger.warning('action_3_%s', 'arg', extra={ + 'notcustom_dimensions': {'key_1': 'value_1'} + }) + handler.close() self.assertEqual(len(os.listdir(handler.storage.path)), 0) post_body = requests_mock.call_args_list[0][1]['data'] - self.assertTrue('action_1' in post_body) - self.assertTrue('action_2' in post_body) + self.assertTrue('action_1_' in post_body) + self.assertTrue('action_2_arg' in post_body) + self.assertTrue('action_3_arg' in post_body) + + self.assertFalse('not_a_dict' in post_body) + self.assertFalse('key_1' in post_body) + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_log_record_sampled(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + logging_sampling_rate=1.0, + ) + logger.addHandler(handler) + logger.warning('Hello_World') + logger.warning('Hello_World2') + logger.warning('Hello_World3') + logger.warning('Hello_World4') + handler.close() + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('Hello_World' in post_body) + self.assertTrue('Hello_World2' in post_body) + self.assertTrue('Hello_World3' in post_body) + self.assertTrue('Hello_World4' in post_body) + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_log_record_not_sampled(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + logging_sampling_rate=0.0, + ) + logger.addHandler(handler) + logger.warning('Hello_World') + logger.warning('Hello_World2') + logger.warning('Hello_World3') + logger.warning('Hello_World4') + handler.close() + self.assertFalse(requests_mock.called) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_metrics_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_metrics_exporter.py index 466f017ce..9c84118f0 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_metrics_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_metrics_exporter.py @@ -12,23 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock import unittest from datetime import datetime +import mock + from opencensus.common import utils from opencensus.ext.azure import metrics_exporter from opencensus.ext.azure.common import Options from opencensus.ext.azure.common.protocol import DataPoint -from opencensus.ext.azure.common.protocol import Envelope from opencensus.ext.azure.metrics_exporter import standard_metrics -from opencensus.metrics import label_key -from opencensus.metrics import label_value -from opencensus.metrics.export import metric -from opencensus.metrics.export import metric_descriptor -from opencensus.metrics.export import point -from opencensus.metrics.export import time_series -from opencensus.metrics.export import value +from opencensus.metrics import label_key, label_value +from opencensus.metrics.export import ( + metric, + metric_descriptor, + point, + time_series, + value, +) from opencensus.metrics.export.metric_descriptor import MetricDescriptorType @@ -55,10 +56,6 @@ def create_metric(): return mm -def create_envelope(): - return Envelope._default - - class TestAzureMetricsExporter(unittest.TestCase): def test_constructor_missing_key(self): instrumentation_key = Options._default.instrumentation_key @@ -68,20 +65,18 @@ def test_constructor_missing_key(self): Options._default.instrumentation_key = instrumentation_key def test_constructor_invalid_batch_size(self): - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd', - max_batch_size=-1) self.assertRaises( ValueError, - lambda: metrics_exporter.MetricsExporter(options=options) - ) + lambda: metrics_exporter.MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + max_batch_size=-1 + )) @mock.patch('requests.post', return_value=mock.Mock()) def test_export_metrics(self, requests_mock): metric = create_metric() - options = Options( + exporter = metrics_exporter.MetricsExporter( instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) requests_mock.return_value.text = '{"itemsReceived":1,'\ '"itemsAccepted":1,'\ '"errors":[]}' @@ -95,18 +90,16 @@ def test_export_metrics(self, requests_mock): def test_export_metrics_histogram(self): metric = create_metric() - options = Options( + exporter = metrics_exporter.MetricsExporter( instrumentation_key='12345678-1234-5678-abcd-12345678abcd') metric.descriptor._type = MetricDescriptorType.CUMULATIVE_DISTRIBUTION - exporter = metrics_exporter.MetricsExporter(options) self.assertIsNone(exporter.export_metrics([metric])) @mock.patch('requests.post', return_value=mock.Mock()) def test_export_metrics_empty(self, requests_mock): - options = Options( + exporter = metrics_exporter.MetricsExporter( instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) exporter.export_metrics([]) self.assertEqual(len(requests_mock.call_args_list), 0) @@ -114,10 +107,9 @@ def test_export_metrics_empty(self, requests_mock): @mock.patch('requests.post', return_value=mock.Mock()) def test_export_metrics_full_batch(self, requests_mock): metric = create_metric() - options = Options( + exporter = metrics_exporter.MetricsExporter( instrumentation_key='12345678-1234-5678-abcd-12345678abcd', max_batch_size=1) - exporter = metrics_exporter.MetricsExporter(options) requests_mock.return_value.status_code = 200 requests_mock.return_value.text = '{"itemsReceived":1,'\ '"itemsAccepted":1,'\ @@ -129,270 +121,13 @@ def test_export_metrics_full_batch(self, requests_mock): self.assertTrue('metrics' in post_body) self.assertTrue('properties' in post_body) - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.warning', return_value=mock.Mock()) - def test_transmit_client_error(self, logger_mock): - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry(mock.Mock()) - - self.assertEqual(len(logger_mock.call_args_list), 1) - - @mock.patch('requests.post', return_value=None) - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.warning', return_value=mock.Mock()) - def test_transmit_no_response(self, requests_mock, logger_mock): - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.warning', return_value=mock.Mock()) - def test_transmit_no_status_code(self, logger_mock): - with mock.patch('requests.post') as requests_mock: - type(requests_mock.return_value).status_code = mock.PropertyMock( - side_effect=Exception()) - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.warning', return_value=mock.Mock()) - def test_transmit_no_response_body(self, logger_mock): - with mock.patch('requests.post') as requests_mock: - type(requests_mock.return_value).text = mock.PropertyMock( - side_effect=Exception()) - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.warning', return_value=mock.Mock()) - def test_transmit_invalid_response_body(self, logger_mock): - with mock.patch('requests.post') as requests_mock: - type(requests_mock.return_value).text = mock.PropertyMock( - return_value='invalid') - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.info', return_value=mock.Mock()) - def test_transmit_success(self, logger_mock): - with mock.patch('requests.post') as requests_mock: - text = '{"itemsReceived":1,'\ - '"itemsAccepted":1,'\ - '"errors":[]}' - type(requests_mock.return_value).text = mock.PropertyMock( - return_value=text) - type(requests_mock.return_value).status_code = mock.PropertyMock( - return_value=200) - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.warning', return_value=mock.Mock()) - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.json.loads', return_value=None) - def test_transmit_none_data_retryable(self, logger_mock, json_mock): - with mock.patch('requests.post') as requests_mock: - text = '{"itemsReceived":1,'\ - '"itemsAccepted":1,'\ - '"errors":[{"statusCode":500, "index":0}]}' - type(requests_mock.return_value).text = mock.PropertyMock( - return_value=text) - type(requests_mock.return_value).status_code = mock.PropertyMock( - return_value=206) - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.error', return_value=mock.Mock()) - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.json.loads', return_value=None) - def test_transmit_none_data_non_retryable(self, logger_mock, json_mock): - with mock.patch('requests.post') as requests_mock: - text = '{"itemsReceived":1,'\ - '"itemsAccepted":1,'\ - '"errors":[{"statusCode":500, "index":0}]}' - type(requests_mock.return_value).text = mock.PropertyMock( - return_value=text) - type(requests_mock.return_value).status_code = mock.PropertyMock( - return_value=402) - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.exception', return_value=mock.Mock()) - def test_transmit_partial_exception(self, logger_mock): - with mock.patch('requests.post') as requests_mock: - text = '{"itemsReceived":1,'\ - '"itemsAccepted":1}' - type(requests_mock.return_value).text = mock.PropertyMock( - return_value=text) - type(requests_mock.return_value).status_code = mock.PropertyMock( - return_value=206) - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.warning', return_value=mock.Mock()) - def test_transmit_partial_retryable(self, logger_mock): - with mock.patch('requests.post') as requests_mock: - text = '{"itemsReceived":1,'\ - '"itemsAccepted":1,'\ - '"errors":[{"statusCode":429, "index":0}]}' - type(requests_mock.return_value).text = mock.PropertyMock( - return_value=text) - type(requests_mock.return_value).status_code = mock.PropertyMock( - return_value=206) - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.error', return_value=mock.Mock()) - def test_transmit_partial_non_retryable(self, logger_mock): - with mock.patch('requests.post') as requests_mock: - text = '{"itemsReceived":1,'\ - '"itemsAccepted":1,'\ - '"errors":[{"statusCode":402,'\ - '"index":0,"message":"error"}]}' - type(requests_mock.return_value).text = mock.PropertyMock( - return_value=text) - type(requests_mock.return_value).status_code = mock.PropertyMock( - return_value=206) - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.error', return_value=mock.Mock()) - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.warning', return_value=mock.Mock()) - def test_transmit_partial_mix_retryable(self, logger_mock, logger2_mock): - with mock.patch('requests.post') as requests_mock: - text = '{"itemsReceived":1,'\ - '"itemsAccepted":0,'\ - '"errors":[{"statusCode":402,'\ - '"index":0,"message":"error"},'\ - '{"statusCode":429, "index":0}]}' - type(requests_mock.return_value).text = mock.PropertyMock( - return_value=text) - type(requests_mock.return_value).status_code = mock.PropertyMock( - return_value=206) - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - self.assertEqual(len(logger2_mock.call_args_list), 1) - - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.warning', return_value=mock.Mock()) - def test_transmit_server_retryable(self, logger_mock): - with mock.patch('requests.post') as requests_mock: - text = '{"itemsReceived":1,'\ - '"itemsAccepted":1,'\ - '"errors":[{"statusCode":500, "index":0}]}' - type(requests_mock.return_value).text = mock.PropertyMock( - return_value=text) - type(requests_mock.return_value).status_code = mock.PropertyMock( - return_value=500) - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - - @mock.patch('opencensus.ext.azure.metrics_exporter' + - '.logger.error', return_value=mock.Mock()) - def test_transmit_server_non_retryable(self, logger_mock): - with mock.patch('requests.post') as requests_mock: - text = '{"itemsReceived":1,'\ - '"itemsAccepted":1,'\ - '"errors":[{"statusCode":402, "index":0}]}' - type(requests_mock.return_value).text = mock.PropertyMock( - return_value=text) - type(requests_mock.return_value).status_code = mock.PropertyMock( - return_value=402) - envelope = create_envelope() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - exporter._transmit_without_retry([envelope]) - - self.assertEqual(len(requests_mock.call_args_list), 1) - self.assertEqual(len(logger_mock.call_args_list), 1) - def test_create_data_points(self): metric = create_metric() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - data_points = exporter.create_data_points(metric.time_series[0], - metric.descriptor) + exporter = metrics_exporter.MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + data_points = exporter._create_data_points(metric.time_series[0], + metric.descriptor) self.assertEqual(len(data_points), 1) data_point = data_points[0] @@ -403,42 +138,42 @@ def test_create_data_points(self): def test_create_properties(self): metric = create_metric() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) - properties = exporter.create_properties(metric.time_series[0], - metric.descriptor) + exporter = metrics_exporter.MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + properties = exporter._create_properties(metric.time_series[0], + metric.descriptor) self.assertEqual(len(properties), 1) self.assertEqual(properties['key'], 'val') def test_create_properties_none(self): metric = create_metric() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) + exporter = metrics_exporter.MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) metric.time_series[0].label_values[0]._value = None - properties = exporter.create_properties(metric.time_series[0], - metric.descriptor) + properties = exporter._create_properties(metric.time_series[0], + metric.descriptor) self.assertEqual(len(properties), 1) self.assertEqual(properties['key'], 'null') def test_create_envelope(self): metric = create_metric() - options = Options( - instrumentation_key='12345678-1234-5678-abcd-12345678abcd') - exporter = metrics_exporter.MetricsExporter(options) + exporter = metrics_exporter.MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) value = metric.time_series[0].points[0].value.value data_point = DataPoint(ns=metric.descriptor.name, name=metric.descriptor.name, value=value) timestamp = datetime(2019, 3, 20, 21, 34, 0, 537954) properties = {'url': 'website.com'} - envelope = exporter.create_envelope(data_point, timestamp, properties) + envelope = exporter._create_envelope(data_point, timestamp, properties) self.assertTrue('iKey' in envelope) - self.assertEqual(envelope.iKey, options.instrumentation_key) + self.assertEqual(envelope.iKey, '12345678-1234-5678-abcd-12345678abcd') self.assertTrue('tags' in envelope) self.assertTrue('time' in envelope) self.assertEqual(envelope.time, timestamp.isoformat()) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py index d884383f7..3543157b5 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py @@ -13,11 +13,12 @@ # limitations under the License. import json -import mock import os import shutil import unittest +import mock + from opencensus.ext.azure import trace_exporter TEST_FOLDER = os.path.abspath('.test.exporter') @@ -100,6 +101,7 @@ def test_span_data_to_envelope(self): from opencensus.trace.span import SpanKind from opencensus.trace.span_context import SpanContext from opencensus.trace.span_data import SpanData + from opencensus.trace.status import Status from opencensus.trace.trace_options import TraceOptions from opencensus.trace.tracestate import Tracestate @@ -121,6 +123,7 @@ def test_span_data_to_envelope(self): span_id='6e0c63257de34c92', parent_span_id='6e0c63257de34c93', attributes={ + 'component': 'HTTP', 'http.method': 'GET', 'http.url': 'https://www.wikipedia.org/wiki/Rabbit', 'http.status_code': 200, @@ -129,7 +132,7 @@ def test_span_data_to_envelope(self): end_time='2010-10-24T07:28:38.234567Z', stack_trace=None, links=None, - status=None, + status=Status(0), annotations=None, message_events=None, same_process_as_parent_span=None, @@ -144,7 +147,7 @@ def test_span_data_to_envelope(self): 'Microsoft.ApplicationInsights.RemoteDependency') self.assertEqual( envelope.tags['ai.operation.parentId'], - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c93.') + '6e0c63257de34c93') self.assertEqual( envelope.tags['ai.operation.id'], '6e0c63257de34c90bf9efcd03927272e') @@ -153,10 +156,16 @@ def test_span_data_to_envelope(self): '2010-10-24T07:28:38.123456Z') self.assertEqual( envelope.data.baseData.name, + 'GET /wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.data, + 'https://www.wikipedia.org/wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.target, 'www.wikipedia.org') self.assertEqual( envelope.data.baseData.id, - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c92.') + '6e0c63257de34c92') self.assertEqual( envelope.data.baseData.resultCode, '200') @@ -187,7 +196,7 @@ def test_span_data_to_envelope(self): end_time='2010-10-24T07:28:38.234567Z', stack_trace=None, links=None, - status=None, + status=Status(0), annotations=None, message_events=None, same_process_as_parent_span=None, @@ -202,7 +211,7 @@ def test_span_data_to_envelope(self): 'Microsoft.ApplicationInsights.RemoteDependency') self.assertEqual( envelope.tags['ai.operation.parentId'], - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c93.') + '6e0c63257de34c93') self.assertEqual( envelope.tags['ai.operation.id'], '6e0c63257de34c90bf9efcd03927272e') @@ -214,7 +223,75 @@ def test_span_data_to_envelope(self): 'test') self.assertEqual( envelope.data.baseData.id, - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c92.') + '6e0c63257de34c92') + self.assertEqual( + envelope.data.baseData.duration, + '0.00:00:00.111') + self.assertEqual( + envelope.data.baseData.type, + None) + self.assertEqual( + envelope.data.baseType, + 'RemoteDependencyData') + + # SpanKind.CLIENT missing method + envelope = exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={ + 'component': 'HTTP', + 'http.url': 'https://www.wikipedia.org/wiki/Rabbit', + 'http.status_code': 200, + }, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(0), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.CLIENT, + )) + self.assertEqual( + envelope.iKey, + '12345678-1234-5678-abcd-12345678abcd') + self.assertEqual( + envelope.name, + 'Microsoft.ApplicationInsights.RemoteDependency') + self.assertEqual( + envelope.tags['ai.operation.parentId'], + '6e0c63257de34c93') + self.assertEqual( + envelope.tags['ai.operation.id'], + '6e0c63257de34c90bf9efcd03927272e') + self.assertEqual( + envelope.time, + '2010-10-24T07:28:38.123456Z') + self.assertEqual( + envelope.data.baseData.name, + 'test') + self.assertEqual( + envelope.data.baseData.data, + 'https://www.wikipedia.org/wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.target, + 'www.wikipedia.org') + self.assertEqual( + envelope.data.baseData.id, + '6e0c63257de34c92') + self.assertEqual( + envelope.data.baseData.resultCode, + '200') self.assertEqual( envelope.data.baseData.duration, '0.00:00:00.111') @@ -238,6 +315,7 @@ def test_span_data_to_envelope(self): span_id='6e0c63257de34c92', parent_span_id='6e0c63257de34c93', attributes={ + 'component': 'HTTP', 'http.method': 'GET', 'http.path': '/wiki/Rabbit', 'http.route': '/wiki/Rabbit', @@ -248,7 +326,7 @@ def test_span_data_to_envelope(self): end_time='2010-10-24T07:28:38.234567Z', stack_trace=None, links=None, - status=None, + status=Status(0), annotations=None, message_events=None, same_process_as_parent_span=None, @@ -263,7 +341,7 @@ def test_span_data_to_envelope(self): 'Microsoft.ApplicationInsights.Request') self.assertEqual( envelope.tags['ai.operation.parentId'], - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c93.') + '6e0c63257de34c93') self.assertEqual( envelope.tags['ai.operation.id'], '6e0c63257de34c90bf9efcd03927272e') @@ -275,7 +353,7 @@ def test_span_data_to_envelope(self): '2010-10-24T07:28:38.123456Z') self.assertEqual( envelope.data.baseData.id, - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c92.') + '6e0c63257de34c92') self.assertEqual( envelope.data.baseData.duration, '0.00:00:00.111') @@ -285,12 +363,18 @@ def test_span_data_to_envelope(self): self.assertEqual( envelope.data.baseData.name, 'GET /wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.properties['request.name'], + 'GET /wiki/Rabbit') self.assertEqual( envelope.data.baseData.success, True) self.assertEqual( envelope.data.baseData.url, 'https://www.wikipedia.org/wiki/Rabbit') + self.assertEqual( + envelope.data.baseData.properties['request.url'], + 'https://www.wikipedia.org/wiki/Rabbit') self.assertEqual( envelope.data.baseType, 'RequestData') @@ -308,6 +392,7 @@ def test_span_data_to_envelope(self): span_id='6e0c63257de34c92', parent_span_id='6e0c63257de34c93', attributes={ + 'component': 'HTTP', 'http.method': 'GET', 'http.path': '/wiki/Rabbit', 'http.route': '/wiki/Rabbit', @@ -318,7 +403,7 @@ def test_span_data_to_envelope(self): end_time='2010-10-24T07:28:38.234567Z', stack_trace=None, links=None, - status=None, + status=Status(0), annotations=None, message_events=None, same_process_as_parent_span=None, @@ -333,7 +418,7 @@ def test_span_data_to_envelope(self): 'Microsoft.ApplicationInsights.Request') self.assertEqual( envelope.tags['ai.operation.parentId'], - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c93.') + '6e0c63257de34c93') self.assertEqual( envelope.tags['ai.operation.id'], '6e0c63257de34c90bf9efcd03927272e') @@ -345,7 +430,7 @@ def test_span_data_to_envelope(self): '2010-10-24T07:28:38.123456Z') self.assertEqual( envelope.data.baseData.id, - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c92.') + '6e0c63257de34c92') self.assertEqual( envelope.data.baseData.duration, '0.00:00:00.111') @@ -382,7 +467,7 @@ def test_span_data_to_envelope(self): end_time='2010-10-24T07:28:38.234567Z', stack_trace=None, links=None, - status=None, + status=Status(0), annotations=None, message_events=None, same_process_as_parent_span=None, @@ -397,7 +482,7 @@ def test_span_data_to_envelope(self): 'Microsoft.ApplicationInsights.Request') self.assertEqual( envelope.tags['ai.operation.parentId'], - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c93.') + '6e0c63257de34c93') self.assertEqual( envelope.tags['ai.operation.id'], '6e0c63257de34c90bf9efcd03927272e') @@ -406,7 +491,7 @@ def test_span_data_to_envelope(self): '2010-10-24T07:28:38.123456Z') self.assertEqual( envelope.data.baseData.id, - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c92.') + '6e0c63257de34c92') self.assertEqual( envelope.data.baseData.duration, '0.00:00:00.111') @@ -431,7 +516,7 @@ def test_span_data_to_envelope(self): end_time='2010-10-24T07:28:38.234567Z', stack_trace=None, links=None, - status=None, + status=Status(0), annotations=None, message_events=None, same_process_as_parent_span=None, @@ -461,14 +546,193 @@ def test_span_data_to_envelope(self): '0.00:00:00.111') self.assertEqual( envelope.data.baseData.id, - '|6e0c63257de34c90bf9efcd03927272e.6e0c63257de34c92.') + '6e0c63257de34c92') self.assertEqual( envelope.data.baseData.type, 'INPROC') + self.assertEqual( + envelope.data.baseData.success, + True + ) self.assertEqual( envelope.data.baseType, 'RemoteDependencyData') + # Status server status code attribute + envelope = exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={ + 'http.status_code': 201 + }, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(0), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.SERVER, + )) + self.assertEqual(envelope.data.baseData.responseCode, "201") + self.assertTrue(envelope.data.baseData.success) + + # Status server status code attribute missing + envelope = exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={}, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(1), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.SERVER, + )) + self.assertFalse(envelope.data.baseData.success) + + # Server route attribute missing + envelope = exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={ + 'component': 'HTTP', + 'http.method': 'GET', + 'http.path': '/wiki/Rabbitz', + 'http.url': 'https://www.wikipedia.org/wiki/Rabbit', + 'http.status_code': 400, + }, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(1), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.SERVER, + )) + self.assertEqual(envelope.data.baseData.properties['request.name'], + 'GET /wiki/Rabbitz') + + # Server route and path attribute missing + envelope = exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={ + 'component': 'HTTP', + 'http.method': 'GET', + 'http.url': 'https://www.wikipedia.org/wiki/Rabbit', + 'http.status_code': 400, + }, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(1), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.SERVER, + )) + self.assertIsNone( + envelope.data.baseData.properties.get('request.name')) + + # Status client status code attribute + envelope = exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={ + 'http.status_code': 201 + }, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(0), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.CLIENT, + )) + self.assertEqual(envelope.data.baseData.resultCode, "201") + self.assertTrue(envelope.data.baseData.success) + + # Status client status code attributes missing + envelope = exporter.span_data_to_envelope(SpanData( + name='test', + context=SpanContext( + trace_id='6e0c63257de34c90bf9efcd03927272e', + span_id='6e0c63257de34c91', + trace_options=TraceOptions('1'), + tracestate=Tracestate(), + from_header=False, + ), + span_id='6e0c63257de34c92', + parent_span_id='6e0c63257de34c93', + attributes={}, + start_time='2010-10-24T07:28:38.123456Z', + end_time='2010-10-24T07:28:38.234567Z', + stack_trace=None, + links=None, + status=Status(1), + annotations=None, + message_events=None, + same_process_as_parent_span=None, + child_span_count=None, + span_kind=SpanKind.CLIENT, + )) + self.assertFalse(envelope.data.baseData.success) + exporter._stop() def test_transmission_nothing(self): diff --git a/contrib/opencensus-ext-azure/tests/test_azure_utils.py b/contrib/opencensus-ext-azure/tests/test_azure_utils.py index 946f71b8a..47ef484d8 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_utils.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_utils.py @@ -37,13 +37,6 @@ def test_timestamp_to_iso_str(self): 1287905318.123456, ), '2010-10-24T07:28:38.123456Z') - def test_url_to_dependency_name(self): - self.assertEqual( - utils.url_to_dependency_name( - 'https://www.wikipedia.org/wiki/Rabbit' - ), - 'www.wikipedia.org') - def test_validate_instrumentation_key(self): key = '1234abcd-5678-4efa-8abc-1234567890ab' self.assertIsNone(utils.validate_instrumentation_key(key)) diff --git a/contrib/opencensus-ext-azure/tests/test_processor.py b/contrib/opencensus-ext-azure/tests/test_processor.py new file mode 100644 index 000000000..7ec01eb83 --- /dev/null +++ b/contrib/opencensus-ext-azure/tests/test_processor.py @@ -0,0 +1,94 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from opencensus.ext.azure.common.processor import ProcessorMixin +from opencensus.ext.azure.common.protocol import Envelope + + +# pylint: disable=W0212 +class TestProcessorMixin(unittest.TestCase): + def test_add(self): + mixin = ProcessorMixin() + mixin._telemetry_processors = [] + mixin.add_telemetry_processor(lambda: True) + self.assertEqual(len(mixin._telemetry_processors), 1) + + def test_clear(self): + mixin = ProcessorMixin() + mixin._telemetry_processors = [] + mixin.add_telemetry_processor(lambda: True) + self.assertEqual(len(mixin._telemetry_processors), 1) + mixin.clear_telemetry_processors() + self.assertEqual(len(mixin._telemetry_processors), 0) + + def test_apply(self): + mixin = ProcessorMixin() + mixin._telemetry_processors = [] + + def callback_function(envelope): + envelope.baseType += '_world' + mixin.add_telemetry_processor(callback_function) + envelope = Envelope() + envelope.baseType = 'type1' + mixin.apply_telemetry_processors([envelope]) + self.assertEqual(envelope.baseType, 'type1_world') + + def test_apply_multiple(self): + mixin = ProcessorMixin() + mixin._telemetry_processors = [] + + def callback_function(envelope): + envelope.baseType += '_world' + + def callback_function2(envelope): + envelope.baseType += '_world2' + mixin.add_telemetry_processor(callback_function) + mixin.add_telemetry_processor(callback_function2) + envelope = Envelope() + envelope.baseType = 'type1' + mixin.apply_telemetry_processors([envelope]) + self.assertEqual(envelope.baseType, 'type1_world_world2') + + def test_apply_exception(self): + mixin = ProcessorMixin() + mixin._telemetry_processors = [] + + def callback_function(envelope): + raise ValueError() + + def callback_function2(envelope): + envelope.baseType += '_world2' + mixin.add_telemetry_processor(callback_function) + mixin.add_telemetry_processor(callback_function2) + envelope = Envelope() + envelope.baseType = 'type1' + mixin.apply_telemetry_processors([envelope]) + self.assertEqual(envelope.baseType, 'type1_world2') + + def test_apply_not_accepted(self): + mixin = ProcessorMixin() + mixin._telemetry_processors = [] + + def callback_function(envelope): + return envelope.baseType == 'type2' + mixin.add_telemetry_processor(callback_function) + envelope = Envelope() + envelope.baseType = 'type1' + envelope2 = Envelope() + envelope2.baseType = 'type2' + envelopes = mixin.apply_telemetry_processors([envelope, envelope2]) + self.assertEqual(len(envelopes), 1) + self.assertEqual(envelopes[0].baseType, 'type2') diff --git a/contrib/opencensus-ext-httplib/CHANGELOG.md b/contrib/opencensus-ext-httplib/CHANGELOG.md index ca917f375..f55dcc589 100644 --- a/contrib/opencensus-ext-httplib/CHANGELOG.md +++ b/contrib/opencensus-ext-httplib/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +## 0.7.3 +Released 2020-02-03 + +- Added `component` span attribute + ## 0.7.2 Released 2019-08-06 diff --git a/contrib/opencensus-ext-httplib/opencensus/ext/httplib/trace.py b/contrib/opencensus-ext-httplib/opencensus/ext/httplib/trace.py index 54ed0653d..55d9de160 100644 --- a/contrib/opencensus-ext-httplib/opencensus/ext/httplib/trace.py +++ b/contrib/opencensus-ext-httplib/opencensus/ext/httplib/trace.py @@ -76,6 +76,10 @@ def call(self, method, url, body, headers, *args, **kwargs): _span.span_kind = span_module.SpanKind.CLIENT _span.name = '[httplib]{}'.format(request_func.__name__) + # Add the component type to attributes + _tracer.add_attribute_to_current_span( + "component", "HTTP") + # Add the request url to attributes _tracer.add_attribute_to_current_span(HTTP_URL, url) diff --git a/contrib/opencensus-ext-httplib/tests/test_httplib_trace.py b/contrib/opencensus-ext-httplib/tests/test_httplib_trace.py index e85bfecdd..eed7d8f42 100644 --- a/contrib/opencensus-ext-httplib/tests/test_httplib_trace.py +++ b/contrib/opencensus-ext-httplib/tests/test_httplib_trace.py @@ -91,7 +91,8 @@ def test_wrap_httplib_request(self): with patch, patch_thread: wrapped(mock_self, method, url, body, headers) - expected_attributes = {'http.url': url, 'http.method': method} + expected_attributes = {'component': 'HTTP', + 'http.url': url, 'http.method': method} expected_name = '[httplib]request' mock_request_func.assert_called_with(mock_self, method, url, body, { diff --git a/contrib/opencensus-ext-httplib/version.py b/contrib/opencensus-ext-httplib/version.py index c652125f7..b7a1f8944 100644 --- a/contrib/opencensus-ext-httplib/version.py +++ b/contrib/opencensus-ext-httplib/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.7.2' +__version__ = '0.7.3' diff --git a/contrib/opencensus-ext-requests/CHANGELOG.md b/contrib/opencensus-ext-requests/CHANGELOG.md index 48a3e48ff..ca380ee9d 100644 --- a/contrib/opencensus-ext-requests/CHANGELOG.md +++ b/contrib/opencensus-ext-requests/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +## 0.7.3 +Released 2020-02-03 + + - Added `component` span attribute + ## 0.7.2 Released 2019-08-26 diff --git a/contrib/opencensus-ext-requests/opencensus/ext/requests/trace.py b/contrib/opencensus-ext-requests/opencensus/ext/requests/trace.py index 1118f57d6..c99862018 100644 --- a/contrib/opencensus-ext-requests/opencensus/ext/requests/trace.py +++ b/contrib/opencensus-ext-requests/opencensus/ext/requests/trace.py @@ -13,18 +13,23 @@ # limitations under the License. import logging + import requests import wrapt + +from opencensus.trace import ( + attributes_helper, + exceptions_status, + execution_context, +) +from opencensus.trace import span as span_module +from opencensus.trace import utils + try: from urllib.parse import urlparse except ImportError: from urlparse import urlparse -from opencensus.trace import attributes_helper -from opencensus.trace import exceptions_status -from opencensus.trace import execution_context -from opencensus.trace import span as span_module -from opencensus.trace import utils log = logging.getLogger(__name__) @@ -86,6 +91,10 @@ def call(url, *args, **kwargs): _span.name = '{}'.format(path) _span.span_kind = span_module.SpanKind.CLIENT + # Add the component type to attributes + _tracer.add_attribute_to_current_span( + "component", "HTTP") + # Add the requests host to attributes _tracer.add_attribute_to_current_span( HTTP_HOST, dest_url) @@ -162,6 +171,10 @@ def wrap_session_request(wrapped, instance, args, kwargs): except Exception: # pragma: NO COVER pass + # Add the component type to attributes + _tracer.add_attribute_to_current_span( + "component", "HTTP") + # Add the requests host to attributes _tracer.add_attribute_to_current_span( HTTP_HOST, dest_url) diff --git a/contrib/opencensus-ext-requests/tests/test_requests_trace.py b/contrib/opencensus-ext-requests/tests/test_requests_trace.py index 10554dfa5..c3957ae71 100644 --- a/contrib/opencensus-ext-requests/tests/test_requests_trace.py +++ b/contrib/opencensus-ext-requests/tests/test_requests_trace.py @@ -16,11 +16,12 @@ import mock import requests -from opencensus.trace.tracers import noop_tracer from opencensus.ext.requests import trace -from opencensus.trace import span as span_module, execution_context +from opencensus.trace import execution_context +from opencensus.trace import span as span_module from opencensus.trace import status as status_module +from opencensus.trace.tracers import noop_tracer class Test_requests_trace(unittest.TestCase): @@ -103,6 +104,7 @@ def test_wrap_requests(self): wrapped(url) expected_attributes = { + 'component': 'HTTP', 'http.host': 'localhost:8080', 'http.method': 'GET', 'http.path': '/test', @@ -246,6 +248,7 @@ def test_wrap_requests_timeout(self): wrapped(url) expected_attributes = { + 'component': 'HTTP', 'http.host': 'localhost:8080', 'http.method': 'GET', 'http.path': '/test', @@ -293,6 +296,7 @@ def test_wrap_requests_invalid_url(self): wrapped(url) expected_attributes = { + 'component': 'HTTP', 'http.host': 'localhost:8080', 'http.method': 'GET', 'http.path': '/test', @@ -341,6 +345,7 @@ def test_wrap_requests_exception(self): wrapped(url) expected_attributes = { + 'component': 'HTTP', 'http.host': 'localhost:8080', 'http.method': 'GET', 'http.path': '/test', @@ -387,6 +392,7 @@ def test_wrap_session_request(self): ) expected_attributes = { + 'component': 'HTTP', 'http.host': 'localhost:8080', 'http.method': 'POST', 'http.path': '/test', @@ -618,6 +624,7 @@ def test_wrap_session_request_timeout(self): ) expected_attributes = { + 'component': 'HTTP', 'http.host': 'localhost:8080', 'http.method': 'POST', 'http.path': '/test', @@ -666,6 +673,7 @@ def test_wrap_session_request_invalid_url(self): ) expected_attributes = { + 'component': 'HTTP', 'http.host': 'localhost:8080', 'http.method': 'POST', 'http.path': '/test', @@ -714,6 +722,7 @@ def test_wrap_session_request_exception(self): ) expected_attributes = { + 'component': 'HTTP', 'http.host': 'localhost:8080', 'http.method': 'POST', 'http.path': '/test', diff --git a/contrib/opencensus-ext-requests/version.py b/contrib/opencensus-ext-requests/version.py index c652125f7..b7a1f8944 100644 --- a/contrib/opencensus-ext-requests/version.py +++ b/contrib/opencensus-ext-requests/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.7.2' +__version__ = '0.7.3' diff --git a/noxfile.py b/noxfile.py index e419f2f89..113ed7dfd 100644 --- a/noxfile.py +++ b/noxfile.py @@ -51,7 +51,6 @@ def _install_dev_packages(session): def _install_test_dependencies(session): session.install('mock') session.install('pytest==4.6.4') - session.install('pytest-cov') session.install('retrying') session.install('unittest2') @@ -70,13 +69,6 @@ def unit(session): session.run( 'py.test', '--quiet', - '--cov=opencensus', - '--cov=context', - '--cov=contrib', - '--cov-append', - '--cov-config=.coveragerc', - '--cov-report=', - '--cov-fail-under=97', 'tests/unit/', 'context/', 'contrib/', @@ -136,17 +128,6 @@ def lint_setup_py(session): 'python', 'setup.py', 'check', '--restructuredtext', '--strict') -@nox.session(python='3.6') -def cover(session): - """Run the final coverage report. - This outputs the coverage report aggregating coverage from the unit - test runs (not system test runs), and then erases coverage data. - """ - session.install('coverage', 'pytest-cov') - session.run('coverage', 'report', '--show-missing', '--fail-under=100') - session.run('coverage', 'erase') - - @nox.session(python='3.6') def docs(session): """Build the docs."""