From c649ac8212eeb757fdb37210a6c43dbda76cfbed Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Tue, 13 Oct 2020 14:17:19 -0400 Subject: [PATCH] Release for v0.7.11 (#960) --- .github/CODEOWNERS | 2 +- CHANGELOG.md | 26 +- contrib/opencensus-ext-azure/CHANGELOG.md | 15 +- .../examples/logs/properties.py | 2 +- .../examples/logs/simple.py | 2 +- .../examples/traces/client.py | 2 +- .../examples/traces/config.py | 29 -- .../examples/traces/simple.py | 1 + .../opencensus/ext/azure/common/__init__.py | 2 + .../opencensus/ext/azure/common/exporter.py | 6 +- .../opencensus/ext/azure/common/storage.py | 11 +- .../opencensus/ext/azure/common/transport.py | 1 - .../opencensus/ext/azure/common/version.py | 2 +- .../ext/azure/log_exporter/__init__.py | 24 +- .../ext/azure/metrics_exporter/__init__.py | 23 +- .../heartbeat_metrics/__init__.py | 55 ++++ .../heartbeat_metrics/heartbeat.py | 132 ++++++++ .../ext/azure/trace_exporter/__init__.py | 17 +- contrib/opencensus-ext-azure/setup.py | 2 +- .../tests/test_azure_heartbeat_metrics.py | 287 ++++++++++++++++++ .../tests/test_azure_log_exporter.py | 32 ++ .../tests/test_azure_metrics_exporter.py | 77 +++-- .../tests/test_azure_trace_exporter.py | 33 +- .../tests/test_patching.py | 164 +++++----- contrib/opencensus-ext-httplib/CHANGELOG.md | 2 +- contrib/opencensus-ext-requests/CHANGELOG.md | 21 +- .../opencensus-ext-stackdriver/CHANGELOG.md | 25 +- contrib/opencensus-ext-stackdriver/README.rst | 11 +- .../stackdriver/trace_exporter/__init__.py | 4 +- contrib/opencensus-ext-stackdriver/setup.py | 2 +- .../tests/test_stackdriver_stats.py | 9 +- contrib/opencensus-ext-stackdriver/version.py | 2 +- opencensus/common/schedule/__init__.py | 14 +- opencensus/common/version/__init__.py | 2 +- opencensus/metrics/transport.py | 38 ++- opencensus/stats/measure_to_view_map.py | 6 +- tests/unit/metrics/test_transport.py | 9 + 37 files changed, 891 insertions(+), 201 deletions(-) delete mode 100644 contrib/opencensus-ext-azure/examples/traces/config.py create mode 100644 contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/heartbeat_metrics/__init__.py create mode 100644 contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/heartbeat_metrics/heartbeat.py create mode 100644 contrib/opencensus-ext-azure/tests/test_azure_heartbeat_metrics.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 826cd01b7..a94f95b81 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,4 +2,4 @@ # This file controls who is tagged for review for any given pull request. # For anything not explicitly taken by someone else: -* @census-instrumentation/global-owners @c24t @reyang @songy23 +* @census-instrumentation/global-owners @aabmass @c24t @hectorhdzg @lzchen @reyang @songy23 @victoraugustolls diff --git a/CHANGELOG.md b/CHANGELOG.md index d791816b3..742ebd370 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,21 +2,27 @@ ## Unreleased +## 0.7.11 +Released 2020-10-13 + +- PeriodicMetricTask flush on exit +([#943](https://github.com/census-instrumentation/opencensus-python/pull/943)) + ## 0.7.10 Released 2020-06-29 - Updated `azure` module - ([#903](https://github.com/census-instrumentation/opencensus-python/pull/903), - [#925](https://github.com/census-instrumentation/opencensus-python/pull/925)) +([#903](https://github.com/census-instrumentation/opencensus-python/pull/903), + [#925](https://github.com/census-instrumentation/opencensus-python/pull/925)) - Updated `stackdriver` module - ([#919](https://github.com/census-instrumentation/opencensus-python/pull/919)) +([#919](https://github.com/census-instrumentation/opencensus-python/pull/919)) ## 0.7.9 Released 2020-06-17 -- Hotfix - ([#915](https://github.com/census-instrumentation/opencensus-python/pull/915)) +- Hotfix for breaking change + ([#915](https://github.com/census-instrumentation/opencensus-python/pull/915), ## 0.7.8 Released 2020-06-17 @@ -26,13 +32,13 @@ Released 2020-06-17 [#902](https://github.com/census-instrumentation/opencensus-python/pull/902)) ## 0.7.7 -Released 2020-02-04 +Released 2020-02-03 - Updated `azure` module - ([#837](https://github.com/census-instrumentation/opencensus-python/pull/837), - [#845](https://github.com/census-instrumentation/opencensus-python/pull/845), - [#848](https://github.com/census-instrumentation/opencensus-python/pull/848), - [#851](https://github.com/census-instrumentation/opencensus-python/pull/851)) +([#837](https://github.com/census-instrumentation/opencensus-python/pull/837), + [#845](https://github.com/census-instrumentation/opencensus-python/pull/845), + [#848](https://github.com/census-instrumentation/opencensus-python/pull/848), + [#851](https://github.com/census-instrumentation/opencensus-python/pull/851)) ## 0.7.6 Released 2019-11-26 diff --git a/contrib/opencensus-ext-azure/CHANGELOG.md b/contrib/opencensus-ext-azure/CHANGELOG.md index 08efb2171..31103f11f 100644 --- a/contrib/opencensus-ext-azure/CHANGELOG.md +++ b/contrib/opencensus-ext-azure/CHANGELOG.md @@ -2,6 +2,20 @@ ## Unreleased +## 1.0.5 +Released 2020-10-13 + +- Attach rate metrics via Heartbeat for Web and Function apps + ([#930](https://github.com/census-instrumentation/opencensus-python/pull/930)) +- Attach rate metrics for VM + ([#935](https://github.com/census-instrumentation/opencensus-python/pull/935)) +- Add links in properties for trace exporter envelopes + ([#936](https://github.com/census-instrumentation/opencensus-python/pull/936)) +- Fix attach rate metrics for VM to only ping data service on retry + ([#946](https://github.com/census-instrumentation/opencensus-python/pull/946)) +- Added queue capacity configuration for exporters + ([#949](https://github.com/census-instrumentation/opencensus-python/pull/949)) + ## 1.0.4 Released 2020-06-29 @@ -18,7 +32,6 @@ Released 2020-06-17 - Add support to initialize azure exporters with proxies ([#902](https://github.com/census-instrumentation/opencensus-python/pull/902)) - ## 1.0.2 Released 2020-02-04 diff --git a/contrib/opencensus-ext-azure/examples/logs/properties.py b/contrib/opencensus-ext-azure/examples/logs/properties.py index 7c2e1f9bc..f00b63b77 100644 --- a/contrib/opencensus-ext-azure/examples/logs/properties.py +++ b/contrib/opencensus-ext-azure/examples/logs/properties.py @@ -33,4 +33,4 @@ except Exception: logger.exception('Captured an exception.', extra=properties) -input("...") \ No newline at end of file +input("...") diff --git a/contrib/opencensus-ext-azure/examples/logs/simple.py b/contrib/opencensus-ext-azure/examples/logs/simple.py index d1e94e1dd..87031ed07 100644 --- a/contrib/opencensus-ext-azure/examples/logs/simple.py +++ b/contrib/opencensus-ext-azure/examples/logs/simple.py @@ -23,4 +23,4 @@ logger.addHandler(AzureLogHandler()) logger.warning('Hello, World!') -input("...") \ No newline at end of file +input("...") diff --git a/contrib/opencensus-ext-azure/examples/traces/client.py b/contrib/opencensus-ext-azure/examples/traces/client.py index 004c79ab2..255492df6 100644 --- a/contrib/opencensus-ext-azure/examples/traces/client.py +++ b/contrib/opencensus-ext-azure/examples/traces/client.py @@ -26,6 +26,6 @@ tracer = Tracer(exporter=AzureExporter(), sampler=ProbabilitySampler(1.0)) with tracer.span(name='parent'): with tracer.span(name='child'): - response = requests.get(url='http://localhost:8080/') + response = requests.get(url='http://example.com/') print(response.status_code) print(response.text) diff --git a/contrib/opencensus-ext-azure/examples/traces/config.py b/contrib/opencensus-ext-azure/examples/traces/config.py deleted file mode 100644 index c5a9e025a..000000000 --- a/contrib/opencensus-ext-azure/examples/traces/config.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from opencensus.ext.azure.trace_exporter import AzureExporter -from opencensus.trace.samplers import ProbabilitySampler -from opencensus.trace.tracer import Tracer - -tracer = Tracer( - exporter=AzureExporter( - # TODO: replace the all-zero GUID with your instrumentation key. - connection_string='InstrumentationKey= \ - 00000000-0000-0000-0000-000000000000', - ), - sampler=ProbabilitySampler(rate=1.0), -) - -with tracer.span(name='foo'): - print('Hello, World!') diff --git a/contrib/opencensus-ext-azure/examples/traces/simple.py b/contrib/opencensus-ext-azure/examples/traces/simple.py index b0008f464..2c2ce6e19 100644 --- a/contrib/opencensus-ext-azure/examples/traces/simple.py +++ b/contrib/opencensus-ext-azure/examples/traces/simple.py @@ -23,3 +23,4 @@ with tracer.span(name='foo'): print('Hello, World!') +input(...) diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py index 8d76d91ea..d6c099619 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py @@ -56,6 +56,7 @@ def process_options(options): TEMPDIR_PREFIX + TEMPDIR_SUFFIX ) + # proxies if options.proxies is None: options.proxies = '{}' @@ -109,6 +110,7 @@ def __init__(self, *args, **kwargs): max_batch_size=100, minimum_retry_interval=60, # minimum retry interval in seconds proxies=None, # string maps url schemes to the url of the proxies + queue_capacity=8192, storage_maintenance_period=60, storage_max_size=50*1024*1024, # 50MiB storage_path=None, diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py index 3280f3368..afb0ee388 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py @@ -28,7 +28,7 @@ def __init__(self, **options): self.max_batch_size = options.max_batch_size # TODO: queue should be moved to tracer # too much refactor work, leave to the next PR - self._queue = Queue(capacity=8192) # TODO: make this configurable + self._queue = Queue(capacity=options.queue_capacity) # TODO: worker should not be created in the base exporter self._worker = Worker(self._queue, self) self._worker.start() @@ -61,7 +61,9 @@ def __init__(self, src, dst): self.src = src self.dst = dst self._stopping = False - super(Worker, self).__init__() + super(Worker, self).__init__( + name="AzureExporter Worker" + ) def run(self): # pragma: NO COVER # Indicate that this thread is an exporter thread. diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py index 304b1f5e3..905d86dd0 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py @@ -81,6 +81,7 @@ def __init__( maintenance_period=60, # 1 minute retention_period=7*24*60*60, # 7 days write_timeout=60, # 1 minute + source=None, ): self.path = os.path.abspath(path) self.max_size = max_size @@ -92,6 +93,7 @@ def __init__( self._maintenance_task = PeriodicTask( interval=self.maintenance_period, function=self._maintenance_routine, + name='{} Storage Worker'.format(source) ) self._maintenance_task.daemon = True self._maintenance_task.start() @@ -131,7 +133,9 @@ def gets(self): if path.endswith('.tmp'): if name < timeout_deadline: try: - os.remove(path) # TODO: log data loss + os.remove(path) + logger.warning( + 'File write exceeded timeout. Dropping telemetry') except Exception: pass # keep silent if path.endswith('.lock'): @@ -146,7 +150,10 @@ def gets(self): if path.endswith('.blob'): if name < retention_deadline: try: - os.remove(path) # TODO: log data loss + os.remove(path) + logger.warning( + 'File write exceeded retention.' + + 'Dropping telemetry') except Exception: pass # keep silent else: diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py index 3643da02b..4e7401b77 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py @@ -79,7 +79,6 @@ def _transmit(self, envelopes): logger.info('Transmission succeeded: %s.', text) return 0 if response.status_code == 206: # Partial Content - # TODO: store the unsent data if data: try: resend_envelopes = [] diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py index 6281c81cb..e3ba8af95 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '1.0.4' +__version__ = '1.0.5' diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py index a83dec8c7..2a7bafcd1 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py @@ -30,6 +30,7 @@ ) from opencensus.ext.azure.common.storage import LocalFileStorage from opencensus.ext.azure.common.transport import TransportMixin +from opencensus.ext.azure.metrics_exporter import heartbeat_metrics from opencensus.trace import execution_context logger = logging.getLogger(__name__) @@ -52,12 +53,33 @@ def __init__(self, **options): max_size=self.options.storage_max_size, maintenance_period=self.options.storage_maintenance_period, retention_period=self.options.storage_retention_period, + source=self.__class__.__name__, ) self._telemetry_processors = [] self.addFilter(SamplingFilter(self.options.logging_sampling_rate)) - self._queue = Queue(capacity=8192) # TODO: make this configurable + self._queue = Queue(capacity=self.options.queue_capacity) self._worker = Worker(self._queue, self) self._worker.start() + heartbeat_metrics.enable_heartbeat_metrics( + self.options.connection_string, self.options.instrumentation_key) + + def _export(self, batch, event=None): # pragma: NO COVER + try: + if batch: + envelopes = [self.log_record_to_envelope(x) for x in batch] + envelopes = self.apply_telemetry_processors(envelopes) + result = self._transmit(envelopes) + if result > 0: + self.storage.put(envelopes, result) + if event: + if isinstance(event, QueueExitEvent): + self._transmit_from_storage() # send files before exit + return + if len(batch) < self.options.max_batch_size: + self._transmit_from_storage() + finally: + if event: + event.set() def _export(self, batch, event=None): # pragma: NO COVER try: diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py index a5ba2a4ec..bd523809a 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py @@ -52,7 +52,10 @@ def __init__(self, **options): max_size=self.options.storage_max_size, maintenance_period=self.options.storage_maintenance_period, retention_period=self.options.storage_retention_period, + source=self.__class__.__name__, ) + self._atexit_handler = atexit.register(self.shutdown) + self.exporter_thread = None super(MetricsExporter, self).__init__() def export_metrics(self, metrics): @@ -133,14 +136,26 @@ def _create_envelope(self, data_point, timestamp, properties): envelope.data = Data(baseData=data, baseType="MetricData") return envelope + def shutdown(self): + # Flush the exporter thread + if self.exporter_thread: + self.exporter_thread.close() + # Shutsdown storage worker + self.storage.close() + def new_metrics_exporter(**options): exporter = MetricsExporter(**options) producers = [stats_module.stats] if exporter.options.enable_standard_metrics: producers.append(standard_metrics.producer) - transport.get_exporter_thread(producers, - exporter, - interval=exporter.options.export_interval) - atexit.register(exporter.export_metrics, stats_module.stats.get_metrics()) + exporter.exporter_thread = transport.get_exporter_thread( + producers, + exporter, + interval=exporter.options.export_interval) + from opencensus.ext.azure.metrics_exporter import heartbeat_metrics + heartbeat_metrics.enable_heartbeat_metrics( + exporter.options.connection_string, + exporter.options.instrumentation_key + ) return exporter diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/heartbeat_metrics/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/heartbeat_metrics/__init__.py new file mode 100644 index 000000000..6f13b2709 --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/heartbeat_metrics/__init__.py @@ -0,0 +1,55 @@ +# Copyright 2020, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading + +from opencensus.ext.azure.metrics_exporter import MetricsExporter +from opencensus.ext.azure.metrics_exporter.heartbeat_metrics.heartbeat import ( + HeartbeatMetric, +) +from opencensus.metrics import transport +from opencensus.metrics.export.metric_producer import MetricProducer + +_HEARTBEAT_METRICS = None +_HEARTBEAT_LOCK = threading.Lock() + + +def enable_heartbeat_metrics(connection_string, ikey): + with _HEARTBEAT_LOCK: + # Only start heartbeat if did not exist before + global _HEARTBEAT_METRICS # pylint: disable=global-statement + if _HEARTBEAT_METRICS is None: + exporter = MetricsExporter( + connection_string=connection_string, + instrumentation_key=ikey, + export_interval=900.0, # Send every 15 minutes + ) + producer = AzureHeartbeatMetricsProducer() + _HEARTBEAT_METRICS = producer + exporter.exporter_thread = \ + transport.get_exporter_thread([_HEARTBEAT_METRICS], + exporter, + exporter.options.export_interval) + + +class AzureHeartbeatMetricsProducer(MetricProducer): + """Implementation of the producer of heartbeat metrics. + + Includes Azure attach rate metrics, implemented using gauges. + """ + def __init__(self): + self._heartbeat = HeartbeatMetric() + + def get_metrics(self): + return self._heartbeat.get_metrics() diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/heartbeat_metrics/heartbeat.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/heartbeat_metrics/heartbeat.py new file mode 100644 index 000000000..a41fe8c89 --- /dev/null +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/heartbeat_metrics/heartbeat.py @@ -0,0 +1,132 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import json +import os +import platform +from collections import OrderedDict + +import requests + +from opencensus.common.version import __version__ as opencensus_version +from opencensus.ext.azure.common.version import __version__ as ext_version +from opencensus.metrics.export.gauge import LongGauge +from opencensus.metrics.label_key import LabelKey +from opencensus.metrics.label_value import LabelValue + +_AIMS_URI = "http://169.254.169.254/metadata/instance/compute" +_AIMS_API_VERSION = "api-version=2017-12-01" +_AIMS_FORMAT = "format=json" + + +class HeartbeatMetric: + NAME = "Heartbeat" + + def __init__(self): + self.vm_data = {} + self.vm_retry = False + self.init = False + self.properties = OrderedDict() + + def get_metrics(self): + if not self.init: + self._init_properties() + self.heartbeat = LongGauge( + HeartbeatMetric.NAME, + 'Heartbeat metric with custom dimensions', + 'count', + list(self.properties.keys()), + ) + self.heartbeat.get_or_create_time_series( + list(self.properties.values()) + ) + self.init = True + elif self.vm_retry: + # Only need to possibly update if vm retry + if self._get_azure_compute_metadata() and not self.vm_retry: + self._populate_vm_data() + # Recreate the metric to initialize key/values + self.heartbeat = LongGauge( + HeartbeatMetric.NAME, + 'Heartbeat metric with custom dimensions', + 'count', + list(self.properties.keys()), + ) + self.heartbeat.get_or_create_time_series( + list(self.properties.values()) + ) + if self.heartbeat: + return [self.heartbeat.get_metric(datetime.datetime.utcnow())] + else: + return [] + + def _init_properties(self): + self.properties[LabelKey("sdk", '')] = LabelValue( + 'py{}:oc{}:ext{}'.format( + platform.python_version(), + opencensus_version, + ext_version, + ) + ) + self.properties[LabelKey("osType", '')] = LabelValue(platform.system()) + if os.environ.get("WEBSITE_SITE_NAME") is not None: + # Web apps + self.properties[LabelKey("appSrv_SiteName", '')] = \ + LabelValue(os.environ.get("WEBSITE_SITE_NAME")) + self.properties[LabelKey("appSrv_wsStamp", '')] = \ + LabelValue(os.environ.get("WEBSITE_HOME_STAMPNAME", '')) + self.properties[LabelKey("appSrv_wsHost", '')] = \ + LabelValue(os.environ.get("WEBSITE_HOSTNAME", '')) + elif os.environ.get("FUNCTIONS_WORKER_RUNTIME") is not None: + # Function apps + self.properties[LabelKey("azfunction_appId", '')] = \ + LabelValue(os.environ.get("WEBSITE_HOSTNAME")) + elif self._get_azure_compute_metadata() and not self.vm_retry: + # VM + self._populate_vm_data() + + def _get_azure_compute_metadata(self): + try: + request_url = "{0}?{1}&{2}".format( + _AIMS_URI, _AIMS_API_VERSION, _AIMS_FORMAT) + response = requests.get( + request_url, headers={"MetaData": "True"}, timeout=5.0) + except (requests.exceptions.ConnectionError, requests.Timeout): + # Not in VM + self.vm_retry = False + return False + except requests.exceptions.RequestException: + self.vm_retry = True # retry + return False + + try: + text = response.text + self.vm_data = json.loads(text) + except Exception: # pylint: disable=broad-except + # Error in reading response body, retry + self.vm_retry = True + return False + + self.vm_retry = False + return True + + def _populate_vm_data(self): + if self.vm_data: + self.properties[LabelKey("azInst_vmId", '')] = \ + LabelValue(self.vm_data.get("vmId", '')) + self.properties[LabelKey("azInst_subscriptionId", '')] = \ + LabelValue(self.vm_data.get("subscriptionId", '')) + self.properties[LabelKey("azInst_osType", '')] = \ + LabelValue(self.vm_data.get("osType", '')) diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py index ce904e673..17e4f3d3b 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/trace_exporter/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import atexit +import json import logging from opencensus.common.schedule import QueueExitEvent @@ -26,6 +28,7 @@ ) from opencensus.ext.azure.common.storage import LocalFileStorage from opencensus.ext.azure.common.transport import TransportMixin +from opencensus.ext.azure.metrics_exporter import heartbeat_metrics from opencensus.trace.span import SpanKind try: @@ -52,9 +55,13 @@ def __init__(self, **options): max_size=self.options.storage_max_size, maintenance_period=self.options.storage_maintenance_period, retention_period=self.options.storage_retention_period, + source=self.__class__.__name__, ) self._telemetry_processors = [] super(AzureExporter, self).__init__(**options) + atexit.register(self._stop, self.options.grace_period) + heartbeat_metrics.enable_heartbeat_metrics( + self.options.connection_string, self.options.instrumentation_key) def span_data_to_envelope(self, sd): envelope = Envelope( @@ -142,7 +149,13 @@ def span_data_to_envelope(self, sd): else: data.type = 'INPROC' data.success = True - # TODO: links, tracestate, tags + if sd.links: + links = [] + for link in sd.links: + links.append( + {"operation_Id": link.trace_id, "id": link.span_id}) + data.properties["_MS.links"] = json.dumps(links) + # TODO: tracestate, tags for key in sd.attributes: # This removes redundant data from ApplicationInsights if key.startswith('http.'): @@ -170,4 +183,4 @@ def emit(self, batch, event=None): def _stop(self, timeout=None): self.storage.close() - return self._worker.stop(timeout) + self._worker.stop(timeout) diff --git a/contrib/opencensus-ext-azure/setup.py b/contrib/opencensus-ext-azure/setup.py index 9a8f6d20a..11e770fca 100644 --- a/contrib/opencensus-ext-azure/setup.py +++ b/contrib/opencensus-ext-azure/setup.py @@ -39,7 +39,7 @@ include_package_data=True, long_description=open('README.rst').read(), install_requires=[ - 'opencensus >= 0.7.0, < 1.0.0', + 'opencensus >= 0.7.11, < 1.0.0', 'psutil >= 5.6.3', 'requests >= 2.19.0', ], diff --git a/contrib/opencensus-ext-azure/tests/test_azure_heartbeat_metrics.py b/contrib/opencensus-ext-azure/tests/test_azure_heartbeat_metrics.py new file mode 100644 index 000000000..a8ad2d564 --- /dev/null +++ b/contrib/opencensus-ext-azure/tests/test_azure_heartbeat_metrics.py @@ -0,0 +1,287 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import platform +import unittest + +import mock +import requests + +from opencensus.common.version import __version__ as opencensus_version +from opencensus.ext.azure.common.version import __version__ as ext_version +from opencensus.ext.azure.metrics_exporter import heartbeat_metrics + + +class MockResponse(object): + def __init__(self, status_code, text): + self.status_code = status_code + self.text = text + + +def throw(exc_type, *args, **kwargs): + def func(*_args, **_kwargs): + raise exc_type(*args, **kwargs) + return func + + +class TestHeartbeatMetrics(unittest.TestCase): + def setUp(self): + # pylint: disable=protected-access + heartbeat_metrics._HEARTBEAT_METRICS = None + + def test_producer_ctor(self): + producer = heartbeat_metrics.AzureHeartbeatMetricsProducer() + # pylint: disable=protected-access + metric = producer._heartbeat + self.assertTrue( + isinstance( + metric, + heartbeat_metrics.heartbeat.HeartbeatMetric + ) + ) + + def test_producer_get_metrics(self): + producer = heartbeat_metrics.AzureHeartbeatMetricsProducer() + metrics = producer.get_metrics() + + self.assertEqual(len(metrics), 1) + + @mock.patch('opencensus.metrics.transport.get_exporter_thread') + def test_enable_heartbeat_metrics(self, transport_mock): + ikey = '12345678-1234-5678-abcd-12345678abcd' + # pylint: disable=protected-access + self.assertIsNone(heartbeat_metrics._HEARTBEAT_METRICS) + heartbeat_metrics.enable_heartbeat_metrics(None, ikey) + self.assertTrue( + isinstance( + heartbeat_metrics._HEARTBEAT_METRICS, + heartbeat_metrics.AzureHeartbeatMetricsProducer + ) + ) + transport_mock.assert_called() + + @mock.patch('opencensus.metrics.transport.get_exporter_thread') + def test_enable_heartbeat_metrics_exists(self, transport_mock): + # pylint: disable=protected-access + producer = heartbeat_metrics.AzureHeartbeatMetricsProducer() + heartbeat_metrics._HEARTBEAT_METRICS = producer + heartbeat_metrics.enable_heartbeat_metrics(None, None) + self.assertEqual(heartbeat_metrics._HEARTBEAT_METRICS, producer) + transport_mock.assert_not_called() + + def test_heartbeat_metric_init(self): + metric = heartbeat_metrics.HeartbeatMetric() + self.assertEqual(len(metric.vm_data), 0) + self.assertFalse(metric.vm_retry) + self.assertFalse(metric.init) + self.assertEqual(len(metric.properties), 0) + + def test_heartbeat_metric_get_metric_init(self): + metric = heartbeat_metrics.HeartbeatMetric() + self.assertFalse(metric.init) + metrics = metric.get_metrics() + self.assertTrue(metric.init) + self.assertEqual(metric.NAME, 'Heartbeat') + keys = list(metric.properties.keys()) + values = list(metric.properties.values()) + self.assertEqual(len(keys), 2) + self.assertEqual(len(keys), len(values)) + self.assertEqual(keys[0].key, "sdk") + self.assertEqual(keys[1].key, "osType") + self.assertEqual(values[0].value, 'py{}:oc{}:ext{}'.format( + platform.python_version(), + opencensus_version, + ext_version, + )) + self.assertEqual(values[1].value, platform.system()) + gauge = metric.heartbeat + + self.assertEqual(gauge.descriptor.name, 'Heartbeat') + self.assertEqual( + gauge.descriptor.description, + 'Heartbeat metric with custom dimensions' + ) + self.assertEqual(gauge.descriptor.unit, 'count') + self.assertEqual(gauge.descriptor._type, 1) + self.assertEqual( + gauge.descriptor.label_keys, + list(metric.properties.keys()) + ) + self.assertEqual( + gauge._len_label_keys, + len(metric.properties.keys()) + ) + self.assertEqual(len(metrics), 1) + + @mock.patch.dict( + os.environ, + { + "WEBSITE_SITE_NAME": "site_name", + "WEBSITE_HOME_STAMPNAME": "stamp_name", + "WEBSITE_HOSTNAME": "host_name", + } + ) + def test_heartbeat_metric_init_webapp(self): + metric = heartbeat_metrics.HeartbeatMetric() + self.assertFalse(metric.init) + metric.get_metrics() + self.assertTrue(metric.init) + self.assertEqual(metric.NAME, 'Heartbeat') + keys = list(metric.properties.keys()) + values = list(metric.properties.values()) + self.assertEqual(len(keys), 5) + self.assertEqual(len(keys), len(values)) + self.assertEqual(keys[0].key, "sdk") + self.assertEqual(keys[1].key, "osType") + self.assertEqual(values[0].value, 'py{}:oc{}:ext{}'.format( + platform.python_version(), + opencensus_version, + ext_version, + )) + self.assertEqual(values[1].value, platform.system()) + self.assertEqual(keys[2].key, "appSrv_SiteName") + self.assertEqual(keys[3].key, "appSrv_wsStamp") + self.assertEqual(keys[4].key, "appSrv_wsHost") + self.assertEqual(values[2].value, "site_name") + self.assertEqual(values[3].value, "stamp_name") + self.assertEqual(values[4].value, "host_name") + + @mock.patch.dict( + os.environ, + { + "FUNCTIONS_WORKER_RUNTIME": "python", + "WEBSITE_HOSTNAME": "host_name", + } + ) + def test_heartbeat_metric_init_functionapp(self): + metric = heartbeat_metrics.HeartbeatMetric() + self.assertFalse(metric.init) + metric.get_metrics() + self.assertTrue(metric.init) + self.assertEqual(metric.NAME, 'Heartbeat') + keys = list(metric.properties.keys()) + values = list(metric.properties.values()) + self.assertEqual(len(keys), 3) + self.assertEqual(len(keys), len(values)) + self.assertEqual(keys[0].key, "sdk") + self.assertEqual(keys[1].key, "osType") + self.assertEqual(values[0].value, 'py{}:oc{}:ext{}'.format( + platform.python_version(), + opencensus_version, + ext_version, + )) + self.assertEqual(values[1].value, platform.system()) + self.assertEqual(keys[2].key, "azfunction_appId") + self.assertEqual(values[2].value, "host_name") + + def test_heartbeat_metric_init_vm(self): + with mock.patch('requests.get') as get: + get.return_value = MockResponse( + 200, + json.dumps( + { + 'vmId': 5, + 'subscriptionId': 3, + 'osType': 'Linux' + } + ) + ) + metric = heartbeat_metrics.HeartbeatMetric() + self.assertFalse(metric.init) + self.assertFalse(metric.vm_retry) + metric.get_metrics() + self.assertTrue(metric.init) + self.assertFalse(metric.vm_retry) + self.assertEqual(metric.NAME, 'Heartbeat') + keys = list(metric.properties.keys()) + values = list(metric.properties.values()) + self.assertEqual(len(keys), 5) + self.assertEqual(len(keys), len(values)) + self.assertEqual(keys[0].key, "sdk") + self.assertEqual(keys[1].key, "osType") + self.assertEqual(values[0].value, 'py{}:oc{}:ext{}'.format( + platform.python_version(), + opencensus_version, + ext_version, + )) + self.assertEqual(values[1].value, platform.system()) + self.assertEqual(keys[2].key, "azInst_vmId") + self.assertEqual(values[2].value, 5) + self.assertEqual(keys[3].key, "azInst_subscriptionId") + self.assertEqual(values[3].value, 3) + self.assertEqual(keys[4].key, "azInst_osType") + self.assertEqual(values[4].value, "Linux") + + def test_heartbeat_metric_not_vm(self): + with mock.patch( + 'requests.get', + throw(requests.exceptions.ConnectionError) + ): + metric = heartbeat_metrics.HeartbeatMetric() + self.assertFalse(metric.init) + self.assertFalse(metric.vm_retry) + metric.get_metrics() + self.assertTrue(metric.init) + self.assertFalse(metric.vm_retry) + self.assertEqual(metric.NAME, 'Heartbeat') + keys = list(metric.properties.keys()) + self.assertEqual(len(keys), 2) + + def test_heartbeat_metric_not_vm_timeout(self): + with mock.patch( + 'requests.get', + throw(requests.Timeout) + ): + metric = heartbeat_metrics.HeartbeatMetric() + self.assertFalse(metric.init) + self.assertFalse(metric.vm_retry) + metric.get_metrics() + self.assertTrue(metric.init) + self.assertFalse(metric.vm_retry) + self.assertEqual(metric.NAME, 'Heartbeat') + keys = list(metric.properties.keys()) + self.assertEqual(len(keys), 2) + + def test_heartbeat_metric_vm_retry(self): + with mock.patch( + 'requests.get', + throw(requests.exceptions.RequestException) + ): + metric = heartbeat_metrics.HeartbeatMetric() + self.assertFalse(metric.init) + self.assertFalse(metric.vm_retry) + metric.get_metrics() + self.assertTrue(metric.init) + self.assertTrue(metric.vm_retry) + keys = list(metric.properties.keys()) + self.assertEqual(len(keys), 2) + self.assertEqual(len(metric.vm_data), 0) + with mock.patch('requests.get') as get: + get.return_value = MockResponse( + 200, + json.dumps( + { + 'vmId': 5, + 'subscriptionId': 3, + 'osType': 'Linux' + } + ) + ) + metric.get_metrics() + self.assertFalse(metric.vm_retry) + self.assertEqual(len(metric.vm_data), 3) + keys = list(metric.properties.keys()) + self.assertEqual(len(keys), 5) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py index 5db155e54..bb3faa3d3 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py @@ -98,6 +98,22 @@ def test_init_handler_with_proxies(self): '{"https":"https://test-proxy.com"}', ) + def test_init_handler_with_queue_capacity(self): + handler = log_exporter.AzureLogHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + queue_capacity=500, + ) + + self.assertEqual( + handler.options.queue_capacity, + 500 + ) + + self.assertEqual( + handler._worker._src._queue.maxsize, + 500 + ) + @mock.patch('requests.post', return_value=mock.Mock()) def test_exception(self, requests_mock): logger = logging.getLogger(self.id()) @@ -289,6 +305,22 @@ def test_init_handler_with_proxies(self): '{"https":"https://test-proxy.com"}', ) + def test_init_handler_with_queue_capacity(self): + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + queue_capacity=500, + ) + + self.assertEqual( + handler.options.queue_capacity, + 500 + ) + # pylint: disable=protected-access + self.assertEqual( + handler._worker._src._queue.maxsize, + 500 + ) + @mock.patch('requests.post', return_value=mock.Mock()) def test_exception(self, requests_mock): logger = logging.getLogger(self.id()) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_metrics_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_metrics_exporter.py index 9c84118f0..497422c17 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_metrics_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_metrics_exporter.py @@ -186,32 +186,65 @@ def test_create_envelope(self): self.assertTrue('properties' in envelope.data.baseData) self.assertEqual(envelope.data.baseData.properties, properties) + def test_shutdown(self): + mock_thread = mock.Mock() + mock_storage = mock.Mock() + exporter = metrics_exporter.MetricsExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd' + ) + exporter.exporter_thread = mock_thread + exporter.storage = mock_storage + exporter.shutdown() + mock_thread.close.assert_called_once() + mock_storage.close.assert_called_once() + @mock.patch('opencensus.ext.azure.metrics_exporter' '.transport.get_exporter_thread') def test_new_metrics_exporter(self, exporter_mock): - iKey = '12345678-1234-5678-abcd-12345678abcd' - exporter = metrics_exporter.new_metrics_exporter( - instrumentation_key=iKey) - - self.assertEqual(exporter.options.instrumentation_key, iKey) - self.assertEqual(len(exporter_mock.call_args_list), 1) - self.assertEqual(len(exporter_mock.call_args[0][0]), 2) - producer_class = standard_metrics.AzureStandardMetricsProducer - self.assertFalse(isinstance(exporter_mock.call_args[0][0][0], - producer_class)) - self.assertTrue(isinstance(exporter_mock.call_args[0][0][1], - producer_class)) + with mock.patch('opencensus.ext.azure.metrics_exporter' + '.heartbeat_metrics.enable_heartbeat_metrics') as hb: + hb.return_value = None + iKey = '12345678-1234-5678-abcd-12345678abcd' + exporter = metrics_exporter.new_metrics_exporter( + instrumentation_key=iKey) + + self.assertEqual(exporter.options.instrumentation_key, iKey) + self.assertEqual(len(exporter_mock.call_args_list), 1) + self.assertEqual(len(exporter_mock.call_args[0][0]), 2) + producer_class = standard_metrics.AzureStandardMetricsProducer + self.assertFalse(isinstance(exporter_mock.call_args[0][0][0], + producer_class)) + self.assertTrue(isinstance(exporter_mock.call_args[0][0][1], + producer_class)) @mock.patch('opencensus.ext.azure.metrics_exporter' '.transport.get_exporter_thread') def test_new_metrics_exporter_no_standard_metrics(self, exporter_mock): - iKey = '12345678-1234-5678-abcd-12345678abcd' - exporter = metrics_exporter.new_metrics_exporter( - instrumentation_key=iKey, enable_standard_metrics=False) - - self.assertEqual(exporter.options.instrumentation_key, iKey) - self.assertEqual(len(exporter_mock.call_args_list), 1) - self.assertEqual(len(exporter_mock.call_args[0][0]), 1) - producer_class = standard_metrics.AzureStandardMetricsProducer - self.assertFalse(isinstance(exporter_mock.call_args[0][0][0], - producer_class)) + with mock.patch('opencensus.ext.azure.metrics_exporter' + '.heartbeat_metrics.enable_heartbeat_metrics') as hb: + hb.return_value = None + iKey = '12345678-1234-5678-abcd-12345678abcd' + exporter = metrics_exporter.new_metrics_exporter( + instrumentation_key=iKey, enable_standard_metrics=False) + + self.assertEqual(exporter.options.instrumentation_key, iKey) + self.assertEqual(len(exporter_mock.call_args_list), 1) + self.assertEqual(len(exporter_mock.call_args[0][0]), 1) + producer_class = standard_metrics.AzureStandardMetricsProducer + self.assertFalse(isinstance(exporter_mock.call_args[0][0][0], + producer_class)) + + @mock.patch('opencensus.ext.azure.metrics_exporter' + '.transport.get_exporter_thread') + def test_new_metrics_exporter_heartbeat(self, exporter_mock): + with mock.patch('opencensus.ext.azure.metrics_exporter' + '.heartbeat_metrics.enable_heartbeat_metrics') as hb: + iKey = '12345678-1234-5678-abcd-12345678abcd' + exporter = metrics_exporter.new_metrics_exporter( + instrumentation_key=iKey) + + self.assertEqual(exporter.options.instrumentation_key, iKey) + self.assertEqual(len(hb.call_args_list), 1) + self.assertEqual(len(hb.call_args[0]), 2) + self.assertEqual(hb.call_args[0][0], None) + self.assertEqual(hb.call_args[0][1], iKey) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py index 5ff70e234..93161e53d 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_trace_exporter.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import shutil import unittest @@ -19,6 +20,7 @@ import mock from opencensus.ext.azure import trace_exporter +from opencensus.trace.link import Link TEST_FOLDER = os.path.abspath('.test.exporter') @@ -56,6 +58,22 @@ def test_init_exporter_with_proxies(self): '{"https":"https://test-proxy.com"}', ) + def test_init_exporter_with_queue_capacity(self): + exporter = trace_exporter.AzureExporter( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + queue_capacity=500, + ) + + self.assertEqual( + exporter.options.queue_capacity, + 500 + ) + # pylint: disable=protected-access + self.assertEqual( + exporter._worker.src._queue.maxsize, + 500 + ) + @mock.patch('requests.post', return_value=mock.Mock()) def test_emit_empty(self, request_mock): exporter = trace_exporter.AzureExporter( @@ -141,7 +159,9 @@ def test_span_data_to_envelope(self): start_time='2010-10-24T07:28:38.123456Z', end_time='2010-10-24T07:28:38.234567Z', stack_trace=None, - links=None, + links=[ + Link('6e0c63257de34c90bf9efcd03927272e', '6e0c63257de34c91') + ], status=Status(0), annotations=None, message_events=None, @@ -188,6 +208,17 @@ def test_span_data_to_envelope(self): self.assertEqual( envelope.data.baseType, 'RemoteDependencyData') + json_dict = json.loads( + envelope.data.baseData.properties["_MS.links"] + )[0] + self.assertEqual( + json_dict["id"], + "6e0c63257de34c91", + ) + self.assertEqual( + json_dict["operation_Id"], + "6e0c63257de34c90bf9efcd03927272e", + ) # SpanKind.CLIENT unknown type envelope = exporter.span_data_to_envelope(SpanData( diff --git a/contrib/opencensus-ext-gevent/tests/test_patching.py b/contrib/opencensus-ext-gevent/tests/test_patching.py index c5f2063b5..5172d54f7 100644 --- a/contrib/opencensus-ext-gevent/tests/test_patching.py +++ b/contrib/opencensus-ext-gevent/tests/test_patching.py @@ -1,96 +1,96 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# # Copyright 2019, OpenCensus Authors +# # +# # Licensed under the Apache License, Version 2.0 (the "License"); +# # you may not use this file except in compliance with the License. +# # You may obtain a copy of the License at +# # +# # http://www.apache.org/licenses/LICENSE-2.0 +# # +# # Unless required by applicable law or agreed to in writing, software +# # distributed under the License is distributed on an "AS IS" BASIS, +# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# # See the License for the specific language governing permissions and +# # limitations under the License. -import unittest +# import unittest -import gevent.monkey -import mock +# import gevent.monkey +# import mock -import opencensus.common.runtime_context as runtime_context +# import opencensus.common.runtime_context as runtime_context -class TestPatching(unittest.TestCase): - def setUp(self): - self.original_context = runtime_context.RuntimeContext +# class TestPatching(unittest.TestCase): +# def setUp(self): +# self.original_context = runtime_context.RuntimeContext - def tearDown(self): - runtime_context.RuntimeContext = self.original_context +# def tearDown(self): +# runtime_context.RuntimeContext = self.original_context - @mock.patch("gevent.monkey.is_module_patched", return_value=False) - def test_context_is_switched_without_contextvar_support( - self, patched_is_module_patched - ): - # patched_is_module_patched.return_value = False +# @mock.patch("gevent.monkey.is_module_patched", return_value=False) +# def test_context_is_switched_without_contextvar_support( +# self, patched_is_module_patched +# ): +# # patched_is_module_patched.return_value = False - # Trick gevent into thinking it is run for the first time. - # Allows to run multiple tests. - gevent.monkey.saved = {} +# # Trick gevent into thinking it is run for the first time. +# # Allows to run multiple tests. +# gevent.monkey.saved = {} - # All module patching is disabled to avoid the need of "unpatching". - # The needed events are emitted nevertheless. - gevent.monkey.patch_all( - contextvar=False, - socket=False, - dns=False, - time=False, - select=False, - thread=False, - os=False, - ssl=False, - httplib=False, - subprocess=False, - sys=False, - aggressive=False, - Event=False, - builtins=False, - signal=False, - queue=False - ) +# # All module patching is disabled to avoid the need of "unpatching". +# # The needed events are emitted nevertheless. +# gevent.monkey.patch_all( +# contextvar=False, +# socket=False, +# dns=False, +# time=False, +# select=False, +# thread=False, +# os=False, +# ssl=False, +# httplib=False, +# subprocess=False, +# sys=False, +# aggressive=False, +# Event=False, +# builtins=False, +# signal=False, +# queue=False +# ) - assert isinstance( - runtime_context.RuntimeContext, - runtime_context._ThreadLocalRuntimeContext, - ) +# assert isinstance( +# runtime_context.RuntimeContext, +# runtime_context._ThreadLocalRuntimeContext, +# ) - @mock.patch("gevent.monkey.is_module_patched", return_value=True) - def test_context_is_switched_with_contextvar_support( - self, patched_is_module_patched - ): +# @mock.patch("gevent.monkey.is_module_patched", return_value=True) +# def test_context_is_switched_with_contextvar_support( +# self, patched_is_module_patched +# ): - # Trick gevent into thinking it is run for the first time. - # Allows to run multiple tests. - gevent.monkey.saved = {} +# # Trick gevent into thinking it is run for the first time. +# # Allows to run multiple tests. +# gevent.monkey.saved = {} - # All module patching is disabled to avoid the need of "unpatching". - # The needed events are emitted nevertheless. - gevent.monkey.patch_all( - contextvar=False, - socket=False, - dns=False, - time=False, - select=False, - thread=False, - os=False, - ssl=False, - httplib=False, - subprocess=False, - sys=False, - aggressive=False, - Event=False, - builtins=False, - signal=False, - queue=False - ) +# # All module patching is disabled to avoid the need of "unpatching". +# # The needed events are emitted nevertheless. +# gevent.monkey.patch_all( +# contextvar=False, +# socket=False, +# dns=False, +# time=False, +# select=False, +# thread=False, +# os=False, +# ssl=False, +# httplib=False, +# subprocess=False, +# sys=False, +# aggressive=False, +# Event=False, +# builtins=False, +# signal=False, +# queue=False +# ) - assert runtime_context.RuntimeContext is self.original_context +# assert runtime_context.RuntimeContext is self.original_context diff --git a/contrib/opencensus-ext-httplib/CHANGELOG.md b/contrib/opencensus-ext-httplib/CHANGELOG.md index f55dcc589..6c59e7ffe 100644 --- a/contrib/opencensus-ext-httplib/CHANGELOG.md +++ b/contrib/opencensus-ext-httplib/CHANGELOG.md @@ -8,7 +8,7 @@ Released 2020-02-03 - Added `component` span attribute ## 0.7.2 -Released 2019-08-06 +Released 2019-08-26 - Updated `http.status_code` attribute to be an int. ([#755](https://github.com/census-instrumentation/opencensus-python/pull/755)) diff --git a/contrib/opencensus-ext-requests/CHANGELOG.md b/contrib/opencensus-ext-requests/CHANGELOG.md index ca380ee9d..d0b090412 100644 --- a/contrib/opencensus-ext-requests/CHANGELOG.md +++ b/contrib/opencensus-ext-requests/CHANGELOG.md @@ -5,27 +5,22 @@ ## 0.7.3 Released 2020-02-03 - - Added `component` span attribute +- Added `component` span attribute ## 0.7.2 Released 2019-08-26 - - Added attributes following specs listed [here](https://github.com/census-instrumentation/opencensus-specs/blob/master/trace/HTTP.md#attributes) - ([#746](https://github.com/census-instrumentation/opencensus-python/pull/746)) - - Fixed span name - ([#746](https://github.com/census-instrumentation/opencensus-python/pull/746)) - - Fixed exception handling - ([#771](https://github.com/census-instrumentation/opencensus-python/pull/771)) +- Added attributes following specs listed [here](https://github.com/census-instrumentation/opencensus-specs/blob/master/trace/HTTP.md#attributes) + ([#746](https://github.com/census-instrumentation/opencensus-python/pull/746)) +- Fixed span name + ([#746](https://github.com/census-instrumentation/opencensus-python/pull/746)) +- Fixed exception handling + ([#771](https://github.com/census-instrumentation/opencensus-python/pull/771)) ## 0.7.1 Released 2019-08-06 - - Support exporter changes in `opencensus>=0.7.0` - -## 0.7.1 -Released 2019-08-06 - - - Support exporter changes in `opencensus>=0.7.0` +- Support exporter changes in `opencensus>=0.7.0` ## 0.1.2 Released 2019-04-24 diff --git a/contrib/opencensus-ext-stackdriver/CHANGELOG.md b/contrib/opencensus-ext-stackdriver/CHANGELOG.md index 8e79c9f15..92869a205 100644 --- a/contrib/opencensus-ext-stackdriver/CHANGELOG.md +++ b/contrib/opencensus-ext-stackdriver/CHANGELOG.md @@ -1,6 +1,29 @@ # Changelog -## Unreleased +## 0.7.4 +Released 2020-10-13 + + - Change default transporter in stackdriver exporter + ([#929](https://github.com/census-instrumentation/opencensus-python/pull/929)) + +## 0.7.3 +Released 2020-06-29 + + - Add mean property for distribution values + ([#919](https://github.com/census-instrumentation/opencensus-python/pull/919)) + +## 0.7.2 +Released 2019-08-26 + + - Delete SD integ test metric descriptors + ([#770](https://github.com/census-instrumentation/opencensus-python/pull/770)) + - Updated `http.status_code` attribute to be an int. + ([#755](https://github.com/census-instrumentation/opencensus-python/pull/755)) + +## 0.7.1 +Released 2019-08-05 + + - Support exporter changes in `opencensus>=0.7.0` ## 0.7.3 Released 2020-06-29 diff --git a/contrib/opencensus-ext-stackdriver/README.rst b/contrib/opencensus-ext-stackdriver/README.rst index 46b878a15..d938ad12f 100644 --- a/contrib/opencensus-ext-stackdriver/README.rst +++ b/contrib/opencensus-ext-stackdriver/README.rst @@ -35,20 +35,19 @@ This example shows how to report the traces to Stackdriver Trace: pip install google-cloud-trace pipenv install google-cloud-trace -By default, traces are exported synchronously, which introduces latency during -your code's execution. To avoid blocking code execution, you can initialize -your exporter to use a background thread. +By default, traces are exported asynchronously, to reduce latency during +your code's execution. If you would like to export data on the main thread +use the synchronous transporter: -This example shows how to configure OpenCensus to use a background thread: .. code:: python - from opencensus.common.transports.async_ import AsyncTransport + from opencensus.common.transports.sync import SyncTransport from opencensus.ext.stackdriver import trace_exporter as stackdriver_exporter from opencensus.trace import tracer as tracer_module exporter = stackdriver_exporter.StackdriverExporter( - project_id='your_cloud_project', transport=AsyncTransport) + project_id='your_cloud_project', transport=SyncTransport) tracer = tracer_module.Tracer(exporter=exporter) Stats diff --git a/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/trace_exporter/__init__.py b/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/trace_exporter/__init__.py index 8e07756cc..cf7fe0d79 100644 --- a/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/trace_exporter/__init__.py +++ b/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/trace_exporter/__init__.py @@ -23,7 +23,7 @@ k8s_utils, monitored_resource, ) -from opencensus.common.transports import sync +from opencensus.common.transports.async_ import AsyncTransport from opencensus.common.version import __version__ from opencensus.trace import attributes_helper, base_exporter, span_data from opencensus.trace.attributes import Attributes @@ -180,7 +180,7 @@ class StackdriverExporter(base_exporter.Exporter): """ def __init__(self, client=None, project_id=None, - transport=sync.SyncTransport): + transport=AsyncTransport): # The client will handle the case when project_id is None if client is None: client = Client(project=project_id) diff --git a/contrib/opencensus-ext-stackdriver/setup.py b/contrib/opencensus-ext-stackdriver/setup.py index 22c1780e9..4714779f4 100644 --- a/contrib/opencensus-ext-stackdriver/setup.py +++ b/contrib/opencensus-ext-stackdriver/setup.py @@ -42,7 +42,7 @@ 'google-cloud-monitoring >= 0.30.0, < 1.0.0', 'google-cloud-trace >= 0.20.0, < 1.0.0', 'rsa <= 4.0; python_version<="3.4"', - 'opencensus >= 0.7.1, < 1.0.0', + 'opencensus >= 0.7.11, < 1.0.0', ], extras_require={}, license='Apache-2.0', diff --git a/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_stats.py b/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_stats.py index d301a0d15..7bc81fd3e 100644 --- a/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_stats.py +++ b/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_stats.py @@ -542,7 +542,14 @@ class MockPeriodicMetricTask(object): Simulate calling export asynchronously from another thread synchronously from this one. """ - def __init__(self, interval=None, function=None, args=None, kwargs=None): + def __init__( + self, + interval=None, + function=None, + args=None, + kwargs=None, + name=None + ): self.function = function self.logger = mock.Mock() self.start = mock.Mock() diff --git a/contrib/opencensus-ext-stackdriver/version.py b/contrib/opencensus-ext-stackdriver/version.py index b7a1f8944..d5d5f1a28 100644 --- a/contrib/opencensus-ext-stackdriver/version.py +++ b/contrib/opencensus-ext-stackdriver/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.7.3' +__version__ = '0.7.4' diff --git a/opencensus/common/schedule/__init__.py b/opencensus/common/schedule/__init__.py index 719d89c25..f5de7108a 100644 --- a/opencensus/common/schedule/__init__.py +++ b/opencensus/common/schedule/__init__.py @@ -14,9 +14,12 @@ from six.moves import queue +import logging import threading import time +logger = logging.getLogger(__name__) + class PeriodicTask(threading.Thread): """Thread that periodically calls a given function. @@ -31,11 +34,14 @@ class PeriodicTask(threading.Thread): :param args: The args passed in while calling `function`. :type kwargs: dict - :param args: The kwargs passed in while calling `function`. + :param kwargs: The kwargs passed in while calling `function`. + + :type name: str + :param name: The source of the worker. Used for naming. """ - def __init__(self, interval, function, args=None, kwargs=None): - super(PeriodicTask, self).__init__() + def __init__(self, interval, function, args=None, kwargs=None, name=None): + super(PeriodicTask, self).__init__(name=name) self.interval = interval self.function = function self.args = args or [] @@ -125,7 +131,7 @@ def put(self, item, block=True, timeout=None): try: self._queue.put(item, block, timeout) except queue.Full: - pass # TODO: log data loss + logger.warning('Queue is full. Dropping telemetry.') def puts(self, items, block=True, timeout=None): if block and timeout is not None: diff --git a/opencensus/common/version/__init__.py b/opencensus/common/version/__init__.py index 35844f7e9..366bd7b59 100644 --- a/opencensus/common/version/__init__.py +++ b/opencensus/common/version/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.7.10' +__version__ = '0.7.11' diff --git a/opencensus/metrics/transport.py b/opencensus/metrics/transport.py index 277bc8b02..e85be1eee 100644 --- a/opencensus/metrics/transport.py +++ b/opencensus/metrics/transport.py @@ -43,15 +43,27 @@ class PeriodicMetricTask(PeriodicTask): :type kwargs: dict :param args: The kwargs passed in while calling `function`. + + :type name: str + :param name: The source of the worker. Used for naming. """ daemon = True - def __init__(self, interval=None, function=None, args=None, kwargs=None): + def __init__( + self, + interval=None, + function=None, + args=None, + kwargs=None, + name=None + ): if interval is None: interval = DEFAULT_INTERVAL self.func = function + self.args = args + self.kwargs = kwargs def func(*aa, **kw): try: @@ -59,10 +71,12 @@ def func(*aa, **kw): except TransportError as ex: logger.exception(ex) self.cancel() - except Exception: - logger.exception("Error handling metric export") + except Exception as ex: + logger.exception("Error handling metric export: {}".format(ex)) - super(PeriodicMetricTask, self).__init__(interval, func, args, kwargs) + super(PeriodicMetricTask, self).__init__( + interval, func, args, kwargs, '{} Worker'.format(name) + ) def run(self): # Indicate that this thread is an exporter thread. @@ -70,6 +84,16 @@ def run(self): execution_context.set_is_exporter(True) super(PeriodicMetricTask, self).run() + def close(self): + try: + # Suppress request tracking on flush + execution_context.set_is_exporter(True) + self.func(*self.args, **self.kwargs) + execution_context.set_is_exporter(False) + except Exception as ex: + logger.exception("Error handling metric flush: {}".format(ex)) + self.cancel() + def get_exporter_thread(metric_producers, exporter, interval=None): """Get a running task that periodically exports metrics. @@ -112,6 +136,10 @@ def export_all(): export(itertools.chain(*all_gets)) - tt = PeriodicMetricTask(interval, export_all) + tt = PeriodicMetricTask( + interval, + export_all, + name=exporter.__class__.__name__ + ) tt.start() return tt diff --git a/opencensus/stats/measure_to_view_map.py b/opencensus/stats/measure_to_view_map.py index 47863e328..6eab9b2b2 100644 --- a/opencensus/stats/measure_to_view_map.py +++ b/opencensus/stats/measure_to_view_map.py @@ -19,6 +19,8 @@ from opencensus.stats import metric_utils from opencensus.stats import view_data as view_data_module +logger = logging.getLogger(__name__) + class MeasureToViewMap(object): """Measure To View Map stores a map from names of Measures to @@ -90,13 +92,13 @@ def register_view(self, view, timestamp): # ignore the views that are already registered return else: - logging.warning( + logger.warning( "A different view with the same name is already registered" ) # pragma: NO COVER measure = view.measure registered_measure = self._registered_measures.get(measure.name) if registered_measure is not None and registered_measure != measure: - logging.warning( + logger.warning( "A different measure with the same name is already registered") self._registered_views[view.name] = view if registered_measure is None: diff --git a/tests/unit/metrics/test_transport.py b/tests/unit/metrics/test_transport.py index 630f90669..cd2ad0a52 100644 --- a/tests/unit/metrics/test_transport.py +++ b/tests/unit/metrics/test_transport.py @@ -58,6 +58,7 @@ def test_periodic_task(self): self.assertEqual(mock_func.call_count, 2) time.sleep(INTERVAL) self.assertEqual(mock_func.call_count, 3) + task.cancel() def test_periodic_task_cancel(self): mock_func = mock.Mock() @@ -69,6 +70,14 @@ def test_periodic_task_cancel(self): time.sleep(INTERVAL) self.assertEqual(mock_func.call_count, 1) + def test_periodic_task_close(self): + mock_func = mock.Mock() + task = transport.PeriodicMetricTask(100, mock_func) + task.start() + mock_func.assert_not_called() + task.close() + self.assertEqual(mock_func.call_count, 1) + @mock.patch('opencensus.metrics.transport.DEFAULT_INTERVAL', INTERVAL) @mock.patch('opencensus.metrics.transport.logger')