From a51cdb9472c662615631ac48afccc0a050f91b32 Mon Sep 17 00:00:00 2001 From: Leighton Chen Date: Mon, 29 Jun 2020 15:32:34 -0700 Subject: [PATCH] Release for v0.7.10 (#926) --- CHANGELOG.md | 10 + context/opencensus-context/CHANGELOG.md | 5 + context/opencensus-context/version.py | 2 +- contrib/opencensus-ext-azure/CHANGELOG.md | 8 + contrib/opencensus-ext-azure/README.rst | 26 +- .../examples/logs/correlated.py | 2 + .../examples/logs/error.py | 1 + .../examples/logs/event.py | 27 + .../examples/logs/properties.py | 2 + .../examples/logs/simple.py | 2 + .../opencensus/ext/azure/common/version.py | 2 +- .../ext/azure/log_exporter/__init__.py | 150 +- .../standard_metrics/__init__.py | 6 +- .../standard_metrics/http_dependency.py | 92 - .../tests/test_azure_log_exporter.py | 195 +- .../tests/test_azure_standard_metrics.py | 363 +-- .../opencensus-ext-stackdriver/CHANGELOG.md | 6 + .../stackdriver/stats_exporter/__init__.py | 1 + contrib/opencensus-ext-stackdriver/setup.py | 1 + .../tests/test_stackdriver_exporter.py | 1739 +++++----- .../tests/test_stackdriver_stats.py | 2796 +++++++++-------- contrib/opencensus-ext-stackdriver/version.py | 2 +- opencensus/common/version/__init__.py | 2 +- opencensus/trace/span.py | 2 +- 24 files changed, 2693 insertions(+), 2749 deletions(-) create mode 100644 contrib/opencensus-ext-azure/examples/logs/event.py delete mode 100644 contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/http_dependency.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d6077a3b..d791816b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Unreleased +## 0.7.10 +Released 2020-06-29 + +- Updated `azure` module + ([#903](https://github.com/census-instrumentation/opencensus-python/pull/903), + [#925](https://github.com/census-instrumentation/opencensus-python/pull/925)) + +- Updated `stackdriver` module + ([#919](https://github.com/census-instrumentation/opencensus-python/pull/919)) + ## 0.7.9 Released 2020-06-17 diff --git a/context/opencensus-context/CHANGELOG.md b/context/opencensus-context/CHANGELOG.md index a470e2320..55518cfea 100644 --- a/context/opencensus-context/CHANGELOG.md +++ b/context/opencensus-context/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +## 0.1.2 +Released 2020-06-29 + +- Release source distribution + ## 0.1.1 Released 2019-05-31 diff --git a/context/opencensus-context/version.py b/context/opencensus-context/version.py index 00ffd3fd7..3f601a176 100644 --- a/context/opencensus-context/version.py +++ b/context/opencensus-context/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.1.1' +__version__ = '0.1.2' diff --git a/contrib/opencensus-ext-azure/CHANGELOG.md b/contrib/opencensus-ext-azure/CHANGELOG.md index 7efa8b779..08efb2171 100644 --- a/contrib/opencensus-ext-azure/CHANGELOG.md +++ b/contrib/opencensus-ext-azure/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +## 1.0.4 +Released 2020-06-29 + +- Remove dependency rate from standard metrics + ([#903](https://github.com/census-instrumentation/opencensus-python/pull/903)) +- Implement customEvents using AzureEventHandler + ([#925](https://github.com/census-instrumentation/opencensus-python/pull/925)) + ## 1.0.3 Released 2020-06-17 diff --git a/contrib/opencensus-ext-azure/README.rst b/contrib/opencensus-ext-azure/README.rst index 49ba625f3..8cb7b64e7 100644 --- a/contrib/opencensus-ext-azure/README.rst +++ b/contrib/opencensus-ext-azure/README.rst @@ -121,6 +121,21 @@ Modifying Logs logger.addHandler(handler) logger.warning('Hello, World!') +Events +###### + +You can send `customEvent` telemetry in exactly the same way you would send `trace` telemetry except using the `AzureEventHandler` instead. + +.. code:: python + + import logging + + from opencensus.ext.azure.log_exporter import AzureEventHandler + + logger = logging.getLogger(__name__) + logger.addHandler(AzureEventHandler(connection_string='InstrumentationKey=')) + logger.setLevel(logging.INFO) + logger.info('Hello, World!') Metrics ~~~~~~~ @@ -178,10 +193,10 @@ The **Azure Monitor Metrics Exporter** allows you to export metrics to `Azure Mo if __name__ == "__main__": main() -Standard Metrics -################ +Performance counters +#################### -The exporter also includes a set of standard metrics that are exported to Azure Monitor by default. +The exporter also includes a set of performance counters that are exported to Azure Monitor by default. .. code:: python @@ -191,7 +206,7 @@ The exporter also includes a set of standard metrics that are exported to Azure from opencensus.ext.azure import metrics_exporter def main(): - # All you need is the next line. You can disable standard metrics by + # All you need is the next line. You can disable performance counters by # passing in enable_standard_metrics=False into the constructor of # new_metrics_exporter() _exporter = metrics_exporter.new_metrics_exporter(connection_string='InstrumentationKey=') @@ -205,13 +220,12 @@ The exporter also includes a set of standard metrics that are exported to Azure if __name__ == "__main__": main() -Below is a list of standard metrics that are currently available: +Below is a list of performance counters that are currently available: - Available Memory (bytes) - CPU Processor Time (percentage) - Incoming Request Rate (per second) - Incoming Request Average Execution Time (milliseconds) -- Outgoing Request Rate (per second) - Process CPU Usage (percentage) - Process Private Bytes (bytes) diff --git a/contrib/opencensus-ext-azure/examples/logs/correlated.py b/contrib/opencensus-ext-azure/examples/logs/correlated.py index 69445e997..9854b9a91 100644 --- a/contrib/opencensus-ext-azure/examples/logs/correlated.py +++ b/contrib/opencensus-ext-azure/examples/logs/correlated.py @@ -36,3 +36,5 @@ with tracer.span(name='test'): logger.warning('In the span') logger.warning('After the span') + +input("...") diff --git a/contrib/opencensus-ext-azure/examples/logs/error.py b/contrib/opencensus-ext-azure/examples/logs/error.py index 772861cf1..f1349226d 100644 --- a/contrib/opencensus-ext-azure/examples/logs/error.py +++ b/contrib/opencensus-ext-azure/examples/logs/error.py @@ -32,3 +32,4 @@ def main(): if __name__ == '__main__': main() + input("...") diff --git a/contrib/opencensus-ext-azure/examples/logs/event.py b/contrib/opencensus-ext-azure/examples/logs/event.py new file mode 100644 index 000000000..d62f82468 --- /dev/null +++ b/contrib/opencensus-ext-azure/examples/logs/event.py @@ -0,0 +1,27 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from opencensus.ext.azure.log_exporter import AzureEventHandler + +logger = logging.getLogger(__name__) +# TODO: you need to specify the instrumentation key in a connection string +# and place it in the APPLICATIONINSIGHTS_CONNECTION_STRING +# environment variable. +logger.addHandler(AzureEventHandler()) +logger.setLevel(logging.INFO) +logger.info('Hello, World!') + +input("...") diff --git a/contrib/opencensus-ext-azure/examples/logs/properties.py b/contrib/opencensus-ext-azure/examples/logs/properties.py index 5cfdd3568..7c2e1f9bc 100644 --- a/contrib/opencensus-ext-azure/examples/logs/properties.py +++ b/contrib/opencensus-ext-azure/examples/logs/properties.py @@ -32,3 +32,5 @@ result = 1 / 0 # generate a ZeroDivisionError except Exception: logger.exception('Captured an exception.', extra=properties) + +input("...") \ No newline at end of file diff --git a/contrib/opencensus-ext-azure/examples/logs/simple.py b/contrib/opencensus-ext-azure/examples/logs/simple.py index 1fba3d668..d1e94e1dd 100644 --- a/contrib/opencensus-ext-azure/examples/logs/simple.py +++ b/contrib/opencensus-ext-azure/examples/logs/simple.py @@ -22,3 +22,5 @@ # environment variable. logger.addHandler(AzureLogHandler()) logger.warning('Hello, World!') + +input("...") \ No newline at end of file diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py index 8ef965cab..6281c81cb 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '1.0.3' +__version__ = '1.0.4' diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py index cc26271ef..a83dec8c7 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py @@ -24,6 +24,7 @@ from opencensus.ext.azure.common.protocol import ( Data, Envelope, + Event, ExceptionData, Message, ) @@ -33,17 +34,51 @@ logger = logging.getLogger(__name__) -__all__ = ['AzureLogHandler'] +__all__ = ['AzureEventHandler', 'AzureLogHandler'] class BaseLogHandler(logging.Handler): - def __init__(self): + + def __init__(self, **options): super(BaseLogHandler, self).__init__() + self.options = Options(**options) + utils.validate_instrumentation_key(self.options.instrumentation_key) + if not 0 <= self.options.logging_sampling_rate <= 1: + raise ValueError('Sampling must be in the range: [0,1]') + self.export_interval = self.options.export_interval + self.max_batch_size = self.options.max_batch_size + self.storage = LocalFileStorage( + path=self.options.storage_path, + max_size=self.options.storage_max_size, + maintenance_period=self.options.storage_maintenance_period, + retention_period=self.options.storage_retention_period, + ) + self._telemetry_processors = [] + self.addFilter(SamplingFilter(self.options.logging_sampling_rate)) self._queue = Queue(capacity=8192) # TODO: make this configurable self._worker = Worker(self._queue, self) self._worker.start() + def _export(self, batch, event=None): # pragma: NO COVER + try: + if batch: + envelopes = [self.log_record_to_envelope(x) for x in batch] + envelopes = self.apply_telemetry_processors(envelopes) + result = self._transmit(envelopes) + if result > 0: + self.storage.put(envelopes, result) + if event: + if isinstance(event, QueueExitEvent): + self._transmit_from_storage() # send files before exit + return + if len(batch) < self.options.max_batch_size: + self._transmit_from_storage() + finally: + if event: + event.set() + def close(self): + self.storage.close() self._worker.stop() def createLock(self): @@ -52,14 +87,7 @@ def createLock(self): def emit(self, record): self._queue.put(record, block=False) - def _export(self, batch, event=None): - try: - return self.export(batch) - finally: - if event: - event.set() - - def export(self, batch): + def log_record_to_envelope(self, record): raise NotImplementedError # pragma: NO COVER def flush(self, timeout=None): @@ -79,7 +107,7 @@ def __init__(self, src, dst): def run(self): # Indicate that this thread is an exporter thread. - # Used to suppress tracking of requests in this thread. + # Used to suppress tracking of requests in this thread execution_context.set_is_exporter(True) src = self._src dst = self._dst @@ -121,66 +149,11 @@ def filter(self, record): class AzureLogHandler(TransportMixin, ProcessorMixin, BaseLogHandler): - """Handler for logging to Microsoft Azure Monitor. - - :param options: Options for the log handler. - """ - - def __init__(self, **options): - self.options = Options(**options) - utils.validate_instrumentation_key(self.options.instrumentation_key) - if not 0 <= self.options.logging_sampling_rate <= 1: - raise ValueError('Sampling must be in the range: [0,1]') - self.export_interval = self.options.export_interval - self.max_batch_size = self.options.max_batch_size - self.storage = LocalFileStorage( - path=self.options.storage_path, - max_size=self.options.storage_max_size, - maintenance_period=self.options.storage_maintenance_period, - retention_period=self.options.storage_retention_period, - ) - self._telemetry_processors = [] - super(AzureLogHandler, self).__init__() - self.addFilter(SamplingFilter(self.options.logging_sampling_rate)) - - def close(self): - self.storage.close() - super(AzureLogHandler, self).close() - - def _export(self, batch, event=None): # pragma: NO COVER - try: - if batch: - envelopes = [self.log_record_to_envelope(x) for x in batch] - envelopes = self.apply_telemetry_processors(envelopes) - result = self._transmit(envelopes) - if result > 0: - self.storage.put(envelopes, result) - if event: - if isinstance(event, QueueExitEvent): - self._transmit_from_storage() # send files before exit - return - if len(batch) < self.options.max_batch_size: - self._transmit_from_storage() - finally: - if event: - event.set() + """Handler for logging to Microsoft Azure Monitor.""" def log_record_to_envelope(self, record): - envelope = Envelope( - iKey=self.options.instrumentation_key, - tags=dict(utils.azure_monitor_context), - time=utils.timestamp_to_iso_str(record.created), - ) + envelope = create_envelope(self.options.instrumentation_key, record) - envelope.tags['ai.operation.id'] = getattr( - record, - 'traceId', - '00000000000000000000000000000000', - ) - envelope.tags['ai.operation.parentId'] = '|{}.{}.'.format( - envelope.tags['ai.operation.id'], - getattr(record, 'spanId', '0000000000000000'), - ) properties = { 'process': record.processName, 'module': record.module, @@ -188,7 +161,6 @@ def log_record_to_envelope(self, record): 'lineNumber': record.lineno, 'level': record.levelname, } - if (hasattr(record, 'custom_dimensions') and isinstance(record.custom_dimensions, dict)): properties.update(record.custom_dimensions) @@ -230,3 +202,43 @@ def log_record_to_envelope(self, record): ) envelope.data = Data(baseData=data, baseType='MessageData') return envelope + + +class AzureEventHandler(TransportMixin, ProcessorMixin, BaseLogHandler): + """Handler for sending custom events to Microsoft Azure Monitor.""" + + def log_record_to_envelope(self, record): + envelope = create_envelope(self.options.instrumentation_key, record) + + properties = {} + if (hasattr(record, 'custom_dimensions') and + isinstance(record.custom_dimensions, dict)): + properties.update(record.custom_dimensions) + + envelope.name = 'Microsoft.ApplicationInsights.Event' + data = Event( + name=self.format(record), + properties=properties, + ) + envelope.data = Data(baseData=data, baseType='EventData') + + return envelope + + +def create_envelope(instrumentation_key, record): + envelope = Envelope( + iKey=instrumentation_key, + tags=dict(utils.azure_monitor_context), + time=utils.timestamp_to_iso_str(record.created), + ) + envelope.tags['ai.operation.id'] = getattr( + record, + 'traceId', + '00000000000000000000000000000000', + ) + envelope.tags['ai.operation.parentId'] = '|{}.{}.'.format( + envelope.tags['ai.operation.id'], + getattr(record, 'spanId', '0000000000000000'), + ) + + return envelope diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/__init__.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/__init__.py index cf4104ad3..3ad5109ca 100644 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/__init__.py +++ b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/__init__.py @@ -15,10 +15,7 @@ from opencensus.ext.azure.metrics_exporter.standard_metrics.cpu import ( ProcessorTimeMetric, ) -from opencensus.ext.azure.metrics_exporter.standard_metrics.http_dependency import ( - DependencyRateMetric, -) -from opencensus.ext.azure.metrics_exporter.standard_metrics.http_requests import ( +from opencensus.ext.azure.metrics_exporter.standard_metrics.http_requests import ( # noqa E501 RequestsAvgExecutionMetric, RequestsRateMetric, ) @@ -34,7 +31,6 @@ # List of standard metrics to track STANDARD_METRICS = [AvailableMemoryMetric, - DependencyRateMetric, ProcessCPUMetric, ProcessMemoryMetric, ProcessorTimeMetric, diff --git a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/http_dependency.py b/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/http_dependency.py deleted file mode 100644 index 0632ba2d2..000000000 --- a/contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics/http_dependency.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2019, OpenCensus Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import threading -import time - -import requests - -from opencensus.metrics.export.gauge import DerivedDoubleGauge -from opencensus.trace import execution_context - -dependency_map = dict() -_dependency_lock = threading.Lock() -ORIGINAL_REQUEST = requests.Session.request - - -def dependency_patch(*args, **kwargs): - result = ORIGINAL_REQUEST(*args, **kwargs) - # Only collect request metric if sent from non-exporter thread - if not execution_context.is_exporter(): - # We don't want multiple threads updating this at once - with _dependency_lock: - count = dependency_map.get('count', 0) - dependency_map['count'] = count + 1 - return result - - -def setup(): - # Patch the requests library functions to track dependency information - requests.Session.request = dependency_patch - - -class DependencyRateMetric(object): - # Dependency call metrics can be found under custom metrics - NAME = "\\ApplicationInsights\\Dependency Calls/Sec" - - def __init__(self): - setup() - - @staticmethod - def get_value(): - current_count = dependency_map.get('count', 0) - current_time = time.time() - last_count = dependency_map.get('last_count', 0) - last_time = dependency_map.get('last_time') - last_result = dependency_map.get('last_result', 0) - - try: - # last_time is None the very first time this function is called - if last_time is not None: - elapsed_seconds = current_time - last_time - interval_count = current_count - last_count - result = interval_count / elapsed_seconds - else: - result = 0 - dependency_map['last_time'] = current_time - dependency_map['last_count'] = current_count - dependency_map['last_result'] = result - return result - except ZeroDivisionError: - # If elapsed_seconds is 0, exporter call made too close to previous - # Return the previous result if this is the case - return last_result - - def __call__(self): - """ Returns a derived gauge for outgoing requests per second - - Calculated by obtaining by getting the number of outgoing requests made - using the requests library within an elapsed time and dividing that - value over the elapsed time. - - :rtype: :class:`opencensus.metrics.export.gauge.DerivedLongGauge` - :return: The gauge representing the outgoing requests metric - """ - gauge = DerivedDoubleGauge( - DependencyRateMetric.NAME, - 'Outgoing Requests per second', - 'rps', - []) - gauge.create_default_time_series(DependencyRateMetric.get_value) - return gauge diff --git a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py index 88b8c436b..5db155e54 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_log_exporter.py @@ -43,7 +43,9 @@ def __init__(self, max_batch_size, callback): self.export_interval = 1 self.max_batch_size = max_batch_size self.callback = callback - super(CustomLogHandler, self).__init__() + super(CustomLogHandler, self).__init__( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + ) def export(self, batch): return self.callback(batch) @@ -259,3 +261,194 @@ def test_log_record_not_sampled(self, requests_mock): logger.warning('Hello_World4') handler.close() self.assertFalse(requests_mock.called) + + +class TestAzureEventHandler(unittest.TestCase): + def test_ctor(self): + from opencensus.ext.azure.common import Options + instrumentation_key = Options._default.instrumentation_key + Options._default.instrumentation_key = None + self.assertRaises(ValueError, lambda: log_exporter.AzureEventHandler()) + Options._default.instrumentation_key = instrumentation_key + + def test_invalid_sampling_rate(self): + with self.assertRaises(ValueError): + log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + logging_sampling_rate=4.0, + ) + + def test_init_handler_with_proxies(self): + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + proxies='{"https":"https://test-proxy.com"}', + ) + + self.assertEqual( + handler.options.proxies, + '{"https":"https://test-proxy.com"}', + ) + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_exception(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + logger.addHandler(handler) + try: + return 1 / 0 # generate a ZeroDivisionError + except Exception: + logger.exception('Captured an exception.') + handler.close() + self.assertEqual(len(requests_mock.call_args_list), 1) + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('ZeroDivisionError' in post_body) + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_exception_with_custom_properties(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + logger.addHandler(handler) + try: + return 1 / 0 # generate a ZeroDivisionError + except Exception: + properties = { + 'custom_dimensions': + { + 'key_1': 'value_1', + 'key_2': 'value_2' + } + } + logger.exception('Captured an exception.', extra=properties) + handler.close() + self.assertEqual(len(requests_mock.call_args_list), 1) + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('ZeroDivisionError' in post_body) + self.assertTrue('key_1' in post_body) + self.assertTrue('key_2' in post_body) + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_export_empty(self, request_mock): + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + handler._export([]) + self.assertEqual(len(os.listdir(handler.storage.path)), 0) + handler.close() + + @mock.patch('opencensus.ext.azure.log_exporter' + '.AzureEventHandler.log_record_to_envelope') + def test_export_failure(self, log_record_to_envelope_mock): + log_record_to_envelope_mock.return_value = ['bar'] + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + with mock.patch('opencensus.ext.azure.log_exporter' + '.AzureEventHandler._transmit') as transmit: + transmit.return_value = 10 + handler._export(['foo']) + self.assertEqual(len(os.listdir(handler.storage.path)), 1) + self.assertIsNone(handler.storage.get()) + handler.close() + + def test_log_record_to_envelope(self): + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + envelope = handler.log_record_to_envelope(mock.MagicMock( + exc_info=None, + levelno=10, + )) + self.assertEqual( + envelope.iKey, + '12345678-1234-5678-abcd-12345678abcd') + handler.close() + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_log_record_with_custom_properties(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + logger.addHandler(handler) + logger.warning('action', extra={ + 'custom_dimensions': + { + 'key_1': 'value_1', + 'key_2': 'value_2' + } + }) + handler.close() + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('action' in post_body) + self.assertTrue('key_1' in post_body) + self.assertTrue('key_2' in post_body) + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_log_with_invalid_custom_properties(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + storage_path=os.path.join(TEST_FOLDER, self.id()), + ) + logger.addHandler(handler) + logger.warning('action_1_%s', None) + logger.warning('action_2_%s', 'arg', extra={ + 'custom_dimensions': 'not_a_dict' + }) + logger.warning('action_3_%s', 'arg', extra={ + 'notcustom_dimensions': {'key_1': 'value_1'} + }) + + handler.close() + self.assertEqual(len(os.listdir(handler.storage.path)), 0) + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('action_1_' in post_body) + self.assertTrue('action_2_arg' in post_body) + self.assertTrue('action_3_arg' in post_body) + + self.assertFalse('not_a_dict' in post_body) + self.assertFalse('key_1' in post_body) + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_log_record_sampled(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + logging_sampling_rate=1.0, + ) + logger.addHandler(handler) + logger.warning('Hello_World') + logger.warning('Hello_World2') + logger.warning('Hello_World3') + logger.warning('Hello_World4') + handler.close() + post_body = requests_mock.call_args_list[0][1]['data'] + self.assertTrue('Hello_World' in post_body) + self.assertTrue('Hello_World2' in post_body) + self.assertTrue('Hello_World3' in post_body) + self.assertTrue('Hello_World4' in post_body) + + @mock.patch('requests.post', return_value=mock.Mock()) + def test_log_record_not_sampled(self, requests_mock): + logger = logging.getLogger(self.id()) + handler = log_exporter.AzureEventHandler( + instrumentation_key='12345678-1234-5678-abcd-12345678abcd', + logging_sampling_rate=0.0, + ) + logger.addHandler(handler) + logger.warning('Hello_World') + logger.warning('Hello_World2') + logger.warning('Hello_World3') + logger.warning('Hello_World4') + handler.close() + self.assertFalse(requests_mock.called) diff --git a/contrib/opencensus-ext-azure/tests/test_azure_standard_metrics.py b/contrib/opencensus-ext-azure/tests/test_azure_standard_metrics.py index 5cbb6e601..3ad5109ca 100644 --- a/contrib/opencensus-ext-azure/tests/test_azure_standard_metrics.py +++ b/contrib/opencensus-ext-azure/tests/test_azure_standard_metrics.py @@ -12,318 +12,51 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections -import sys -import unittest - -import mock -import requests - -from opencensus.ext.azure.metrics_exporter import standard_metrics -from opencensus.trace import execution_context - -if sys.version_info < (3,): - from BaseHTTPServer import HTTPServer -else: - from http.server import HTTPServer - -ORIGINAL_FUNCTION = requests.Session.request -ORIGINAL_CONS = HTTPServer.__init__ - - -class TestStandardMetrics(unittest.TestCase): - def setUp(self): - standard_metrics.http_dependency.dependency_map.clear() - standard_metrics.http_requests.requests_map.clear() - requests.Session.request = ORIGINAL_FUNCTION - standard_metrics.http_dependency.ORIGINAL_REQUEST = ORIGINAL_FUNCTION - standard_metrics.http_requests.ORIGINAL_CONSTRUCTOR = ORIGINAL_CONS - - @mock.patch('opencensus.ext.azure.metrics_exporter' - '.standard_metrics.register_metrics') - def test_producer_ctor(self, avail_mock): - standard_metrics.AzureStandardMetricsProducer() - - self.assertEqual(len(avail_mock.call_args_list), 1) - - def test_producer_get_metrics(self): - producer = standard_metrics.AzureStandardMetricsProducer() - metrics = producer.get_metrics() - - self.assertEqual(len(metrics), 7) - - def test_register_metrics(self): - registry = standard_metrics.register_metrics() - - self.assertEqual(len(registry.get_metrics()), 7) - - def test_get_available_memory_metric(self): - metric = standard_metrics.AvailableMemoryMetric() - gauge = metric() - - self.assertEqual(gauge.descriptor.name, '\\Memory\\Available Bytes') - - @mock.patch('psutil.virtual_memory') - def test_get_available_memory(self, psutil_mock): - memory = collections.namedtuple('memory', 'available') - vmem = memory(available=100) - psutil_mock.return_value = vmem - mem = standard_metrics.AvailableMemoryMetric.get_value() - - self.assertEqual(mem, 100) - - def test_get_process_private_bytes_metric(self): - metric = standard_metrics.ProcessMemoryMetric() - gauge = metric() - - # TODO: Refactor names to be platform generic - self.assertEqual(gauge.descriptor.name, - '\\Process(??APP_WIN32_PROC??)\\Private Bytes') - - def test_get_process_private_bytes(self): - with mock.patch('opencensus.ext.azure.metrics_exporter' + - '.standard_metrics.process.PROCESS') as process_mock: - memory = collections.namedtuple('memory', 'rss') - pmem = memory(rss=100) - process_mock.memory_info.return_value = pmem - mem = standard_metrics.ProcessMemoryMetric.get_value() - - self.assertEqual(mem, 100) - - @mock.patch('opencensus.ext.azure.metrics_exporter' - '.standard_metrics.process.logger') - def test_get_process_private_bytes_exception(self, logger_mock): - with mock.patch('opencensus.ext.azure.metrics_exporter' + - '.standard_metrics.process.PROCESS') as process_mock: - process_mock.memory_info.side_effect = Exception() - standard_metrics.ProcessMemoryMetric.get_value() - - logger_mock.exception.assert_called() - - def test_get_processor_time_metric(self): - metric = standard_metrics.ProcessorTimeMetric() - gauge = metric() - - self.assertEqual(gauge.descriptor.name, - '\\Processor(_Total)\\% Processor Time') - - def test_get_processor_time(self): - with mock.patch('psutil.cpu_times_percent') as processor_mock: - cpu = collections.namedtuple('cpu', 'idle') - cpu_times = cpu(idle=94.5) - processor_mock.return_value = cpu_times - processor_time = standard_metrics.ProcessorTimeMetric.get_value() - - self.assertEqual(processor_time, 5.5) - - def test_get_process_cpu_usage_metric(self): - metric = standard_metrics.ProcessCPUMetric() - gauge = metric() - - self.assertEqual(gauge.descriptor.name, - '\\Process(??APP_WIN32_PROC??)\\% Processor Time') - - @mock.patch('opencensus.ext.azure.metrics_exporter' - '.standard_metrics.process.psutil') - def test_get_process_cpu_usage(self, psutil_mock): - with mock.patch('opencensus.ext.azure.metrics_exporter' - '.standard_metrics.process.PROCESS') as process_mock: - process_mock.cpu_percent.return_value = 44.4 - psutil_mock.cpu_count.return_value = 2 - cpu_usage = standard_metrics.ProcessCPUMetric.get_value() - - self.assertEqual(cpu_usage, 22.2) - - @mock.patch('opencensus.ext.azure.metrics_exporter' - '.standard_metrics.process.logger') - def test_get_process_cpu_usage_exception(self, logger_mock): - with mock.patch('opencensus.ext.azure.metrics_exporter' - '.standard_metrics.process.psutil') as psutil_mock: - psutil_mock.cpu_count.return_value = None - standard_metrics.ProcessCPUMetric.get_value() - - logger_mock.exception.assert_called() - - def test_dependency_patch(self): - map = standard_metrics.http_dependency.dependency_map - standard_metrics.http_dependency.ORIGINAL_REQUEST = lambda x: None - session = requests.Session() - execution_context.set_is_exporter(False) - result = standard_metrics.http_dependency.dependency_patch(session) - - self.assertEqual(map['count'], 1) - self.assertIsNone(result) - - def test_dependency_patch_exporter_thread(self): - map = standard_metrics.http_dependency.dependency_map - standard_metrics.http_dependency.ORIGINAL_REQUEST = lambda x: None - session = mock.Mock() - execution_context.set_is_exporter(True) - result = standard_metrics.http_dependency.dependency_patch(session) - - self.assertIsNone(map.get('count')) - self.assertIsNone(result) - - def test_get_dependency_rate_metric(self): - metric = standard_metrics.DependencyRateMetric() - gauge = metric() - - self.assertEqual(gauge.descriptor.name, - '\\ApplicationInsights\\Dependency Calls/Sec') - - def test_get_dependency_rate_first_time(self): - rate = standard_metrics.DependencyRateMetric.get_value() - - self.assertEqual(rate, 0) - - @mock.patch('opencensus.ext.azure.metrics_exporter' - '.standard_metrics.http_dependency.time') - def test_get_dependency_rate(self, time_mock): - time_mock.time.return_value = 100 - standard_metrics.http_dependency.dependency_map['last_time'] = 98 - standard_metrics.http_dependency.dependency_map['count'] = 4 - rate = standard_metrics.DependencyRateMetric.get_value() - - self.assertEqual(rate, 2) - - @mock.patch('opencensus.ext.azure.metrics_exporter' - '.standard_metrics.http_dependency.time') - def test_get_dependency_rate_error(self, time_mock): - time_mock.time.return_value = 100 - standard_metrics.http_dependency.dependency_map['last_result'] = 5 - standard_metrics.http_dependency.dependency_map['last_time'] = 100 - result = standard_metrics.DependencyRateMetric.get_value() - - self.assertEqual(result, 5) - - def test_request_patch(self): - map = standard_metrics.http_requests.requests_map - func = mock.Mock() - new_func = standard_metrics.http_requests.request_patch(func) - new_func() - - self.assertEqual(map['count'], 1) - self.assertIsNotNone(map['duration']) - self.assertEqual(len(func.call_args_list), 1) - - def test_server_patch(self): - standard_metrics. \ - http_requests. \ - ORIGINAL_CONSTRUCTOR = lambda x, y, z: None - with mock.patch('opencensus.ext.azure.metrics_exporter' - '.standard_metrics.http_requests' - '.request_patch') as request_mock: - handler = mock.Mock() - handler.do_DELETE.return_value = None - handler.do_GET.return_value = None - handler.do_HEAD.return_value = None - handler.do_OPTIONS.return_value = None - handler.do_POST.return_value = None - handler.do_PUT.return_value = None - result = standard_metrics. \ - http_requests. \ - server_patch(None, None, handler) - handler.do_DELETE() - handler.do_GET() - handler.do_HEAD() - handler.do_OPTIONS() - handler.do_POST() - handler.do_PUT() - - self.assertEqual(result, None) - self.assertEqual(len(request_mock.call_args_list), 6) - - def test_server_patch_no_methods(self): - standard_metrics. \ - http_requests. \ - ORIGINAL_CONSTRUCTOR = lambda x, y, z: None - with mock.patch('opencensus.ext.azure.metrics_exporter' - '.standard_metrics.http_requests' - '.request_patch') as request_mock: - handler = mock.Mock() - result = standard_metrics. \ - http_requests. \ - server_patch(None, None, handler) - handler.do_DELETE() - handler.do_GET() - handler.do_HEAD() - handler.do_OPTIONS() - handler.do_POST() - handler.do_PUT() - - self.assertEqual(result, None) - self.assertEqual(len(request_mock.call_args_list), 0) - - def test_server_patch_no_args(self): - standard_metrics \ - .http_requests \ - .ORIGINAL_CONSTRUCTOR = lambda x, y: None - r = standard_metrics.http_requests.server_patch(None, None) - - self.assertEqual(r, None) - - def test_server_patch_no_handler(self): - standard_metrics \ - .http_requests \ - .ORIGINAL_CONSTRUCTOR = lambda x, y, z: None - r = standard_metrics.http_requests.server_patch(None, None, None) - - self.assertEqual(r, None) - - def test_get_requests_rate_metric(self): - metric = standard_metrics.RequestsRateMetric() - gauge = metric() - - name = '\\ASP.NET Applications(??APP_W3SVC_PROC??)\\Requests/Sec' - self.assertEqual(gauge.descriptor.name, name) - - def test_get_requests_rate_first_time(self): - rate = standard_metrics.http_requests.get_requests_rate() - - self.assertEqual(rate, 0) - - @mock.patch('opencensus.ext.azure.metrics_exporter' - '.standard_metrics.http_requests.time') - def test_get_requests_rate(self, time_mock): - time_mock.time.return_value = 100 - standard_metrics.http_requests.requests_map['last_time'] = 98 - standard_metrics.http_requests.requests_map['count'] = 4 - rate = standard_metrics.http_requests.get_requests_rate() - - self.assertEqual(rate, 2) - - @mock.patch('opencensus.ext.azure.metrics_exporter' - '.standard_metrics.http_requests.time') - def test_get_requests_rate_error(self, time_mock): - time_mock.time.return_value = 100 - standard_metrics.http_requests.requests_map['last_rate'] = 5 - standard_metrics.http_requests.requests_map['last_time'] = 100 - result = standard_metrics.http_requests.get_requests_rate() - - self.assertEqual(result, 5) - - def test_get_requests_execution_metric(self): - metric = standard_metrics.RequestsAvgExecutionMetric() - gauge = metric() - - name = '\\ASP.NET Applications(??APP_W3SVC_PROC??)' \ - '\\Request Execution Time' - self.assertEqual(gauge.descriptor.name, name) - - def test_get_requests_execution(self): - map = standard_metrics.http_requests.requests_map - map['duration'] = 0.1 - map['count'] = 10 - map['last_count'] = 5 - result = standard_metrics.http_requests.get_average_execution_time() - - self.assertEqual(result, 20) - - def test_get_requests_execution_error(self): - map = standard_metrics.http_requests.requests_map - map['duration'] = 0.1 - map['count'] = 10 - map['last_count'] = 10 - result = standard_metrics.http_requests.get_average_execution_time() - - self.assertEqual(result, 0) +from opencensus.ext.azure.metrics_exporter.standard_metrics.cpu import ( + ProcessorTimeMetric, +) +from opencensus.ext.azure.metrics_exporter.standard_metrics.http_requests import ( # noqa E501 + RequestsAvgExecutionMetric, + RequestsRateMetric, +) +from opencensus.ext.azure.metrics_exporter.standard_metrics.memory import ( + AvailableMemoryMetric, +) +from opencensus.ext.azure.metrics_exporter.standard_metrics.process import ( + ProcessCPUMetric, + ProcessMemoryMetric, +) +from opencensus.metrics.export.gauge import Registry +from opencensus.metrics.export.metric_producer import MetricProducer + +# List of standard metrics to track +STANDARD_METRICS = [AvailableMemoryMetric, + ProcessCPUMetric, + ProcessMemoryMetric, + ProcessorTimeMetric, + RequestsAvgExecutionMetric, + RequestsRateMetric] + + +def register_metrics(): + registry = Registry() + for standard_metric in STANDARD_METRICS: + metric = standard_metric() + registry.add_gauge(metric()) + return registry + + +class AzureStandardMetricsProducer(MetricProducer): + """Implementation of the producer of standard metrics. + + Includes Azure specific standard metrics, implemented + using gauges. + """ + def __init__(self): + self.registry = register_metrics() + + def get_metrics(self): + return self.registry.get_metrics() + + +producer = AzureStandardMetricsProducer() diff --git a/contrib/opencensus-ext-stackdriver/CHANGELOG.md b/contrib/opencensus-ext-stackdriver/CHANGELOG.md index a3b8b4c7b..8e79c9f15 100644 --- a/contrib/opencensus-ext-stackdriver/CHANGELOG.md +++ b/contrib/opencensus-ext-stackdriver/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +## 0.7.3 +Released 2020-06-29 + + - Add mean property for distribution values + ([#919](https://github.com/census-instrumentation/opencensus-python/pull/919)) + ## 0.7.2 Released 2019-08-26 diff --git a/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/stats_exporter/__init__.py b/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/stats_exporter/__init__.py index 9249294ff..b47d44b81 100644 --- a/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/stats_exporter/__init__.py +++ b/contrib/opencensus-ext-stackdriver/opencensus/ext/stackdriver/stats_exporter/__init__.py @@ -201,6 +201,7 @@ def _convert_point(self, metric, ts, point, sd_point): sd_dist_val.count = point.value.count sd_dist_val.sum_of_squared_deviation =\ point.value.sum_of_squared_deviation + sd_dist_val.mean = point.value.sum / sd_dist_val.count assert sd_dist_val.bucket_options.explicit_buckets.bounds == [] sd_dist_val.bucket_options.explicit_buckets.bounds.extend( diff --git a/contrib/opencensus-ext-stackdriver/setup.py b/contrib/opencensus-ext-stackdriver/setup.py index 86cf5fc04..22c1780e9 100644 --- a/contrib/opencensus-ext-stackdriver/setup.py +++ b/contrib/opencensus-ext-stackdriver/setup.py @@ -41,6 +41,7 @@ install_requires=[ 'google-cloud-monitoring >= 0.30.0, < 1.0.0', 'google-cloud-trace >= 0.20.0, < 1.0.0', + 'rsa <= 4.0; python_version<="3.4"', 'opencensus >= 0.7.1, < 1.0.0', ], extras_require={}, diff --git a/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_exporter.py b/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_exporter.py index 16169ea55..c48e21681 100644 --- a/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_exporter.py +++ b/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_exporter.py @@ -1,870 +1,869 @@ -# noqa: E501 -# # Copyright 2017, OpenCensus Authors -# # -# # Licensed under the Apache License, Version 2.0 (the "License"); -# # you may not use this file except in compliance with the License. -# # You may obtain a copy of the License at -# # -# # http://www.apache.org/licenses/LICENSE-2.0 -# # -# # Unless required by applicable law or agreed to in writing, software -# # distributed under the License is distributed on an "AS IS" BASIS, -# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# # See the License for the specific language governing permissions and -# # limitations under the License. - -# import unittest - -# import mock - -# from opencensus.common.version import __version__ -# from opencensus.ext.stackdriver import trace_exporter -# from opencensus.trace import span_context -# from opencensus.trace import span_data as span_data_module - - -# class _Client(object): -# def __init__(self, project=None): -# if project is None: -# project = 'PROJECT' - -# self.project = project - - -# class TestStackdriverExporter(unittest.TestCase): -# def test_constructor_default(self): -# patch = mock.patch( -# 'opencensus.ext.stackdriver.trace_exporter.Client', -# new=_Client) - -# with patch: -# exporter = trace_exporter.StackdriverExporter() - -# project_id = 'PROJECT' -# self.assertEqual(exporter.project_id, project_id) - -# def test_constructor_explicit(self): -# client = mock.Mock() -# project_id = 'PROJECT' -# client.project = project_id -# transport = mock.Mock() - -# exporter = trace_exporter.StackdriverExporter( -# client=client, project_id=project_id, transport=transport) - -# self.assertIs(exporter.client, client) -# self.assertEqual(exporter.project_id, project_id) - -# def test_export(self): -# client = mock.Mock() -# project_id = 'PROJECT' -# client.project = project_id -# exporter = trace_exporter.StackdriverExporter( -# client=client, project_id=project_id, transport=MockTransport) -# exporter.export({}) - -# self.assertTrue(exporter.transport.export_called) - -# @mock.patch('opencensus.ext.stackdriver.trace_exporter.' -# 'monitored_resource.get_instance', -# return_value=None) -# def test_emit(self, mr_mock): -# trace_id = '6e0c63257de34c92bf9efcd03927272e' -# span_datas = [ -# span_data_module.SpanData( -# name='span', -# context=span_context.SpanContext(trace_id=trace_id), -# span_id='1111', -# parent_span_id=None, -# attributes=None, -# start_time=None, -# end_time=None, -# child_span_count=None, -# stack_trace=None, -# annotations=None, -# message_events=None, -# links=None, -# status=None, -# same_process_as_parent_span=None, -# span_kind=0, -# ) -# ] - -# stackdriver_spans = { -# 'spans': [{ -# 'status': -# None, -# 'childSpanCount': -# None, -# 'links': -# None, -# 'startTime': -# None, -# 'spanId': -# '1111', -# 'attributes': { -# 'attributeMap': { -# 'g.co/agent': { -# 'string_value': { -# 'truncated_byte_count': -# 0, -# 'value': -# 'opencensus-python [{}]'.format(__version__) -# } -# } -# } -# }, -# 'stackTrace': -# None, -# 'displayName': { -# 'truncated_byte_count': 0, -# 'value': 'span' -# }, -# 'name': -# 'projects/PROJECT/traces/{}/spans/1111'.format(trace_id), -# 'timeEvents': -# None, -# 'endTime': -# None, -# 'sameProcessAsParentSpan': -# None -# }] -# } - -# client = mock.Mock() -# project_id = 'PROJECT' -# client.project = project_id - -# exporter = trace_exporter.StackdriverExporter( -# client=client, project_id=project_id) - -# exporter.emit(span_datas) - -# name = 'projects/{}'.format(project_id) - -# client.batch_write_spans.assert_called_with(name, stackdriver_spans) -# self.assertTrue(client.batch_write_spans.called) - -# @mock.patch('opencensus.ext.stackdriver.trace_exporter.' -# 'monitored_resource.get_instance', -# return_value=None) -# def test_translate_to_stackdriver(self, mr_mock): -# project_id = 'PROJECT' -# trace_id = '6e0c63257de34c92bf9efcd03927272e' -# span_name = 'test span' -# span_id = '6e0c63257de34c92' -# attributes = { -# 'attributeMap': { -# 'key': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'value' -# } -# }, -# 'key_double': { -# 'double_value': { -# 'value': 123.45 -# } -# }, -# 'http.host': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'host' -# } -# } -# } -# } -# parent_span_id = '6e0c63257de34c93' -# start_time = 'test start time' -# end_time = 'test end time' -# trace = { -# 'spans': [{ -# 'displayName': { -# 'value': span_name, -# 'truncated_byte_count': 0 -# }, -# 'spanId': -# span_id, -# 'startTime': -# start_time, -# 'endTime': -# end_time, -# 'parentSpanId': -# parent_span_id, -# 'attributes': -# attributes, -# 'someRandomKey': -# 'this should not be included in result', -# 'childSpanCount': -# 0 -# }], -# 'traceId': -# trace_id -# } - -# client = mock.Mock() -# client.project = project_id -# exporter = trace_exporter.StackdriverExporter( -# client=client, project_id=project_id) - -# spans = list(exporter.translate_to_stackdriver(trace)) - -# expected_traces = [{ -# 'name': 'projects/{}/traces/{}/spans/{}'.format( -# project_id, trace_id, span_id), -# 'displayName': { -# 'value': span_name, -# 'truncated_byte_count': 0 -# }, -# 'attributes': { -# 'attributeMap': { -# 'g.co/agent': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': -# 'opencensus-python [{}]'.format(__version__) -# } -# }, -# 'key': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'value' -# } -# }, -# 'key_double': { -# 'double_value': { -# 'value': 123.45 -# } -# }, -# '/http/host': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'host' -# } -# } -# } -# }, -# 'spanId': str(span_id), -# 'startTime': start_time, -# 'endTime': end_time, -# 'parentSpanId': str(parent_span_id), -# 'status': None, -# 'links': None, -# 'stackTrace': None, -# 'timeEvents': None, -# 'childSpanCount': 0, -# 'sameProcessAsParentSpan': None -# }] - -# self.assertEqual(spans, expected_traces) - -# def test_translate_common_attributes_to_stackdriver_no_attribute_map(self): -# project_id = 'PROJECT' -# client = mock.Mock() -# client.project = project_id -# exporter = trace_exporter.StackdriverExporter( -# client=client, project_id=project_id) - -# attributes = {'outer key': 'some value'} -# expected_attributes = {'outer key': 'some value'} - -# exporter.map_attributes(attributes) -# self.assertEqual(attributes, expected_attributes) - -# def test_translate_common_attributes_to_stackdriver_none(self): -# project_id = 'PROJECT' -# client = mock.Mock() -# client.project = project_id -# exporter = trace_exporter.StackdriverExporter( -# client=client, project_id=project_id) - -# # does not throw -# self.assertIsNone(exporter.map_attributes(None)) - -# def test_translate_common_attributes_to_stackdriver(self): -# project_id = 'PROJECT' -# client = mock.Mock() -# client.project = project_id -# exporter = trace_exporter.StackdriverExporter( -# client=client, project_id=project_id) - -# attributes = { -# 'outer key': 'some value', -# 'attributeMap': { -# 'key': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'value' -# } -# }, -# 'component': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'http' -# } -# }, -# 'error.message': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'error message' -# } -# }, -# 'error.name': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'error name' -# } -# }, -# 'http.host': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'host' -# } -# }, -# 'http.method': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'GET' -# } -# }, -# 'http.status_code': { -# 'int_value': { -# 'value': 200 -# } -# }, -# 'http.url': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'http://host:port/path?query' -# } -# }, -# 'http.user_agent': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'some user agent' -# } -# }, -# 'http.client_city': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'Redmond' -# } -# }, -# 'http.client_country': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'USA' -# } -# }, -# 'http.client_protocol': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'HTTP 1.1' -# } -# }, -# 'http.client_region': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'WA' -# } -# }, -# 'http.request_size': { -# 'int_value': { -# 'value': 100 -# } -# }, -# 'http.response_size': { -# 'int_value': { -# 'value': 10 -# } -# }, -# 'pid': { -# 'int_value': { -# 'value': 123456789 -# } -# }, -# 'tid': { -# 'int_value': { -# 'value': 987654321 -# } -# }, -# 'stacktrace': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'at unknown' -# } -# }, -# 'grpc.host_port': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'localhost:50051' -# } -# }, -# 'grpc.method': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'post' -# } -# } -# } -# } - -# expected_attributes = { -# 'outer key': 'some value', -# 'attributeMap': { -# 'key': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'value' -# } -# }, -# '/component': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'http' -# } -# }, -# '/error/message': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'error message' -# } -# }, -# '/error/name': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'error name' -# } -# }, -# '/http/host': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'host' -# } -# }, -# '/http/method': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'GET' -# } -# }, -# '/http/status_code': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': '200' -# } -# }, -# '/http/url': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'http://host:port/path?query' -# } -# }, -# '/http/user_agent': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'some user agent' -# } -# }, -# '/http/client_city': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'Redmond' -# } -# }, -# '/http/client_country': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'USA' -# } -# }, -# '/http/client_protocol': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'HTTP 1.1' -# } -# }, -# '/http/client_region': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'WA' -# } -# }, -# '/http/request/size': { -# 'int_value': { -# 'value': 100 -# } -# }, -# '/http/response/size': { -# 'int_value': { -# 'value': 10 -# } -# }, -# '/pid': { -# 'int_value': { -# 'value': 123456789 -# } -# }, -# '/tid': { -# 'int_value': { -# 'value': 987654321 -# } -# }, -# '/stacktrace': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'at unknown' -# } -# }, -# '/grpc/host_port': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'localhost:50051' -# } -# }, -# '/grpc/method': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'post' -# } -# } -# } -# } - -# exporter.map_attributes(attributes) -# self.assertEqual(attributes, expected_attributes) - -# def test_translate_common_attributes_status_code(self): -# project_id = 'PROJECT' -# client = mock.Mock() -# client.project = project_id -# exporter = trace_exporter.StackdriverExporter( -# client=client, project_id=project_id) - -# attributes = { -# 'outer key': 'some value', -# 'attributeMap': { -# 'http.status_code': { -# 'int_value': 200 -# } -# } -# } - -# expected_attributes = { -# 'outer key': 'some value', -# 'attributeMap': { -# '/http/status_code': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': '200' -# } -# } -# } -# } - -# exporter.map_attributes(attributes) -# self.assertEqual(attributes, expected_attributes) - - -# class Test_set_attributes_gae(unittest.TestCase): -# @mock.patch('opencensus.ext.stackdriver.trace_exporter.' -# 'monitored_resource.get_instance', -# return_value=None) -# def test_set_attributes_gae(self, mr_mock): -# import os - -# trace = {'spans': [{'attributes': {}}]} - -# expected = { -# 'attributes': { -# 'attributeMap': { -# 'g.co/gae/app/module': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'service' -# } -# }, -# 'g.co/gae/app/instance': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'flex' -# } -# }, -# 'g.co/gae/app/version': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'version' -# } -# }, -# 'g.co/gae/app/project': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'project' -# } -# }, -# 'g.co/agent': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': -# 'opencensus-python [{}]'.format(__version__) -# } -# }, -# } -# } -# } - -# with mock.patch.dict( -# os.environ, { -# trace_exporter._APPENGINE_FLEXIBLE_ENV_VM: 'vm', -# trace_exporter._APPENGINE_FLEXIBLE_ENV_FLEX: 'flex', -# 'GOOGLE_CLOUD_PROJECT': 'project', -# 'GAE_SERVICE': 'service', -# 'GAE_VERSION': 'version' -# }): -# self.assertTrue(trace_exporter.is_gae_environment()) -# trace_exporter.set_attributes(trace) - -# span = trace.get('spans')[0] -# self.assertEqual(span, expected) - - -# class TestMonitoredResourceAttributes(unittest.TestCase): -# @mock.patch('opencensus.ext.stackdriver.trace_exporter.' -# 'monitored_resource.get_instance') -# def test_monitored_resource_attributes_gke(self, gmr_mock): -# import os - -# trace = {'spans': [{'attributes': {}}]} - -# expected = { -# 'attributes': { -# 'attributeMap': { -# 'g.co/gae/app/module': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'service' -# } -# }, -# 'g.co/gae/app/instance': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'flex' -# } -# }, -# 'g.co/gae/app/version': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'version' -# } -# }, -# 'g.co/gae/app/project': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'project' -# } -# }, -# 'g.co/agent': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': -# 'opencensus-python [{}]'.format(__version__) -# } -# }, -# 'g.co/r/k8s_container/project_id': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'my_project' -# } -# }, -# 'g.co/r/k8s_container/location': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'zone1' -# } -# }, -# 'g.co/r/k8s_container/namespace_name': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'namespace' -# } -# }, -# 'g.co/r/k8s_container/pod_name': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'pod' -# } -# }, -# 'g.co/r/k8s_container/cluster_name': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'cluster' -# } -# }, -# 'g.co/r/k8s_container/container_name': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'c1' -# } -# }, -# } -# } -# } - -# mock_resource = mock.Mock() -# mock_resource.get_type.return_value = 'k8s_container' -# mock_resource.get_labels.return_value = { -# 'k8s.io/pod/name': 'pod', -# 'k8s.io/cluster/name': 'cluster', -# 'k8s.io/namespace/name': 'namespace', -# 'k8s.io/container/name': 'c1', -# 'project_id': 'my_project', -# 'zone': 'zone1' -# } -# gmr_mock.return_value = mock_resource -# with mock.patch.dict( -# os.environ, { -# trace_exporter._APPENGINE_FLEXIBLE_ENV_VM: 'vm', -# trace_exporter._APPENGINE_FLEXIBLE_ENV_FLEX: 'flex', -# 'GOOGLE_CLOUD_PROJECT': 'project', -# 'GAE_SERVICE': 'service', -# 'GAE_VERSION': 'version' -# }): -# self.assertTrue(trace_exporter.is_gae_environment()) -# trace_exporter.set_attributes(trace) - -# span = trace.get('spans')[0] -# self.assertEqual(span, expected) - -# @mock.patch('opencensus.ext.stackdriver.trace_exporter.' -# 'monitored_resource.get_instance') -# def test_monitored_resource_attributes_gce(self, gmr_mock): -# trace = {'spans': [{'attributes': {}}]} - -# expected = { -# 'attributes': { -# 'attributeMap': { -# 'g.co/agent': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': -# 'opencensus-python [{}]'.format(__version__) -# } -# }, -# 'g.co/r/gce_instance/project_id': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'my_project' -# } -# }, -# 'g.co/r/gce_instance/instance_id': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': '12345' -# } -# }, -# 'g.co/r/gce_instance/zone': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'zone1' -# } -# }, -# } -# } -# } - -# mock_resource = mock.Mock() -# mock_resource.get_type.return_value = 'gce_instance' -# mock_resource.get_labels.return_value = { -# 'project_id': 'my_project', -# 'instance_id': '12345', -# 'zone': 'zone1' -# } -# gmr_mock.return_value = mock_resource -# trace_exporter.set_attributes(trace) -# span = trace.get('spans')[0] -# self.assertEqual(span, expected) - -# @mock.patch('opencensus.ext.stackdriver.trace_exporter.' -# 'monitored_resource.get_instance') -# def test_monitored_resource_attributes_aws(self, amr_mock): -# trace = {'spans': [{'attributes': {}}]} - -# expected = { -# 'attributes': { -# 'attributeMap': { -# 'g.co/agent': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': -# 'opencensus-python [{}]'.format(__version__) -# } -# }, -# 'g.co/r/aws_ec2_instance/aws_account': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': '123456789012' -# } -# }, -# 'g.co/r/aws_ec2_instance/region': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': 'aws:us-west-2' -# } -# }, -# } -# } -# } - -# mock_resource = mock.Mock() -# mock_resource.get_type.return_value = 'aws_ec2_instance' -# mock_resource.get_labels.return_value = { -# 'aws_account': '123456789012', -# 'region': 'us-west-2' -# } -# amr_mock.return_value = mock_resource - -# trace_exporter.set_attributes(trace) -# span = trace.get('spans')[0] -# self.assertEqual(span, expected) - -# @mock.patch('opencensus.ext.stackdriver.trace_exporter.' -# 'monitored_resource.get_instance') -# def test_monitored_resource_attributes_None(self, mr_mock): -# trace = {'spans': [{'attributes': {}}]} - -# expected = { -# 'attributes': { -# 'attributeMap': { -# 'g.co/agent': { -# 'string_value': { -# 'truncated_byte_count': 0, -# 'value': -# 'opencensus-python [{}]'.format(__version__) -# } -# } -# } -# } -# } - -# mr_mock.return_value = None -# trace_exporter.set_attributes(trace) -# span = trace.get('spans')[0] -# self.assertEqual(span, expected) - -# mock_resource = mock.Mock() -# mock_resource.get_type.return_value = mock.Mock() -# mock_resource.get_labels.return_value = mock.Mock() -# mr_mock.return_value = mock_resource - -# trace_exporter.set_attributes(trace) -# span = trace.get('spans')[0] -# self.assertEqual(span, expected) - - -# class MockTransport(object): -# def __init__(self, exporter=None): -# self.export_called = False -# self.exporter = exporter - -# def export(self, trace): -# self.export_called = True +# Copyright 2017, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import mock + +from opencensus.common.version import __version__ +from opencensus.ext.stackdriver import trace_exporter +from opencensus.trace import span_context +from opencensus.trace import span_data as span_data_module + + +class _Client(object): + def __init__(self, project=None): + if project is None: + project = 'PROJECT' + + self.project = project + + +class TestStackdriverExporter(unittest.TestCase): + def test_constructor_default(self): + patch = mock.patch( + 'opencensus.ext.stackdriver.trace_exporter.Client', + new=_Client) + + with patch: + exporter = trace_exporter.StackdriverExporter() + + project_id = 'PROJECT' + self.assertEqual(exporter.project_id, project_id) + + def test_constructor_explicit(self): + client = mock.Mock() + project_id = 'PROJECT' + client.project = project_id + transport = mock.Mock() + + exporter = trace_exporter.StackdriverExporter( + client=client, project_id=project_id, transport=transport) + + self.assertIs(exporter.client, client) + self.assertEqual(exporter.project_id, project_id) + + def test_export(self): + client = mock.Mock() + project_id = 'PROJECT' + client.project = project_id + exporter = trace_exporter.StackdriverExporter( + client=client, project_id=project_id, transport=MockTransport) + exporter.export({}) + + self.assertTrue(exporter.transport.export_called) + + @mock.patch('opencensus.ext.stackdriver.trace_exporter.' + 'monitored_resource.get_instance', + return_value=None) + def test_emit(self, mr_mock): + trace_id = '6e0c63257de34c92bf9efcd03927272e' + span_datas = [ + span_data_module.SpanData( + name='span', + context=span_context.SpanContext(trace_id=trace_id), + span_id='1111', + parent_span_id=None, + attributes=None, + start_time=None, + end_time=None, + child_span_count=None, + stack_trace=None, + annotations=None, + message_events=None, + links=None, + status=None, + same_process_as_parent_span=None, + span_kind=0, + ) + ] + + stackdriver_spans = { + 'spans': [{ + 'status': + None, + 'childSpanCount': + None, + 'links': + None, + 'startTime': + None, + 'spanId': + '1111', + 'attributes': { + 'attributeMap': { + 'g.co/agent': { + 'string_value': { + 'truncated_byte_count': + 0, + 'value': + 'opencensus-python [{}]'.format(__version__) + } + } + } + }, + 'stackTrace': + None, + 'displayName': { + 'truncated_byte_count': 0, + 'value': 'span' + }, + 'name': + 'projects/PROJECT/traces/{}/spans/1111'.format(trace_id), + 'timeEvents': + None, + 'endTime': + None, + 'sameProcessAsParentSpan': + None + }] + } + + client = mock.Mock() + project_id = 'PROJECT' + client.project = project_id + + exporter = trace_exporter.StackdriverExporter( + client=client, project_id=project_id) + + exporter.emit(span_datas) + + name = 'projects/{}'.format(project_id) + + client.batch_write_spans.assert_called_with(name, stackdriver_spans) + self.assertTrue(client.batch_write_spans.called) + + @mock.patch('opencensus.ext.stackdriver.trace_exporter.' + 'monitored_resource.get_instance', + return_value=None) + def test_translate_to_stackdriver(self, mr_mock): + project_id = 'PROJECT' + trace_id = '6e0c63257de34c92bf9efcd03927272e' + span_name = 'test span' + span_id = '6e0c63257de34c92' + attributes = { + 'attributeMap': { + 'key': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'value' + } + }, + 'key_double': { + 'double_value': { + 'value': 123.45 + } + }, + 'http.host': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'host' + } + } + } + } + parent_span_id = '6e0c63257de34c93' + start_time = 'test start time' + end_time = 'test end time' + trace = { + 'spans': [{ + 'displayName': { + 'value': span_name, + 'truncated_byte_count': 0 + }, + 'spanId': + span_id, + 'startTime': + start_time, + 'endTime': + end_time, + 'parentSpanId': + parent_span_id, + 'attributes': + attributes, + 'someRandomKey': + 'this should not be included in result', + 'childSpanCount': + 0 + }], + 'traceId': + trace_id + } + + client = mock.Mock() + client.project = project_id + exporter = trace_exporter.StackdriverExporter( + client=client, project_id=project_id) + + spans = list(exporter.translate_to_stackdriver(trace)) + + expected_traces = [{ + 'name': 'projects/{}/traces/{}/spans/{}'.format( + project_id, trace_id, span_id), + 'displayName': { + 'value': span_name, + 'truncated_byte_count': 0 + }, + 'attributes': { + 'attributeMap': { + 'g.co/agent': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': + 'opencensus-python [{}]'.format(__version__) + } + }, + 'key': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'value' + } + }, + 'key_double': { + 'double_value': { + 'value': 123.45 + } + }, + '/http/host': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'host' + } + } + } + }, + 'spanId': str(span_id), + 'startTime': start_time, + 'endTime': end_time, + 'parentSpanId': str(parent_span_id), + 'status': None, + 'links': None, + 'stackTrace': None, + 'timeEvents': None, + 'childSpanCount': 0, + 'sameProcessAsParentSpan': None + }] + + self.assertEqual(spans, expected_traces) + + def test_translate_common_attributes_to_stackdriver_no_attribute_map(self): + project_id = 'PROJECT' + client = mock.Mock() + client.project = project_id + exporter = trace_exporter.StackdriverExporter( + client=client, project_id=project_id) + + attributes = {'outer key': 'some value'} + expected_attributes = {'outer key': 'some value'} + + exporter.map_attributes(attributes) + self.assertEqual(attributes, expected_attributes) + + def test_translate_common_attributes_to_stackdriver_none(self): + project_id = 'PROJECT' + client = mock.Mock() + client.project = project_id + exporter = trace_exporter.StackdriverExporter( + client=client, project_id=project_id) + + # does not throw + self.assertIsNone(exporter.map_attributes(None)) + + def test_translate_common_attributes_to_stackdriver(self): + project_id = 'PROJECT' + client = mock.Mock() + client.project = project_id + exporter = trace_exporter.StackdriverExporter( + client=client, project_id=project_id) + + attributes = { + 'outer key': 'some value', + 'attributeMap': { + 'key': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'value' + } + }, + 'component': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'http' + } + }, + 'error.message': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'error message' + } + }, + 'error.name': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'error name' + } + }, + 'http.host': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'host' + } + }, + 'http.method': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'GET' + } + }, + 'http.status_code': { + 'int_value': { + 'value': 200 + } + }, + 'http.url': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'http://host:port/path?query' + } + }, + 'http.user_agent': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'some user agent' + } + }, + 'http.client_city': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'Redmond' + } + }, + 'http.client_country': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'USA' + } + }, + 'http.client_protocol': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'HTTP 1.1' + } + }, + 'http.client_region': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'WA' + } + }, + 'http.request_size': { + 'int_value': { + 'value': 100 + } + }, + 'http.response_size': { + 'int_value': { + 'value': 10 + } + }, + 'pid': { + 'int_value': { + 'value': 123456789 + } + }, + 'tid': { + 'int_value': { + 'value': 987654321 + } + }, + 'stacktrace': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'at unknown' + } + }, + 'grpc.host_port': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'localhost:50051' + } + }, + 'grpc.method': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'post' + } + } + } + } + + expected_attributes = { + 'outer key': 'some value', + 'attributeMap': { + 'key': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'value' + } + }, + '/component': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'http' + } + }, + '/error/message': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'error message' + } + }, + '/error/name': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'error name' + } + }, + '/http/host': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'host' + } + }, + '/http/method': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'GET' + } + }, + '/http/status_code': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': '200' + } + }, + '/http/url': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'http://host:port/path?query' + } + }, + '/http/user_agent': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'some user agent' + } + }, + '/http/client_city': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'Redmond' + } + }, + '/http/client_country': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'USA' + } + }, + '/http/client_protocol': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'HTTP 1.1' + } + }, + '/http/client_region': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'WA' + } + }, + '/http/request/size': { + 'int_value': { + 'value': 100 + } + }, + '/http/response/size': { + 'int_value': { + 'value': 10 + } + }, + '/pid': { + 'int_value': { + 'value': 123456789 + } + }, + '/tid': { + 'int_value': { + 'value': 987654321 + } + }, + '/stacktrace': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'at unknown' + } + }, + '/grpc/host_port': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'localhost:50051' + } + }, + '/grpc/method': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'post' + } + } + } + } + + exporter.map_attributes(attributes) + self.assertEqual(attributes, expected_attributes) + + def test_translate_common_attributes_status_code(self): + project_id = 'PROJECT' + client = mock.Mock() + client.project = project_id + exporter = trace_exporter.StackdriverExporter( + client=client, project_id=project_id) + + attributes = { + 'outer key': 'some value', + 'attributeMap': { + 'http.status_code': { + 'int_value': 200 + } + } + } + + expected_attributes = { + 'outer key': 'some value', + 'attributeMap': { + '/http/status_code': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': '200' + } + } + } + } + + exporter.map_attributes(attributes) + self.assertEqual(attributes, expected_attributes) + + +class Test_set_attributes_gae(unittest.TestCase): + @mock.patch('opencensus.ext.stackdriver.trace_exporter.' + 'monitored_resource.get_instance', + return_value=None) + def test_set_attributes_gae(self, mr_mock): + import os + + trace = {'spans': [{'attributes': {}}]} + + expected = { + 'attributes': { + 'attributeMap': { + 'g.co/gae/app/module': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'service' + } + }, + 'g.co/gae/app/instance': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'flex' + } + }, + 'g.co/gae/app/version': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'version' + } + }, + 'g.co/gae/app/project': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'project' + } + }, + 'g.co/agent': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': + 'opencensus-python [{}]'.format(__version__) + } + }, + } + } + } + + with mock.patch.dict( + os.environ, { + trace_exporter._APPENGINE_FLEXIBLE_ENV_VM: 'vm', + trace_exporter._APPENGINE_FLEXIBLE_ENV_FLEX: 'flex', + 'GOOGLE_CLOUD_PROJECT': 'project', + 'GAE_SERVICE': 'service', + 'GAE_VERSION': 'version' + }): + self.assertTrue(trace_exporter.is_gae_environment()) + trace_exporter.set_attributes(trace) + + span = trace.get('spans')[0] + self.assertEqual(span, expected) + + +class TestMonitoredResourceAttributes(unittest.TestCase): + @mock.patch('opencensus.ext.stackdriver.trace_exporter.' + 'monitored_resource.get_instance') + def test_monitored_resource_attributes_gke(self, gmr_mock): + import os + + trace = {'spans': [{'attributes': {}}]} + + expected = { + 'attributes': { + 'attributeMap': { + 'g.co/gae/app/module': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'service' + } + }, + 'g.co/gae/app/instance': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'flex' + } + }, + 'g.co/gae/app/version': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'version' + } + }, + 'g.co/gae/app/project': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'project' + } + }, + 'g.co/agent': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': + 'opencensus-python [{}]'.format(__version__) + } + }, + 'g.co/r/k8s_container/project_id': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'my_project' + } + }, + 'g.co/r/k8s_container/location': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'zone1' + } + }, + 'g.co/r/k8s_container/namespace_name': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'namespace' + } + }, + 'g.co/r/k8s_container/pod_name': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'pod' + } + }, + 'g.co/r/k8s_container/cluster_name': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'cluster' + } + }, + 'g.co/r/k8s_container/container_name': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'c1' + } + }, + } + } + } + + mock_resource = mock.Mock() + mock_resource.get_type.return_value = 'k8s_container' + mock_resource.get_labels.return_value = { + 'k8s.io/pod/name': 'pod', + 'k8s.io/cluster/name': 'cluster', + 'k8s.io/namespace/name': 'namespace', + 'k8s.io/container/name': 'c1', + 'project_id': 'my_project', + 'zone': 'zone1' + } + gmr_mock.return_value = mock_resource + with mock.patch.dict( + os.environ, { + trace_exporter._APPENGINE_FLEXIBLE_ENV_VM: 'vm', + trace_exporter._APPENGINE_FLEXIBLE_ENV_FLEX: 'flex', + 'GOOGLE_CLOUD_PROJECT': 'project', + 'GAE_SERVICE': 'service', + 'GAE_VERSION': 'version' + }): + self.assertTrue(trace_exporter.is_gae_environment()) + trace_exporter.set_attributes(trace) + + span = trace.get('spans')[0] + self.assertEqual(span, expected) + + @mock.patch('opencensus.ext.stackdriver.trace_exporter.' + 'monitored_resource.get_instance') + def test_monitored_resource_attributes_gce(self, gmr_mock): + trace = {'spans': [{'attributes': {}}]} + + expected = { + 'attributes': { + 'attributeMap': { + 'g.co/agent': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': + 'opencensus-python [{}]'.format(__version__) + } + }, + 'g.co/r/gce_instance/project_id': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'my_project' + } + }, + 'g.co/r/gce_instance/instance_id': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': '12345' + } + }, + 'g.co/r/gce_instance/zone': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'zone1' + } + }, + } + } + } + + mock_resource = mock.Mock() + mock_resource.get_type.return_value = 'gce_instance' + mock_resource.get_labels.return_value = { + 'project_id': 'my_project', + 'instance_id': '12345', + 'zone': 'zone1' + } + gmr_mock.return_value = mock_resource + trace_exporter.set_attributes(trace) + span = trace.get('spans')[0] + self.assertEqual(span, expected) + + @mock.patch('opencensus.ext.stackdriver.trace_exporter.' + 'monitored_resource.get_instance') + def test_monitored_resource_attributes_aws(self, amr_mock): + trace = {'spans': [{'attributes': {}}]} + + expected = { + 'attributes': { + 'attributeMap': { + 'g.co/agent': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': + 'opencensus-python [{}]'.format(__version__) + } + }, + 'g.co/r/aws_ec2_instance/aws_account': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': '123456789012' + } + }, + 'g.co/r/aws_ec2_instance/region': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': 'aws:us-west-2' + } + }, + } + } + } + + mock_resource = mock.Mock() + mock_resource.get_type.return_value = 'aws_ec2_instance' + mock_resource.get_labels.return_value = { + 'aws_account': '123456789012', + 'region': 'us-west-2' + } + amr_mock.return_value = mock_resource + + trace_exporter.set_attributes(trace) + span = trace.get('spans')[0] + self.assertEqual(span, expected) + + @mock.patch('opencensus.ext.stackdriver.trace_exporter.' + 'monitored_resource.get_instance') + def test_monitored_resource_attributes_None(self, mr_mock): + trace = {'spans': [{'attributes': {}}]} + + expected = { + 'attributes': { + 'attributeMap': { + 'g.co/agent': { + 'string_value': { + 'truncated_byte_count': 0, + 'value': + 'opencensus-python [{}]'.format(__version__) + } + } + } + } + } + + mr_mock.return_value = None + trace_exporter.set_attributes(trace) + span = trace.get('spans')[0] + self.assertEqual(span, expected) + + mock_resource = mock.Mock() + mock_resource.get_type.return_value = mock.Mock() + mock_resource.get_labels.return_value = mock.Mock() + mr_mock.return_value = mock_resource + + trace_exporter.set_attributes(trace) + span = trace.get('spans')[0] + self.assertEqual(span, expected) + + +class MockTransport(object): + def __init__(self, exporter=None): + self.export_called = False + self.exporter = exporter + + def export(self, trace): + self.export_called = True diff --git a/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_stats.py b/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_stats.py index df82bd496..d301a0d15 100644 --- a/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_stats.py +++ b/contrib/opencensus-ext-stackdriver/tests/test_stackdriver_stats.py @@ -1,1386 +1,1410 @@ -# noqa: E501 -# # Copyright 2018, OpenCensus Authors -# # -# # Licensed under the Apache License, Version 2.0 (the "License"); -# # you may not use this file except in compliance with the License. -# # You may obtain a copy of the License at -# # -# # http://www.apache.org/licenses/LICENSE-2.0 -# # -# # Unless required by applicable law or agreed to in writing, software -# # distributed under the License is distributed on an "AS IS" BASIS, -# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# # See the License for the specific language governing permissions and -# # limitations under the License. - -# import unittest -# from datetime import datetime - -# import google.auth -# import mock -# from google.cloud import monitoring_v3 - -# from opencensus.common import utils -# from opencensus.common.version import __version__ -# from opencensus.ext.stackdriver import stats_exporter as stackdriver -# from opencensus.metrics import label_key, label_value -# from opencensus.metrics import transport as transport_module -# from opencensus.metrics.export import (metric, metric_descriptor, point, -# time_series, value) -# from opencensus.stats import aggregation as aggregation_module -# from opencensus.stats import aggregation_data as aggregation_data_module -# from opencensus.stats import execution_context -# from opencensus.stats import measure as measure_module -# from opencensus.stats import metric_utils -# from opencensus.stats import stats as stats_module -# from opencensus.stats import view as view_module -# from opencensus.stats import view_data as view_data_module -# from opencensus.tags import tag_key as tag_key_module -# from opencensus.tags import tag_map as tag_map_module -# from opencensus.tags import tag_value as tag_value_module - -# MiB = 1 << 20 -# FRONTEND_KEY = tag_key_module.TagKey("my.org/keys/frontend") -# FRONTEND_KEY_FLOAT = tag_key_module.TagKey("my.org/keys/frontend-FLOAT") -# FRONTEND_KEY_INT = tag_key_module.TagKey("my.org/keys/frontend-INT") -# FRONTEND_KEY_STR = tag_key_module.TagKey("my.org/keys/frontend-STR") - -# FRONTEND_KEY_CLEAN = "my_org_keys_frontend" -# FRONTEND_KEY_FLOAT_CLEAN = "my_org_keys_frontend_FLOAT" -# FRONTEND_KEY_INT_CLEAN = "my_org_keys_frontend_INT" -# FRONTEND_KEY_STR_CLEAN = "my_org_keys_frontend_STR" - -# VIDEO_SIZE_MEASURE = measure_module.MeasureFloat( -# "my.org/measure/video_size_test2", "size of processed videos", "By") -# VIDEO_SIZE_MEASURE_2 = measure_module.MeasureFloat( -# "my.org/measure/video_size_test_2", "size of processed videos", "By") - -# VIDEO_SIZE_MEASURE_FLOAT = measure_module.MeasureFloat( -# "my.org/measure/video_size_test-float", "size of processed videos-float", -# "By") - -# VIDEO_SIZE_VIEW_NAME = "my.org/views/video_size_test2" -# VIDEO_SIZE_DISTRIBUTION = aggregation_module.DistributionAggregation( -# [16.0 * MiB, 256.0 * MiB]) -# VIDEO_SIZE_VIEW = view_module.View( -# VIDEO_SIZE_VIEW_NAME, "processed video size over time", [FRONTEND_KEY], -# VIDEO_SIZE_MEASURE, VIDEO_SIZE_DISTRIBUTION) - -# TEST_TIME = datetime(2018, 12, 25, 1, 2, 3, 4) -# TEST_TIME_STR = utils.to_iso_str(TEST_TIME) - - -# class _Client(object): -# def __init__(self, client_info=None): -# self.client_info = client_info - - -# class TestOptions(unittest.TestCase): -# def test_options_blank(self): -# options = stackdriver.Options() - -# self.assertEqual(options.project_id, "") -# self.assertEqual(options.resource, "") - -# def test_options_parameters(self): -# options = stackdriver.Options( -# project_id="project-id", metric_prefix="sample") -# self.assertEqual(options.project_id, "project-id") -# self.assertEqual(options.metric_prefix, "sample") - -# def test_default_monitoring_labels(self): -# options = stackdriver.Options(default_monitoring_labels={ -# label_key.LabelKey('lk_key', 'lk_desc'): -# label_value.LabelValue('lk_value') -# }) - -# self.assertEqual(len(options.default_monitoring_labels), 1) -# [[lk, lv]] = options.default_monitoring_labels.items() -# self.assertEqual(lk.key, 'lk_key') -# self.assertEqual(lk.description, 'lk_desc') -# self.assertEqual(lv.value, 'lk_value') - -# def test_default_monitoring_labels_blank(self): -# with mock.patch('opencensus.ext.stackdriver.stats_exporter' -# '.get_task_value') as mock_gtv: -# options = stackdriver.Options() - -# mock_gtv.assert_called() - -# self.assertEqual(len(options.default_monitoring_labels), 1) -# [[lk, lv]] = options.default_monitoring_labels.items() -# self.assertEqual(lk.key, stackdriver.OPENCENSUS_TASK) -# self.assertEqual(lk.description, -# stackdriver.OPENCENSUS_TASK_DESCRIPTION) -# self.assertEqual(lv.value, mock_gtv()) - -# def test_bad_default_monitoring_labels(self): -# with self.assertRaises(AttributeError): -# stackdriver.Options( -# default_monitoring_labels=[ -# 'not a dict' -# ]) - -# with self.assertRaises(TypeError): -# stackdriver.Options( -# default_monitoring_labels={ -# 'bad key': -# label_value.LabelValue('clk_value') -# }) - -# with self.assertRaises(TypeError): -# stackdriver.Options( -# default_monitoring_labels={ -# label_key.LabelKey('clk_key', 'clk_desc'): -# 'bad value' -# }) - - -# class TestStackdriverStatsExporter(unittest.TestCase): -# def test_constructor(self): -# exporter = stackdriver.StackdriverStatsExporter() - -# self.assertIsNone(exporter.client) - -# def test_constructor_param(self): -# exporter = stackdriver.StackdriverStatsExporter( -# options=stackdriver.Options(project_id=1)) -# self.assertEqual(exporter.options.project_id, 1) - -# def test_null_options(self): -# # Check that we don't suppress auth errors -# auth_error = google.auth.exceptions.DefaultCredentialsError -# mock_auth_error = mock.Mock() -# mock_auth_error.side_effect = auth_error -# with mock.patch('opencensus.ext.stackdriver.stats_exporter' -# '.google.auth.default', mock_auth_error): -# with self.assertRaises(auth_error): -# stackdriver.new_stats_exporter() - -# # Check that we get the default credentials' project ID -# mock_auth_ok = mock.Mock() -# mock_auth_ok.return_value = (None, 123) -# with mock.patch('opencensus.ext.stackdriver.stats_exporter' -# '.google.auth.default', mock_auth_ok): -# sdse = stackdriver.new_stats_exporter() -# self.assertEqual(sdse.options.project_id, 123) - -# # Check that we raise if auth works but the project is empty -# mock_auth_no_project = mock.Mock() -# mock_auth_no_project.return_value = (None, '') -# with mock.patch('opencensus.ext.stackdriver.stats_exporter' -# '.google.auth.default', mock_auth_no_project): -# with self.assertRaises(ValueError): -# stackdriver.new_stats_exporter() - -# def test_blank_project(self): -# self.assertRaises(ValueError, stackdriver.new_stats_exporter, -# stackdriver.Options(project_id="")) - -# def test_not_blank_project(self): -# patch_client = mock.patch( -# ('opencensus.ext.stackdriver.stats_exporter' -# '.monitoring_v3.MetricServiceClient'), _Client) - -# with patch_client: -# exporter_created = stackdriver.new_stats_exporter( -# stackdriver.Options(project_id=1)) - -# self.assertIsInstance(exporter_created, -# stackdriver.StackdriverStatsExporter) - -# def test_get_user_agent_slug(self): -# self.assertIn(__version__, stackdriver.get_user_agent_slug()) - -# def test_client_info_user_agent(self): -# """Check that the monitoring client sets a user agent. - -# The user agent should include the library version. Note that this -# assumes MetricServiceClient calls ClientInfo.to_user_agent to attach -# the user agent as metadata to metric service API calls. -# """ -# patch_client = mock.patch( -# 'opencensus.ext.stackdriver.stats_exporter.monitoring_v3' -# '.MetricServiceClient', _Client) - -# with patch_client: -# exporter = stackdriver.new_stats_exporter( -# stackdriver.Options(project_id=1)) - -# self.assertIn(stackdriver.get_user_agent_slug(), -# exporter.client.client_info.to_user_agent()) - -# def test_sanitize(self): -# # empty -# result = stackdriver.sanitize_label("") -# self.assertEqual(result, "") - -# # all invalid -# result = stackdriver.sanitize_label("/*^#$") -# self.assertEqual(result, "key_") - -# # all valid -# result = stackdriver.sanitize_label("abc") -# self.assertEqual(result, "abc") - -# # mixed -# result = stackdriver.sanitize_label("a.b/c") -# self.assertEqual(result, "a_b_c") - -# # starts with '_' -# result = stackdriver.sanitize_label("_abc") -# self.assertEqual(result, "key_abc") - -# # starts with digit -# result = stackdriver.sanitize_label("0abc") -# self.assertEqual(result, "key_0abc") - -# # too long -# result = stackdriver.sanitize_label("0123456789" * 10) -# self.assertEqual(len(result), 100) -# self.assertEqual(result, "key_" + "0123456789" * 9 + "012345") - -# def test_get_task_value(self): -# task_value = stackdriver.get_task_value() -# self.assertNotEqual(task_value, "") - -# def test_namespaced_views(self): -# view_name = "view-1" -# expected_view_name_namespaced = ( -# "custom.googleapis.com/opencensus/{}".format(view_name)) -# view_name_namespaced = stackdriver.namespaced_view_name(view_name, "") -# self.assertEqual(expected_view_name_namespaced, view_name_namespaced) - -# expected_view_name_namespaced = "kubernetes.io/myorg/%s" % view_name -# view_name_namespaced = stackdriver.namespaced_view_name( -# view_name, "kubernetes.io/myorg") -# self.assertEqual(expected_view_name_namespaced, view_name_namespaced) - -# def test_stackdriver_register_exporter(self): -# stats = stats_module.stats -# view_manager = stats.view_manager - -# exporter = mock.Mock() -# if len(view_manager.measure_to_view_map.exporters) > 0: -# view_manager.unregister_exporter( -# view_manager.measure_to_view_map.exporters[0]) -# view_manager.register_exporter(exporter) - -# registered_exporters = len(view_manager.measure_to_view_map.exporters) - -# self.assertEqual(registered_exporters, 1) - -# @mock.patch('os.getpid', return_value=12345) -# @mock.patch( -# 'platform.uname', -# return_value=('system', 'node', 'release', 'version', 'machine', -# 'processor')) -# def test_get_task_value_with_hostname(self, mock_uname, mock_pid): -# self.assertEqual(stackdriver.get_task_value(), "py-12345@node") - -# @mock.patch('os.getpid', return_value=12345) -# @mock.patch( -# 'platform.uname', -# return_value=('system', '', 'release', 'version', 'machine', -# 'processor')) -# def test_get_task_value_without_hostname(self, mock_uname, mock_pid): -# self.assertEqual(stackdriver.get_task_value(), "py-12345@localhost") - -# def test_default_default_monitoring_labels(self): -# """Check that metrics include OC task label by default.""" -# exporter = stackdriver.StackdriverStatsExporter( -# options=stackdriver.Options(project_id='project_id'), -# client=mock.Mock()) - -# lv = label_value.LabelValue('val') -# val = value.ValueLong(value=123) -# dt = datetime(2019, 3, 20, 21, 34, 0, 537954) -# pp = point.Point(value=val, timestamp=dt) -# ts = [ -# time_series.TimeSeries(label_values=[lv], points=[pp], -# start_timestamp=utils.to_iso_str(dt)) -# ] - -# desc = metric_descriptor.MetricDescriptor( -# name='name', -# description='description', -# unit='unit', -# type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, -# label_keys=[label_key.LabelKey('key', 'description')] -# ) -# mm = metric.Metric(descriptor=desc, time_series=ts) - -# sd_md = exporter.get_metric_descriptor(desc) -# self.assertEqual(len(sd_md.labels), 2) -# sd_descriptors = {ld.key: ld.description for ld in sd_md.labels} -# self.assertIn('key', sd_descriptors) -# self.assertEqual(sd_descriptors['key'], 'description') -# self.assertIn(stackdriver.OPENCENSUS_TASK, sd_descriptors) -# self.assertEqual( -# sd_descriptors[stackdriver.OPENCENSUS_TASK], -# stackdriver.OPENCENSUS_TASK_DESCRIPTION -# ) - -# sd_ts_list = exporter.create_time_series_list(mm) -# self.assertEqual(len(sd_ts_list), 1) -# [sd_ts] = sd_ts_list -# self.assertIn('key', sd_ts.metric.labels) -# self.assertEqual(sd_ts.metric.labels['key'], 'val') -# self.assertIn(stackdriver.OPENCENSUS_TASK, sd_ts.metric.labels) - -# def test_empty_default_monitoring_labels(self): -# """Check that it's possible to remove the default OC task label.""" -# exporter = stackdriver.StackdriverStatsExporter( -# options=stackdriver.Options( -# project_id='project_id', -# default_monitoring_labels={}), -# client=mock.Mock()) - -# lv = label_value.LabelValue('val') -# val = value.ValueLong(value=123) -# dt = datetime(2019, 3, 20, 21, 34, 0, 537954) -# pp = point.Point(value=val, timestamp=dt) -# ts = [ -# time_series.TimeSeries(label_values=[lv], points=[pp], -# start_timestamp=utils.to_iso_str(dt)) -# ] - -# desc = metric_descriptor.MetricDescriptor( -# name='name', -# description='description', -# unit='unit', -# type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, -# label_keys=[label_key.LabelKey('key', 'description')] -# ) -# mm = metric.Metric(descriptor=desc, time_series=ts) - -# sd_md = exporter.get_metric_descriptor(desc) -# self.assertEqual(len(sd_md.labels), 1) -# [sd_label] = sd_md.labels -# self.assertEqual(sd_label.key, 'key') -# self.assertEqual(sd_label.description, 'description') - -# sd_ts_list = exporter.create_time_series_list(mm) -# self.assertEqual(len(sd_ts_list), 1) -# [sd_ts] = sd_ts_list -# self.assertIn('key', sd_ts.metric.labels) -# self.assertEqual(sd_ts.metric.labels['key'], 'val') -# self.assertNotIn(stackdriver.OPENCENSUS_TASK, sd_ts.metric.labels) - -# def test_custom_default_monitoring_labels(self): -# """Check that custom labels are exported and included in descriptor.""" -# exporter = stackdriver.StackdriverStatsExporter( -# options=stackdriver.Options( -# project_id='project_id', -# default_monitoring_labels={ -# label_key.LabelKey('clk_key', 'clk_desc'): -# label_value.LabelValue('clk_value') -# }), -# client=mock.Mock()) - -# lv = label_value.LabelValue('val') -# val = value.ValueLong(value=123) -# dt = datetime(2019, 3, 20, 21, 34, 0, 537954) -# pp = point.Point(value=val, timestamp=dt) -# ts = [ -# time_series.TimeSeries(label_values=[lv], points=[pp], -# start_timestamp=utils.to_iso_str(dt)) -# ] - -# desc = metric_descriptor.MetricDescriptor( -# name='name', -# description='description', -# unit='unit', -# type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, -# label_keys=[label_key.LabelKey('key', 'description')] -# ) -# mm = metric.Metric(descriptor=desc, time_series=ts) - -# sd_md = exporter.get_metric_descriptor(desc) -# self.assertEqual(len(sd_md.labels), 2) -# sd_descriptors = {ld.key: ld.description for ld in sd_md.labels} -# self.assertIn('key', sd_descriptors) -# self.assertEqual(sd_descriptors['key'], 'description') -# self.assertIn('clk_key', sd_descriptors) -# self.assertEqual(sd_descriptors['clk_key'], 'clk_desc') - -# sd_ts_list = exporter.create_time_series_list(mm) -# self.assertEqual(len(sd_ts_list), 1) -# [sd_ts] = sd_ts_list -# self.assertIn('key', sd_ts.metric.labels) -# self.assertEqual(sd_ts.metric.labels['key'], 'val') -# self.assertIn('clk_key', sd_ts.metric.labels) -# self.assertEqual(sd_ts.metric.labels['clk_key'], 'clk_value') - -# def test_get_metric_descriptor(self): -# exporter = stackdriver.StackdriverStatsExporter( -# options=stackdriver.Options( -# project_id='project_id'), -# client=mock.Mock()) - -# oc_md = metric_descriptor.MetricDescriptor( -# name='name', -# description='description', -# unit='unit', -# type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, -# label_keys=[label_key.LabelKey('ck', 'cd')] -# ) - -# sd_md = exporter.get_metric_descriptor(oc_md) -# self.assertEqual( -# sd_md.metric_kind, -# monitoring_v3.enums.MetricDescriptor.MetricKind.GAUGE) -# self.assertEqual( -# sd_md.value_type, -# monitoring_v3.enums.MetricDescriptor.ValueType.INT64) - -# self.assertIsInstance(sd_md, monitoring_v3.types.MetricDescriptor) -# exporter.client.create_metric_descriptor.assert_not_called() - -# def test_get_metric_descriptor_bad_type(self): -# exporter = stackdriver.StackdriverStatsExporter( -# options=stackdriver.Options(project_id='project_id'), -# client=mock.Mock()) - -# bad_type_oc_md = metric_descriptor.MetricDescriptor( -# name='name', -# description='description', -# unit='unit', -# # Need a valid type to create the descriptor -# type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, -# label_keys=[label_key.LabelKey('key', 'description')] -# ) -# bad_type_oc_md._type = 100 - -# with self.assertRaises(TypeError): -# exporter.get_metric_descriptor(bad_type_oc_md) - -# def test_get_metric_descriptor_custom_prefix(self): - -# exporter = stackdriver.StackdriverStatsExporter( -# options=stackdriver.Options( -# metric_prefix='metric_prefix', -# project_id='project_id'), -# client=mock.Mock()) - -# oc_md = metric_descriptor.MetricDescriptor( -# name='name', -# description='description', -# unit='unit', -# type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, -# label_keys=[label_key.LabelKey('ck', 'cd')] -# ) - -# sd_md = exporter.get_metric_descriptor(oc_md) -# self.assertIn('metric_prefix', sd_md.type) -# self.assertIn('metric_prefix', sd_md.name) - -# def test_register_metric_descriptor(self): -# exporter = stackdriver.StackdriverStatsExporter( -# options=stackdriver.Options( -# metric_prefix='metric_prefix', -# project_id='project_id'), -# client=mock.Mock()) - -# oc_md = metric_descriptor.MetricDescriptor( -# name='name', -# description='description', -# unit='unit', -# type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, -# label_keys=[label_key.LabelKey('key', 'description')] -# ) - -# exporter.register_metric_descriptor(oc_md) -# self.assertEqual( -# exporter.client.create_metric_descriptor.call_count, -# 1 -# ) -# exporter.register_metric_descriptor(oc_md) -# self.assertEqual( -# exporter.client.create_metric_descriptor.call_count, -# 1 -# ) - -# def test_export_metrics(self): -# lv = label_value.LabelValue('val') -# val = value.ValueLong(value=123) -# dt = datetime(2019, 3, 20, 21, 34, 0, 537954) -# pp = point.Point(value=val, timestamp=dt) - -# ts = [ -# time_series.TimeSeries(label_values=[lv], points=[pp], -# start_timestamp=utils.to_iso_str(dt)) -# ] - -# desc = metric_descriptor.MetricDescriptor( -# name='name', -# description='description', -# unit='unit', -# type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, -# label_keys=[label_key.LabelKey('key', 'description')] -# ) - -# mm = metric.Metric(descriptor=desc, time_series=ts) - -# exporter = stackdriver.StackdriverStatsExporter(client=mock.Mock()) -# exporter.export_metrics([mm]) - -# self.assertEqual(exporter.client.create_time_series.call_count, 1) -# sd_args = exporter.client.create_time_series.call_args[0][1] -# self.assertEqual(len(sd_args), 1) -# [sd_arg] = exporter.client.create_time_series.call_args[0][1] -# self.assertEqual(sd_arg.points[0].value.int64_value, 123) - - -# class MockPeriodicMetricTask(object): -# """Testing mock of metrics.transport.PeriodicMetricTask. - -# Simulate calling export asynchronously from another thread synchronously -# from this one. -# """ -# def __init__(self, interval=None, function=None, args=None, kwargs=None): -# self.function = function -# self.logger = mock.Mock() -# self.start = mock.Mock() -# self.run = mock.Mock() - -# def step(self): -# try: -# self.function() -# except transport_module.TransportError as ex: -# self.logger.exception(ex) -# self.cancel() -# except Exception: -# self.logger.exception("Error handling metric export") - - -# class MockGetExporterThread(object): -# """Intercept calls to get_exporter_thread. - -# To get a reference to the running PeriodicMetricTask created by -# get_exporter_thread. -# """ -# def __init__(self): -# self.transport = None - -# def __enter__(self): -# original_func = transport_module.get_exporter_thread - -# def get_exporter_thread(*aa, **kw): -# self.transport = original_func(*aa, **kw) - -# mock_get = mock.Mock() -# mock_get.side_effect = get_exporter_thread -# self.patcher = mock.patch( -# ('opencensus.ext.stackdriver.stats_exporter' -# '.transport.get_exporter_thread'), -# mock_get) -# self.patcher.start() -# return self - -# def __exit__(self, type, value, traceback): -# self.patcher.stop() - - -# @mock.patch('opencensus.ext.stackdriver.stats_exporter' -# '.monitoring_v3.MetricServiceClient') -# @mock.patch('opencensus.ext.stackdriver.stats_exporter' -# '.stats.stats') -# class TestAsyncStatsExport(unittest.TestCase): -# """Check that metrics are exported using the exporter thread.""" - -# def setUp(self): -# patcher = mock.patch( -# 'opencensus.metrics.transport.PeriodicMetricTask', -# MockPeriodicMetricTask) -# patcher.start() -# self.addCleanup(patcher.stop) - -# def test_export_empty(self, mock_stats, mock_client): -# """Check that we don't attempt to export empty metric sets.""" - -# mock_stats.get_metrics.return_value = [] - -# with MockGetExporterThread() as mget: -# exporter = stackdriver.new_stats_exporter( -# stackdriver.Options(project_id=1)) -# mget.transport.step() - -# exporter.client.create_metric_descriptor.assert_not_called() -# exporter.client.create_time_series.assert_not_called() - -# def test_export_single_metric(self, mock_stats, mock_client): -# """Check that we can export a set of a single metric.""" - -# lv = label_value.LabelValue('val') -# val = value.ValueLong(value=123) -# dt = datetime(2019, 3, 20, 21, 34, 0, 537954) -# pp = point.Point(value=val, timestamp=dt) - -# ts = [ -# time_series.TimeSeries(label_values=[lv], points=[pp], -# start_timestamp=utils.to_iso_str(dt)) -# ] - -# desc = metric_descriptor.MetricDescriptor( -# name='name2', -# description='description2', -# unit='unit2', -# type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, -# label_keys=[label_key.LabelKey('key', 'description')] -# ) - -# mm = metric.Metric(descriptor=desc, time_series=ts) -# mock_stats.get_metrics.return_value = [mm] - -# with MockGetExporterThread() as mget: -# exporter = stackdriver.new_stats_exporter( -# stackdriver.Options(project_id=1)) -# mget.transport.step() - -# exporter.client.create_metric_descriptor.assert_called() -# self.assertEqual( -# exporter.client.create_metric_descriptor.call_count, -# 1) -# md_call_arg =\ -# exporter.client.create_metric_descriptor.call_args[0][1] -# self.assertEqual( -# md_call_arg.metric_kind, -# monitoring_v3.enums.MetricDescriptor.MetricKind.GAUGE -# ) -# self.assertEqual( -# md_call_arg.value_type, -# monitoring_v3.enums.MetricDescriptor.ValueType.INT64 -# ) - -# exporter.client.create_time_series.assert_called() -# self.assertEqual( -# exporter.client.create_time_series.call_count, -# 1) -# ts_call_arg = exporter.client.create_time_series.call_args[0][1] -# self.assertEqual(len(ts_call_arg), 1) -# self.assertEqual(len(ts_call_arg[0].points), 1) -# self.assertEqual(ts_call_arg[0].points[0].value.int64_value, 123) - - -# class TestCreateTimeseries(unittest.TestCase): - -# def setUp(self): -# patcher = mock.patch( -# 'opencensus.ext.stackdriver.stats_exporter.stats.stats', -# stats_module._Stats()) -# patcher.start() -# self.addCleanup(patcher.stop) - -# def check_labels(self, -# actual_labels, -# expected_labels, -# include_opencensus=False): -# actual_labels = dict(actual_labels) -# if include_opencensus: -# opencensus_tag = actual_labels.pop(stackdriver.OPENCENSUS_TASK) -# self.assertIsNotNone(opencensus_tag) -# self.assertIn("py-", opencensus_tag) -# self.assertDictEqual(actual_labels, expected_labels) - -# @mock.patch('opencensus.ext.stackdriver.stats_exporter.' -# 'monitored_resource.get_instance', -# return_value=None) -# def test_create_batched_time_series(self, monitor_resource_mock): -# client = mock.Mock() -# v_data = view_data_module.ViewData( -# view=VIDEO_SIZE_VIEW, -# start_time=TEST_TIME_STR, -# end_time=TEST_TIME_STR) -# v_data.record(context=tag_map_module.TagMap(), value=2, timestamp=None) -# view_data = [v_data] - -# option = stackdriver.Options(project_id="project-test") -# exporter = stackdriver.StackdriverStatsExporter( -# options=option, client=client) - -# view_data = [metric_utils.view_data_to_metric(view_data[0], TEST_TIME)] - -# time_series_batches = exporter.create_batched_time_series(view_data, 1) - -# self.assertEqual(len(time_series_batches), 1) -# [time_series_batch] = time_series_batches -# self.assertEqual(len(time_series_batch), 1) -# [time_series] = time_series_batch -# self.assertEqual( -# time_series.metric.type, -# 'custom.googleapis.com/opencensus/' + VIDEO_SIZE_VIEW_NAME) -# self.check_labels( -# time_series.metric.labels, {}, include_opencensus=True) - -# @mock.patch('opencensus.ext.stackdriver.stats_exporter.' -# 'monitored_resource.get_instance', -# return_value=None) -# def test_create_batched_time_series_with_many(self, monitor_resource_mock): -# client = mock.Mock() - -# # First view with 3 -# view_name1 = "view-name1" -# view1 = view_module.View(view_name1, "test description", ['test'], -# VIDEO_SIZE_MEASURE, -# aggregation_module.LastValueAggregation()) -# v_data1 = view_data_module.ViewData( -# view=view1, start_time=TEST_TIME_STR, end_time=TEST_TIME_STR) -# v_data1.record(context=tag_map_module.TagMap({'test': '1'}), value=7, -# timestamp=None) -# v_data1.record(context=tag_map_module.TagMap({'test': '2'}), value=5, -# timestamp=None) -# v_data1.record(context=tag_map_module.TagMap({'test': '3'}), value=3, -# timestamp=None) - -# # Second view with 2 -# view_name2 = "view-name2" -# view2 = view_module.View(view_name2, "test description", ['test'], -# VIDEO_SIZE_MEASURE, -# aggregation_module.LastValueAggregation()) -# v_data2 = view_data_module.ViewData( -# view=view2, start_time=TEST_TIME_STR, end_time=TEST_TIME_STR) -# v_data2.record(context=tag_map_module.TagMap({'test': '1'}), value=7, -# timestamp=None) -# v_data2.record(context=tag_map_module.TagMap({'test': '2'}), value=5, -# timestamp=None) - -# view_data = [v_data1, v_data2] -# view_data = [metric_utils.view_data_to_metric(vd, TEST_TIME) -# for vd in view_data] - -# option = stackdriver.Options(project_id="project-test") -# exporter = stackdriver.StackdriverStatsExporter( -# options=option, client=client) - -# time_series_batches = exporter.create_batched_time_series(view_data, 2) - -# self.assertEqual(len(time_series_batches), 3) -# [tsb1, tsb2, tsb3] = time_series_batches -# self.assertEqual(len(tsb1), 2) -# self.assertEqual(len(tsb2), 2) -# self.assertEqual(len(tsb3), 1) - -# def setup_create_timeseries_test(self): -# client = mock.Mock() -# execution_context.clear() - -# option = stackdriver.Options( -# project_id="project-test", resource="global") -# exporter = stackdriver.StackdriverStatsExporter( -# options=option, client=client) - -# stats = stats_module.stats -# view_manager = stats.view_manager -# stats_recorder = stats.stats_recorder - -# if len(view_manager.measure_to_view_map.exporters) > 0: -# view_manager.unregister_exporter( -# view_manager.measure_to_view_map.exporters[0]) - -# view_manager.register_exporter(exporter) -# return view_manager, stats_recorder, exporter - -# @mock.patch('opencensus.ext.stackdriver.stats_exporter.' -# 'monitored_resource.get_instance', -# return_value=None) -# def test_create_timeseries(self, monitor_resource_mock): -# view_manager, stats_recorder, exporter = \ -# self.setup_create_timeseries_test() - -# view_manager.register_view(VIDEO_SIZE_VIEW) - -# tag_value = tag_value_module.TagValue("1200") -# tag_map = tag_map_module.TagMap() -# tag_map.insert(FRONTEND_KEY, tag_value) - -# measure_map = stats_recorder.new_measurement_map() -# measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 25 * MiB) -# measure_map.record(tag_map) - -# v_data = measure_map.measure_to_view_map.get_view( -# VIDEO_SIZE_VIEW_NAME, None) - -# v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) - -# time_series_list = exporter.create_time_series_list(v_data) - -# self.assertEqual(len(time_series_list), 1) -# time_series = time_series_list[0] -# self.assertEqual(time_series.resource.type, "global") -# self.assertEqual( -# time_series_list[0].metric.type, -# "custom.googleapis.com/opencensus/my.org/views/video_size_test2") -# self.check_labels( -# time_series.metric.labels, {FRONTEND_KEY_CLEAN: "1200"}, -# include_opencensus=True) -# self.assertIsNotNone(time_series.resource) - -# self.assertEqual(len(time_series.points), 1) -# value = time_series.points[0].value -# self.assertEqual(value.distribution_value.count, 1) - -# time_series_list = exporter.create_time_series_list(v_data) - -# self.assertEqual(len(time_series_list), 1) -# time_series = time_series_list[0] -# self.check_labels( -# time_series.metric.labels, {FRONTEND_KEY_CLEAN: "1200"}, -# include_opencensus=True) -# self.assertIsNotNone(time_series.resource) - -# self.assertEqual(len(time_series.points), 1) -# value = time_series.points[0].value -# self.assertEqual(value.distribution_value.count, 1) - -# @mock.patch('opencensus.ext.stackdriver.stats_exporter.' -# 'monitored_resource.get_instance') -# def test_create_timeseries_with_resource(self, monitor_resource_mock): - -# client = mock.Mock() -# execution_context.clear() - -# option = stackdriver.Options(project_id="project-test", resource="") -# exporter = stackdriver.StackdriverStatsExporter( -# options=option, client=client) - -# stats = stats_module.stats -# view_manager = stats.view_manager -# stats_recorder = stats.stats_recorder - -# if len(view_manager.measure_to_view_map.exporters) > 0: -# view_manager.unregister_exporter( -# view_manager.measure_to_view_map.exporters[0]) - -# view_manager.register_exporter(exporter) -# view_manager.register_view(VIDEO_SIZE_VIEW) - -# tag_value = tag_value_module.TagValue("1200") -# tag_map = tag_map_module.TagMap() -# tag_map.insert(FRONTEND_KEY, tag_value) - -# measure_map = stats_recorder.new_measurement_map() -# measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 25 * MiB) -# measure_map.record(tag_map) - -# v_data = measure_map.measure_to_view_map.get_view( -# VIDEO_SIZE_VIEW_NAME, None) - -# v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) - -# # check for gce_instance monitored resource -# mocked_labels = { -# 'instance_id': 'my-instance', -# 'project_id': 'my-project', -# 'zone': 'us-east1', -# 'k8s.io/pod/name': 'localhost', -# 'k8s.io/namespace/name': 'namespace', -# } - -# mock_resource = mock.Mock() -# mock_resource.get_type.return_value = 'gce_instance' -# mock_resource.get_labels.return_value = mocked_labels -# monitor_resource_mock.return_value = mock_resource - -# time_series_list = exporter.create_time_series_list(v_data) -# self.assertEqual(len(time_series_list), 1) -# time_series = time_series_list[0] -# self.assertEqual(time_series.resource.type, "gce_instance") -# self.check_labels( -# time_series.resource.labels, { -# 'instance_id': 'my-instance', -# 'project_id': 'my-project', -# 'zone': 'us-east1', -# }) -# self.assertEqual( -# time_series.metric.type, -# "custom.googleapis.com/opencensus/my.org/views/video_size_test2") -# self.assertIsNotNone(time_series) - -# time_series_list = exporter.create_time_series_list(v_data) -# self.assertEqual(len(time_series_list), 1) -# time_series = time_series_list[0] - -# self.assertEqual( -# time_series.metric.type, -# "custom.googleapis.com/opencensus/my.org/views/video_size_test2") - -# # check for k8s_container monitored resource -# mocked_labels = { -# 'instance_id': 'my-instance', -# 'project_id': 'my-project', -# 'zone': 'us-east1', -# 'k8s.io/pod/name': 'localhost', -# 'k8s.io/cluster/name': 'cluster', -# 'k8s.io/namespace/name': 'namespace', -# } - -# mock_resource = mock.Mock() -# mock_resource.get_type.return_value = 'k8s_container' -# mock_resource.get_labels.return_value = mocked_labels -# monitor_resource_mock.return_value = mock_resource - -# time_series_list = exporter.create_time_series_list(v_data) -# self.assertEqual(len(time_series_list), 1) -# time_series = time_series_list[0] -# self.assertEqual(time_series.resource.type, "k8s_container") -# self.check_labels( -# time_series.resource.labels, { -# 'project_id': 'my-project', -# 'location': 'us-east1', -# 'cluster_name': 'cluster', -# 'pod_name': 'localhost', -# 'namespace_name': 'namespace', -# }) -# self.assertEqual( -# time_series.metric.type, -# "custom.googleapis.com/opencensus/my.org/views/video_size_test2") -# self.assertIsNotNone(time_series) - -# # check for aws_ec2_instance monitored resource -# mocked_labels = { -# 'instance_id': 'my-instance', -# 'aws_account': 'my-project', -# 'region': 'us-east1', -# } - -# mock_resource = mock.Mock() -# mock_resource.get_type.return_value = 'aws_ec2_instance' -# mock_resource.get_labels.return_value = mocked_labels -# monitor_resource_mock.return_value = mock_resource - -# time_series_list = exporter.create_time_series_list(v_data) -# self.assertEqual(len(time_series_list), 1) -# time_series = time_series_list[0] -# self.assertEqual(time_series.resource.type, "aws_ec2_instance") -# self.check_labels( -# time_series.resource.labels, { -# 'instance_id': 'my-instance', -# 'aws_account': 'my-project', -# 'region': 'aws:us-east1', -# }) -# self.assertEqual( -# time_series.metric.type, -# "custom.googleapis.com/opencensus/my.org/views/video_size_test2") -# self.assertIsNotNone(time_series) - -# # check for out of box monitored resource -# mock_resource = mock.Mock() -# mock_resource.get_type.return_value = '' -# mock_resource.get_labels.return_value = mock.Mock() -# monitor_resource_mock.return_value = mock_resource - -# time_series_list = exporter.create_time_series_list(v_data) -# self.assertEqual(len(time_series_list), 1) -# time_series = time_series_list[0] -# self.assertEqual(time_series.resource.type, 'global') -# self.check_labels(time_series.resource.labels, {}) -# self.assertEqual( -# time_series.metric.type, -# "custom.googleapis.com/opencensus/my.org/views/video_size_test2") -# self.assertIsNotNone(time_series) - -# @mock.patch('opencensus.ext.stackdriver.stats_exporter.' -# 'monitored_resource.get_instance', -# return_value=None) -# def test_create_timeseries_str_tagvalue(self, monitor_resource_mock): -# view_manager, stats_recorder, exporter = \ -# self.setup_create_timeseries_test() - -# agg_1 = aggregation_module.LastValueAggregation(value=2) -# view_name1 = "view-name1" -# new_view1 = view_module.View( -# view_name1, "processed video size over time", [FRONTEND_KEY_INT], -# VIDEO_SIZE_MEASURE_2, agg_1) - -# view_manager.register_view(new_view1) - -# tag_value_int = tag_value_module.TagValue("Abc") -# tag_map = tag_map_module.TagMap() -# tag_map.insert(FRONTEND_KEY_INT, tag_value_int) - -# measure_map = stats_recorder.new_measurement_map() -# measure_map.measure_int_put(VIDEO_SIZE_MEASURE_2, 25 * MiB) -# measure_map.record(tag_map) - -# v_data = measure_map.measure_to_view_map.get_view(view_name1, None) - -# v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) - -# time_series_list = exporter.create_time_series_list(v_data) -# self.assertEqual(len(time_series_list), 1) -# time_series = time_series_list[0] - -# self.check_labels( -# time_series.metric.labels, {FRONTEND_KEY_INT_CLEAN: "Abc"}, -# include_opencensus=True) -# self.assertIsNotNone(time_series.resource) - -# self.assertEqual(len(time_series.points), 1) -# expected_value = monitoring_v3.types.TypedValue() -# # TODO: #565 -# expected_value.double_value = 25.0 * MiB -# self.assertEqual(time_series.points[0].value, expected_value) - -# @mock.patch('opencensus.ext.stackdriver.stats_exporter.' -# 'monitored_resource.get_instance', -# return_value=None) -# def test_create_timeseries_str_tagvalue_count_aggregtation( -# self, monitor_resource_mock): -# view_manager, stats_recorder, exporter = \ -# self.setup_create_timeseries_test() - -# agg_1 = aggregation_module.CountAggregation(count=2) -# view_name1 = "view-name1" -# new_view1 = view_module.View( -# view_name1, "processed video size over time", [FRONTEND_KEY_INT], -# VIDEO_SIZE_MEASURE_2, agg_1) - -# view_manager.register_view(new_view1) - -# tag_value_int = tag_value_module.TagValue("Abc") -# tag_map = tag_map_module.TagMap() -# tag_map.insert(FRONTEND_KEY_INT, tag_value_int) - -# measure_map = stats_recorder.new_measurement_map() -# measure_map.measure_int_put(VIDEO_SIZE_MEASURE_2, 25 * MiB) -# measure_map.record(tag_map) - -# v_data = measure_map.measure_to_view_map.get_view(view_name1, None) - -# v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) - -# time_series_list = exporter.create_time_series_list(v_data) -# self.assertEqual(len(time_series_list), 1) -# time_series = time_series_list[0] -# self.check_labels( -# time_series.metric.labels, {FRONTEND_KEY_INT_CLEAN: "Abc"}, -# include_opencensus=True) -# self.assertIsNotNone(time_series.resource) - -# self.assertEqual(len(time_series.points), 1) -# expected_value = monitoring_v3.types.TypedValue() -# expected_value.int64_value = 3 -# self.assertEqual(time_series.points[0].value, expected_value) - -# @mock.patch('opencensus.ext.stackdriver.stats_exporter.' -# 'monitored_resource.get_instance', -# return_value=None) -# def test_create_timeseries_last_value_float_tagvalue( -# self, monitor_resource_mock): -# view_manager, stats_recorder, exporter = \ -# self.setup_create_timeseries_test() - -# agg_2 = aggregation_module.LastValueAggregation(value=2.2 * MiB) -# view_name2 = "view-name2" -# new_view2 = view_module.View( -# view_name2, "processed video size over time", [FRONTEND_KEY_FLOAT], -# VIDEO_SIZE_MEASURE_FLOAT, agg_2) - -# view_manager.register_view(new_view2) - -# tag_value_float = tag_value_module.TagValue("Abc") -# tag_map = tag_map_module.TagMap() -# tag_map.insert(FRONTEND_KEY_FLOAT, tag_value_float) - -# measure_map = stats_recorder.new_measurement_map() -# measure_map.measure_float_put(VIDEO_SIZE_MEASURE_FLOAT, 25.7 * MiB) -# measure_map.record(tag_map) - -# v_data = measure_map.measure_to_view_map.get_view(view_name2, None) - -# v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) - -# time_series_list = exporter.create_time_series_list(v_data) -# self.assertEqual(len(time_series_list), 1) -# time_series = time_series_list[0] -# self.check_labels( -# time_series.metric.labels, {FRONTEND_KEY_FLOAT_CLEAN: "Abc"}, -# include_opencensus=True) -# self.assertIsNotNone(time_series.resource) - -# self.assertEqual(len(time_series.points), 1) -# expected_value = monitoring_v3.types.TypedValue() -# expected_value.double_value = 25.7 * MiB -# self.assertEqual(time_series.points[0].value, expected_value) - -# @mock.patch('opencensus.ext.stackdriver.stats_exporter.' -# 'monitored_resource.get_instance', -# return_value=None) -# def test_create_timeseries_float_tagvalue(self, monitor_resource_mock): -# client = mock.Mock() - -# option = stackdriver.Options( -# project_id="project-test", resource="global") -# exporter = stackdriver.StackdriverStatsExporter( -# options=option, client=client) - -# stats = stats_module.stats -# view_manager = stats.view_manager -# stats_recorder = stats.stats_recorder - -# if len(view_manager.measure_to_view_map.exporters) > 0: -# view_manager.unregister_exporter( -# view_manager.measure_to_view_map.exporters[0]) - -# view_manager.register_exporter(exporter) - -# agg_3 = aggregation_module.SumAggregation(sum=2.2) -# view_name3 = "view-name3" -# new_view3 = view_module.View( -# view_name3, "processed video size over time", [FRONTEND_KEY_FLOAT], -# VIDEO_SIZE_MEASURE_FLOAT, agg_3) - -# view_manager.register_view(new_view3) - -# tag_value_float = tag_value_module.TagValue("1200") -# tag_map = tag_map_module.TagMap() -# tag_map.insert(FRONTEND_KEY_FLOAT, tag_value_float) - -# measure_map = stats_recorder.new_measurement_map() -# measure_map.measure_float_put(VIDEO_SIZE_MEASURE_FLOAT, 25 * MiB) -# measure_map.record(tag_map) - -# v_data = measure_map.measure_to_view_map.get_view(view_name3, None) - -# v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) - -# time_series_list = exporter.create_time_series_list(v_data) -# self.assertEqual(len(time_series_list), 1) -# [time_series] = time_series_list -# self.assertEqual(time_series.metric.type, -# "custom.googleapis.com/opencensus/view-name3") -# self.check_labels( -# time_series.metric.labels, {FRONTEND_KEY_FLOAT_CLEAN: "1200"}, -# include_opencensus=True) -# self.assertIsNotNone(time_series.resource) - -# self.assertEqual(len(time_series.points), 1) -# expected_value = monitoring_v3.types.TypedValue() -# expected_value.double_value = 2.2 + 25 * MiB -# self.assertEqual(time_series.points[0].value, expected_value) - -# @mock.patch('opencensus.ext.stackdriver.stats_exporter.' -# 'monitored_resource.get_instance', -# return_value=None) -# def test_create_timeseries_multiple_tag_values(self, -# monitoring_resoure_mock): -# view_manager, stats_recorder, exporter = \ -# self.setup_create_timeseries_test() - -# view_manager.register_view(VIDEO_SIZE_VIEW) - -# measure_map = stats_recorder.new_measurement_map() - -# # Add first point with one tag value -# tag_map = tag_map_module.TagMap() -# tag_map.insert(FRONTEND_KEY, tag_value_module.TagValue("1200")) -# measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 25 * MiB) -# measure_map.record(tag_map) - -# # Add second point with different tag value -# tag_map = tag_map_module.TagMap() -# tag_map.insert(FRONTEND_KEY, tag_value_module.TagValue("1400")) -# measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 12 * MiB) -# measure_map.record(tag_map) - -# v_data = measure_map.measure_to_view_map.get_view( -# VIDEO_SIZE_VIEW_NAME, None) - -# v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) - -# time_series_list = exporter.create_time_series_list(v_data) - -# self.assertEqual(len(time_series_list), 2) -# ts_by_frontend = { -# ts.metric.labels.get(FRONTEND_KEY_CLEAN): ts -# for ts in time_series_list -# } -# self.assertEqual(set(ts_by_frontend.keys()), {"1200", "1400"}) -# ts1 = ts_by_frontend["1200"] -# ts2 = ts_by_frontend["1400"] - -# # Verify first time series -# self.assertEqual(ts1.resource.type, "global") -# self.assertEqual( -# ts1.metric.type, -# "custom.googleapis.com/opencensus/my.org/views/video_size_test2") -# self.assertIsNotNone(ts1.resource) - -# self.assertEqual(len(ts1.points), 1) -# value1 = ts1.points[0].value -# self.assertEqual(value1.distribution_value.count, 1) - -# # Verify second time series -# self.assertEqual(ts2.resource.type, "global") -# self.assertEqual( -# ts2.metric.type, -# "custom.googleapis.com/opencensus/my.org/views/video_size_test2") -# self.assertIsNotNone(ts2.resource) - -# self.assertEqual(len(ts2.points), 1) -# value2 = ts2.points[0].value -# self.assertEqual(value2.distribution_value.count, 1) - -# @mock.patch('opencensus.ext.stackdriver.stats_exporter.' -# 'monitored_resource.get_instance', -# return_value=None) -# def test_create_timeseries_disjoint_tags(self, monitoring_resoure_mock): -# view_manager, stats_recorder, exporter = \ -# self.setup_create_timeseries_test() - -# # Register view with two tags -# view_name = "view-name" -# view = view_module.View(view_name, "test description", -# [FRONTEND_KEY, FRONTEND_KEY_FLOAT], -# VIDEO_SIZE_MEASURE, -# aggregation_module.SumAggregation()) - -# view_manager.register_view(view) - -# # Add point with one tag in common and one different tag -# measure_map = stats_recorder.new_measurement_map() -# tag_map = tag_map_module.TagMap() -# tag_map.insert(FRONTEND_KEY, tag_value_module.TagValue("1200")) -# tag_map.insert(FRONTEND_KEY_STR, tag_value_module.TagValue("1800")) -# measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 25 * MiB) -# measure_map.record(tag_map) - -# v_data = measure_map.measure_to_view_map.get_view(view_name, None) - -# v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) - -# time_series_list = exporter.create_time_series_list(v_data) - -# self.assertEqual(len(time_series_list), 1) -# [time_series] = time_series_list - -# # Verify first time series -# self.assertEqual(time_series.resource.type, "global") -# self.assertEqual(time_series.metric.type, -# "custom.googleapis.com/opencensus/" + view_name) -# self.check_labels( -# time_series.metric.labels, {FRONTEND_KEY_CLEAN: "1200"}, -# include_opencensus=True) -# self.assertIsNotNone(time_series.resource) - -# self.assertEqual(len(time_series.points), 1) -# expected_value = monitoring_v3.types.TypedValue() -# # TODO: #565 -# expected_value.double_value = 25.0 * MiB -# self.assertEqual(time_series.points[0].value, expected_value) - -# def test_create_timeseries_from_distribution(self): -# """Check for explicit 0-bound bucket for SD export.""" -# agg = aggregation_module.DistributionAggregation() - -# view = view_module.View( -# name="example.org/test_view", -# description="example.org/test_view", -# columns=['tag_key'], -# measure=mock.Mock(), -# aggregation=agg, -# ) - -# v_data = view_data_module.ViewData( -# view=view, -# start_time=TEST_TIME_STR, -# end_time=TEST_TIME_STR, -# ) - -# # Aggregation over (10 * range(10)) for buckets [2, 4, 6, 8] -# dad = aggregation_data_module.DistributionAggregationData( -# mean_data=4.5, -# count_data=100, -# sum_of_sqd_deviations=825, -# counts_per_bucket=[20, 20, 20, 20, 20], -# bounds=[2, 4, 6, 8], -# exemplars={mock.Mock() for ii in range(5)} -# ) -# v_data._tag_value_aggregation_data_map = {('tag_value',): dad} - -# v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) - -# exporter = stackdriver.StackdriverStatsExporter() -# time_series_list = exporter.create_time_series_list(v_data) -# self.assertEqual(len(time_series_list), 1) -# [time_series] = time_series_list - -# self.check_labels( -# time_series.metric.labels, {'tag_key': 'tag_value'}, -# include_opencensus=True) -# self.assertEqual(len(time_series.points), 1) -# [point] = time_series.points -# dv = point.value.distribution_value -# self.assertEqual(100, dv.count) -# self.assertEqual(825.0, dv.sum_of_squared_deviation) -# self.assertEqual([0, 20, 20, 20, 20, 20], dv.bucket_counts) -# self.assertEqual([0, 2, 4, 6, 8], -# dv.bucket_options.explicit_buckets.bounds) - -# def test_create_timeseries_multiple_tags(self): -# """Check that exporter creates timeseries for multiple tag values. - -# create_time_series_list should return a time series for each set of -# values in the tag value aggregation map. -# """ -# agg = aggregation_module.CountAggregation() - -# view = view_module.View( -# name="example.org/test_view", -# description="example.org/test_view", -# columns=[tag_key_module.TagKey('color'), -# tag_key_module.TagKey('shape')], -# measure=mock.Mock(), -# aggregation=agg, -# ) - -# v_data = view_data_module.ViewData( -# view=view, -# start_time=TEST_TIME_STR, -# end_time=TEST_TIME_STR, -# ) - -# rs_count = aggregation_data_module.CountAggregationData(10) -# bc_count = aggregation_data_module.CountAggregationData(20) -# v_data._tag_value_aggregation_data_map = { -# ('red', 'square'): rs_count, -# ('blue', 'circle'): bc_count, -# } - -# v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) - -# exporter = stackdriver.StackdriverStatsExporter() -# time_series_list = exporter.create_time_series_list(v_data) - -# self.assertEqual(len(time_series_list), 2) -# self.assertEqual(len(time_series_list[0].points), 1) -# self.assertEqual(len(time_series_list[1].points), 1) - -# ts_by_color = {ts.metric.labels.get('color'): ts -# for ts in time_series_list} -# rs_ts = ts_by_color['red'] -# bc_ts = ts_by_color['blue'] -# self.assertEqual(rs_ts.metric.labels.get('shape'), 'square') -# self.assertEqual(bc_ts.metric.labels.get('shape'), 'circle') -# self.assertEqual(rs_ts.points[0].value.int64_value, 10) -# self.assertEqual(bc_ts.points[0].value.int64_value, 20) - -# def test_create_timeseries_invalid_aggregation(self): -# v_data = mock.Mock(spec=view_data_module.ViewData) -# v_data.view.name = "example.org/base_view" -# v_data.view.columns = [tag_key_module.TagKey('base_key')] -# v_data.start_time = TEST_TIME_STR -# v_data.end_time = TEST_TIME_STR - -# base_data = None -# v_data.tag_value_aggregation_data_map = { -# (None,): base_data, -# } - -# exporter = stackdriver.StackdriverStatsExporter( -# options=mock.Mock(), -# client=mock.Mock(), -# ) -# self.assertRaises(TypeError, exporter.create_time_series_list, v_data, -# "", "") +# Copyright 2018, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from datetime import datetime + +import google.auth +import mock +from google.cloud import monitoring_v3 + +from opencensus.common import utils +from opencensus.common.version import __version__ +from opencensus.ext.stackdriver import stats_exporter as stackdriver +from opencensus.metrics import label_key, label_value +from opencensus.metrics import transport as transport_module +from opencensus.metrics.export import ( + metric, + metric_descriptor, + point, + time_series, + value, +) +from opencensus.stats import aggregation as aggregation_module +from opencensus.stats import aggregation_data as aggregation_data_module +from opencensus.stats import execution_context +from opencensus.stats import measure as measure_module +from opencensus.stats import metric_utils +from opencensus.stats import stats as stats_module +from opencensus.stats import view as view_module +from opencensus.stats import view_data as view_data_module +from opencensus.tags import tag_key as tag_key_module +from opencensus.tags import tag_map as tag_map_module +from opencensus.tags import tag_value as tag_value_module + +MiB = 1 << 20 +FRONTEND_KEY = tag_key_module.TagKey("my.org/keys/frontend") +FRONTEND_KEY_FLOAT = tag_key_module.TagKey("my.org/keys/frontend-FLOAT") +FRONTEND_KEY_INT = tag_key_module.TagKey("my.org/keys/frontend-INT") +FRONTEND_KEY_STR = tag_key_module.TagKey("my.org/keys/frontend-STR") + +FRONTEND_KEY_CLEAN = "my_org_keys_frontend" +FRONTEND_KEY_FLOAT_CLEAN = "my_org_keys_frontend_FLOAT" +FRONTEND_KEY_INT_CLEAN = "my_org_keys_frontend_INT" +FRONTEND_KEY_STR_CLEAN = "my_org_keys_frontend_STR" + +VIDEO_SIZE_MEASURE = measure_module.MeasureFloat( + "my.org/measure/video_size_test2", "size of processed videos", "By") +VIDEO_SIZE_MEASURE_2 = measure_module.MeasureFloat( + "my.org/measure/video_size_test_2", "size of processed videos", "By") + +VIDEO_SIZE_MEASURE_FLOAT = measure_module.MeasureFloat( + "my.org/measure/video_size_test-float", "size of processed videos-float", + "By") + +VIDEO_SIZE_VIEW_NAME = "my.org/views/video_size_test2" +VIDEO_SIZE_DISTRIBUTION = aggregation_module.DistributionAggregation( + [16.0 * MiB, 256.0 * MiB]) +VIDEO_SIZE_VIEW = view_module.View( + VIDEO_SIZE_VIEW_NAME, "processed video size over time", [FRONTEND_KEY], + VIDEO_SIZE_MEASURE, VIDEO_SIZE_DISTRIBUTION) + +TEST_TIME = datetime(2018, 12, 25, 1, 2, 3, 4) +TEST_TIME_STR = utils.to_iso_str(TEST_TIME) + + +class _Client(object): + def __init__(self, client_info=None): + self.client_info = client_info + + +class TestOptions(unittest.TestCase): + def test_options_blank(self): + options = stackdriver.Options() + + self.assertEqual(options.project_id, "") + self.assertEqual(options.resource, "") + + def test_options_parameters(self): + options = stackdriver.Options( + project_id="project-id", metric_prefix="sample") + self.assertEqual(options.project_id, "project-id") + self.assertEqual(options.metric_prefix, "sample") + + def test_default_monitoring_labels(self): + options = stackdriver.Options(default_monitoring_labels={ + label_key.LabelKey('lk_key', 'lk_desc'): + label_value.LabelValue('lk_value') + }) + + self.assertEqual(len(options.default_monitoring_labels), 1) + [[lk, lv]] = options.default_monitoring_labels.items() + self.assertEqual(lk.key, 'lk_key') + self.assertEqual(lk.description, 'lk_desc') + self.assertEqual(lv.value, 'lk_value') + + def test_default_monitoring_labels_blank(self): + with mock.patch('opencensus.ext.stackdriver.stats_exporter' + '.get_task_value') as mock_gtv: + options = stackdriver.Options() + + mock_gtv.assert_called() + + self.assertEqual(len(options.default_monitoring_labels), 1) + [[lk, lv]] = options.default_monitoring_labels.items() + self.assertEqual(lk.key, stackdriver.OPENCENSUS_TASK) + self.assertEqual(lk.description, + stackdriver.OPENCENSUS_TASK_DESCRIPTION) + self.assertEqual(lv.value, mock_gtv()) + + def test_bad_default_monitoring_labels(self): + with self.assertRaises(AttributeError): + stackdriver.Options( + default_monitoring_labels=[ + 'not a dict' + ]) + + with self.assertRaises(TypeError): + stackdriver.Options( + default_monitoring_labels={ + 'bad key': + label_value.LabelValue('clk_value') + }) + + with self.assertRaises(TypeError): + stackdriver.Options( + default_monitoring_labels={ + label_key.LabelKey('clk_key', 'clk_desc'): + 'bad value' + }) + + +class TestStackdriverStatsExporter(unittest.TestCase): + def test_constructor(self): + exporter = stackdriver.StackdriverStatsExporter() + + self.assertIsNone(exporter.client) + + def test_constructor_param(self): + exporter = stackdriver.StackdriverStatsExporter( + options=stackdriver.Options(project_id=1)) + self.assertEqual(exporter.options.project_id, 1) + + def test_null_options(self): + # Check that we don't suppress auth errors + auth_error = google.auth.exceptions.DefaultCredentialsError + mock_auth_error = mock.Mock() + mock_auth_error.side_effect = auth_error + with mock.patch('opencensus.ext.stackdriver.stats_exporter' + '.google.auth.default', mock_auth_error): + with self.assertRaises(auth_error): + stackdriver.new_stats_exporter() + + # Check that we get the default credentials' project ID + mock_auth_ok = mock.Mock() + mock_auth_ok.return_value = (None, 123) + with mock.patch('opencensus.ext.stackdriver.stats_exporter' + '.google.auth.default', mock_auth_ok): + sdse = stackdriver.new_stats_exporter() + self.assertEqual(sdse.options.project_id, 123) + + # Check that we raise if auth works but the project is empty + mock_auth_no_project = mock.Mock() + mock_auth_no_project.return_value = (None, '') + with mock.patch('opencensus.ext.stackdriver.stats_exporter' + '.google.auth.default', mock_auth_no_project): + with self.assertRaises(ValueError): + stackdriver.new_stats_exporter() + + def test_blank_project(self): + self.assertRaises(ValueError, stackdriver.new_stats_exporter, + stackdriver.Options(project_id="")) + + def test_not_blank_project(self): + patch_client = mock.patch( + ('opencensus.ext.stackdriver.stats_exporter' + '.monitoring_v3.MetricServiceClient'), _Client) + + with patch_client: + exporter_created = stackdriver.new_stats_exporter( + stackdriver.Options(project_id=1)) + + self.assertIsInstance(exporter_created, + stackdriver.StackdriverStatsExporter) + + def test_get_user_agent_slug(self): + self.assertIn(__version__, stackdriver.get_user_agent_slug()) + + def test_client_info_user_agent(self): + """Check that the monitoring client sets a user agent. + + The user agent should include the library version. Note that this + assumes MetricServiceClient calls ClientInfo.to_user_agent to attach + the user agent as metadata to metric service API calls. + """ + patch_client = mock.patch( + 'opencensus.ext.stackdriver.stats_exporter.monitoring_v3' + '.MetricServiceClient', _Client) + + with patch_client: + exporter = stackdriver.new_stats_exporter( + stackdriver.Options(project_id=1)) + + self.assertIn(stackdriver.get_user_agent_slug(), + exporter.client.client_info.to_user_agent()) + + def test_sanitize(self): + # empty + result = stackdriver.sanitize_label("") + self.assertEqual(result, "") + + # all invalid + result = stackdriver.sanitize_label("/*^#$") + self.assertEqual(result, "key_") + + # all valid + result = stackdriver.sanitize_label("abc") + self.assertEqual(result, "abc") + + # mixed + result = stackdriver.sanitize_label("a.b/c") + self.assertEqual(result, "a_b_c") + + # starts with '_' + result = stackdriver.sanitize_label("_abc") + self.assertEqual(result, "key_abc") + + # starts with digit + result = stackdriver.sanitize_label("0abc") + self.assertEqual(result, "key_0abc") + + # too long + result = stackdriver.sanitize_label("0123456789" * 10) + self.assertEqual(len(result), 100) + self.assertEqual(result, "key_" + "0123456789" * 9 + "012345") + + def test_get_task_value(self): + task_value = stackdriver.get_task_value() + self.assertNotEqual(task_value, "") + + def test_namespaced_views(self): + view_name = "view-1" + expected_view_name_namespaced = ( + "custom.googleapis.com/opencensus/{}".format(view_name)) + view_name_namespaced = stackdriver.namespaced_view_name(view_name, "") + self.assertEqual(expected_view_name_namespaced, view_name_namespaced) + + expected_view_name_namespaced = "kubernetes.io/myorg/%s" % view_name + view_name_namespaced = stackdriver.namespaced_view_name( + view_name, "kubernetes.io/myorg") + self.assertEqual(expected_view_name_namespaced, view_name_namespaced) + + def test_stackdriver_register_exporter(self): + stats = stats_module.stats + view_manager = stats.view_manager + + exporter = mock.Mock() + if len(view_manager.measure_to_view_map.exporters) > 0: + view_manager.unregister_exporter( + view_manager.measure_to_view_map.exporters[0]) + view_manager.register_exporter(exporter) + + registered_exporters = len(view_manager.measure_to_view_map.exporters) + + self.assertEqual(registered_exporters, 1) + + @mock.patch('os.getpid', return_value=12345) + @mock.patch( + 'platform.uname', + return_value=('system', 'node', 'release', 'version', 'machine', + 'processor')) + def test_get_task_value_with_hostname(self, mock_uname, mock_pid): + self.assertEqual(stackdriver.get_task_value(), "py-12345@node") + + @mock.patch('os.getpid', return_value=12345) + @mock.patch( + 'platform.uname', + return_value=('system', '', 'release', 'version', 'machine', + 'processor')) + def test_get_task_value_without_hostname(self, mock_uname, mock_pid): + self.assertEqual(stackdriver.get_task_value(), "py-12345@localhost") + + def test_default_default_monitoring_labels(self): + """Check that metrics include OC task label by default.""" + exporter = stackdriver.StackdriverStatsExporter( + options=stackdriver.Options(project_id='project_id'), + client=mock.Mock()) + + lv = label_value.LabelValue('val') + val = value.ValueLong(value=123) + dt = datetime(2019, 3, 20, 21, 34, 0, 537954) + pp = point.Point(value=val, timestamp=dt) + ts = [ + time_series.TimeSeries(label_values=[lv], points=[pp], + start_timestamp=utils.to_iso_str(dt)) + ] + + desc = metric_descriptor.MetricDescriptor( + name='name', + description='description', + unit='unit', + type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, + label_keys=[label_key.LabelKey('key', 'description')] + ) + mm = metric.Metric(descriptor=desc, time_series=ts) + + sd_md = exporter.get_metric_descriptor(desc) + self.assertEqual(len(sd_md.labels), 2) + sd_descriptors = {ld.key: ld.description for ld in sd_md.labels} + self.assertIn('key', sd_descriptors) + self.assertEqual(sd_descriptors['key'], 'description') + self.assertIn(stackdriver.OPENCENSUS_TASK, sd_descriptors) + self.assertEqual( + sd_descriptors[stackdriver.OPENCENSUS_TASK], + stackdriver.OPENCENSUS_TASK_DESCRIPTION + ) + + sd_ts_list = exporter.create_time_series_list(mm) + self.assertEqual(len(sd_ts_list), 1) + [sd_ts] = sd_ts_list + self.assertIn('key', sd_ts.metric.labels) + self.assertEqual(sd_ts.metric.labels['key'], 'val') + self.assertIn(stackdriver.OPENCENSUS_TASK, sd_ts.metric.labels) + + def test_empty_default_monitoring_labels(self): + """Check that it's possible to remove the default OC task label.""" + exporter = stackdriver.StackdriverStatsExporter( + options=stackdriver.Options( + project_id='project_id', + default_monitoring_labels={}), + client=mock.Mock()) + + lv = label_value.LabelValue('val') + val = value.ValueLong(value=123) + dt = datetime(2019, 3, 20, 21, 34, 0, 537954) + pp = point.Point(value=val, timestamp=dt) + ts = [ + time_series.TimeSeries(label_values=[lv], points=[pp], + start_timestamp=utils.to_iso_str(dt)) + ] + + desc = metric_descriptor.MetricDescriptor( + name='name', + description='description', + unit='unit', + type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, + label_keys=[label_key.LabelKey('key', 'description')] + ) + mm = metric.Metric(descriptor=desc, time_series=ts) + + sd_md = exporter.get_metric_descriptor(desc) + self.assertEqual(len(sd_md.labels), 1) + [sd_label] = sd_md.labels + self.assertEqual(sd_label.key, 'key') + self.assertEqual(sd_label.description, 'description') + + sd_ts_list = exporter.create_time_series_list(mm) + self.assertEqual(len(sd_ts_list), 1) + [sd_ts] = sd_ts_list + self.assertIn('key', sd_ts.metric.labels) + self.assertEqual(sd_ts.metric.labels['key'], 'val') + self.assertNotIn(stackdriver.OPENCENSUS_TASK, sd_ts.metric.labels) + + def test_custom_default_monitoring_labels(self): + """Check that custom labels are exported and included in descriptor.""" + exporter = stackdriver.StackdriverStatsExporter( + options=stackdriver.Options( + project_id='project_id', + default_monitoring_labels={ + label_key.LabelKey('clk_key', 'clk_desc'): + label_value.LabelValue('clk_value') + }), + client=mock.Mock()) + + lv = label_value.LabelValue('val') + val = value.ValueLong(value=123) + dt = datetime(2019, 3, 20, 21, 34, 0, 537954) + pp = point.Point(value=val, timestamp=dt) + ts = [ + time_series.TimeSeries(label_values=[lv], points=[pp], + start_timestamp=utils.to_iso_str(dt)) + ] + + desc = metric_descriptor.MetricDescriptor( + name='name', + description='description', + unit='unit', + type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, + label_keys=[label_key.LabelKey('key', 'description')] + ) + mm = metric.Metric(descriptor=desc, time_series=ts) + + sd_md = exporter.get_metric_descriptor(desc) + self.assertEqual(len(sd_md.labels), 2) + sd_descriptors = {ld.key: ld.description for ld in sd_md.labels} + self.assertIn('key', sd_descriptors) + self.assertEqual(sd_descriptors['key'], 'description') + self.assertIn('clk_key', sd_descriptors) + self.assertEqual(sd_descriptors['clk_key'], 'clk_desc') + + sd_ts_list = exporter.create_time_series_list(mm) + self.assertEqual(len(sd_ts_list), 1) + [sd_ts] = sd_ts_list + self.assertIn('key', sd_ts.metric.labels) + self.assertEqual(sd_ts.metric.labels['key'], 'val') + self.assertIn('clk_key', sd_ts.metric.labels) + self.assertEqual(sd_ts.metric.labels['clk_key'], 'clk_value') + + def test_get_metric_descriptor(self): + exporter = stackdriver.StackdriverStatsExporter( + options=stackdriver.Options( + project_id='project_id'), + client=mock.Mock()) + + oc_md = metric_descriptor.MetricDescriptor( + name='name', + description='description', + unit='unit', + type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, + label_keys=[label_key.LabelKey('ck', 'cd')] + ) + + sd_md = exporter.get_metric_descriptor(oc_md) + self.assertEqual( + sd_md.metric_kind, + monitoring_v3.enums.MetricDescriptor.MetricKind.GAUGE) + self.assertEqual( + sd_md.value_type, + monitoring_v3.enums.MetricDescriptor.ValueType.INT64) + + self.assertIsInstance(sd_md, monitoring_v3.types.MetricDescriptor) + exporter.client.create_metric_descriptor.assert_not_called() + + def test_get_metric_descriptor_bad_type(self): + exporter = stackdriver.StackdriverStatsExporter( + options=stackdriver.Options(project_id='project_id'), + client=mock.Mock()) + + bad_type_oc_md = metric_descriptor.MetricDescriptor( + name='name', + description='description', + unit='unit', + # Need a valid type to create the descriptor + type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, + label_keys=[label_key.LabelKey('key', 'description')] + ) + bad_type_oc_md._type = 100 + + with self.assertRaises(TypeError): + exporter.get_metric_descriptor(bad_type_oc_md) + + def test_get_metric_descriptor_custom_prefix(self): + + exporter = stackdriver.StackdriverStatsExporter( + options=stackdriver.Options( + metric_prefix='metric_prefix', + project_id='project_id'), + client=mock.Mock()) + + oc_md = metric_descriptor.MetricDescriptor( + name='name', + description='description', + unit='unit', + type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, + label_keys=[label_key.LabelKey('ck', 'cd')] + ) + + sd_md = exporter.get_metric_descriptor(oc_md) + self.assertIn('metric_prefix', sd_md.type) + self.assertIn('metric_prefix', sd_md.name) + + def test_register_metric_descriptor(self): + exporter = stackdriver.StackdriverStatsExporter( + options=stackdriver.Options( + metric_prefix='metric_prefix', + project_id='project_id'), + client=mock.Mock()) + + oc_md = metric_descriptor.MetricDescriptor( + name='name', + description='description', + unit='unit', + type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, + label_keys=[label_key.LabelKey('key', 'description')] + ) + + exporter.register_metric_descriptor(oc_md) + self.assertEqual( + exporter.client.create_metric_descriptor.call_count, + 1 + ) + exporter.register_metric_descriptor(oc_md) + self.assertEqual( + exporter.client.create_metric_descriptor.call_count, + 1 + ) + + def test_export_metrics(self): + lv = label_value.LabelValue('val') + val = value.ValueLong(value=123) + dt = datetime(2019, 3, 20, 21, 34, 0, 537954) + pp = point.Point(value=val, timestamp=dt) + + ts = [ + time_series.TimeSeries(label_values=[lv], points=[pp], + start_timestamp=utils.to_iso_str(dt)) + ] + + desc = metric_descriptor.MetricDescriptor( + name='name', + description='description', + unit='unit', + type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, + label_keys=[label_key.LabelKey('key', 'description')] + ) + + mm = metric.Metric(descriptor=desc, time_series=ts) + + exporter = stackdriver.StackdriverStatsExporter(client=mock.Mock()) + exporter.export_metrics([mm]) + + self.assertEqual(exporter.client.create_time_series.call_count, 1) + sd_args = exporter.client.create_time_series.call_args[0][1] + self.assertEqual(len(sd_args), 1) + [sd_arg] = exporter.client.create_time_series.call_args[0][1] + self.assertEqual(sd_arg.points[0].value.int64_value, 123) + + +class MockPeriodicMetricTask(object): + """Testing mock of metrics.transport.PeriodicMetricTask. + + Simulate calling export asynchronously from another thread synchronously + from this one. + """ + def __init__(self, interval=None, function=None, args=None, kwargs=None): + self.function = function + self.logger = mock.Mock() + self.start = mock.Mock() + self.run = mock.Mock() + + def step(self): + try: + self.function() + except transport_module.TransportError as ex: + self.logger.exception(ex) + self.cancel() + except Exception: + self.logger.exception("Error handling metric export") + + +class MockGetExporterThread(object): + """Intercept calls to get_exporter_thread. + + To get a reference to the running PeriodicMetricTask created by + get_exporter_thread. + """ + def __init__(self): + self.transport = None + + def __enter__(self): + original_func = transport_module.get_exporter_thread + + def get_exporter_thread(*aa, **kw): + self.transport = original_func(*aa, **kw) + + mock_get = mock.Mock() + mock_get.side_effect = get_exporter_thread + self.patcher = mock.patch( + ('opencensus.ext.stackdriver.stats_exporter' + '.transport.get_exporter_thread'), + mock_get) + self.patcher.start() + return self + + def __exit__(self, type, value, traceback): + self.patcher.stop() + + +@mock.patch('opencensus.ext.stackdriver.stats_exporter' + '.monitoring_v3.MetricServiceClient') +@mock.patch('opencensus.ext.stackdriver.stats_exporter' + '.stats.stats') +class TestAsyncStatsExport(unittest.TestCase): + """Check that metrics are exported using the exporter thread.""" + + def setUp(self): + patcher = mock.patch( + 'opencensus.metrics.transport.PeriodicMetricTask', + MockPeriodicMetricTask) + patcher.start() + self.addCleanup(patcher.stop) + + def test_export_empty(self, mock_stats, mock_client): + """Check that we don't attempt to export empty metric sets.""" + + mock_stats.get_metrics.return_value = [] + + with MockGetExporterThread() as mget: + exporter = stackdriver.new_stats_exporter( + stackdriver.Options(project_id=1)) + mget.transport.step() + + exporter.client.create_metric_descriptor.assert_not_called() + exporter.client.create_time_series.assert_not_called() + + def test_export_single_metric(self, mock_stats, mock_client): + """Check that we can export a set of a single metric.""" + + lv = label_value.LabelValue('val') + val = value.ValueLong(value=123) + dt = datetime(2019, 3, 20, 21, 34, 0, 537954) + pp = point.Point(value=val, timestamp=dt) + + ts = [ + time_series.TimeSeries(label_values=[lv], points=[pp], + start_timestamp=utils.to_iso_str(dt)) + ] + + desc = metric_descriptor.MetricDescriptor( + name='name2', + description='description2', + unit='unit2', + type_=metric_descriptor.MetricDescriptorType.GAUGE_INT64, + label_keys=[label_key.LabelKey('key', 'description')] + ) + + mm = metric.Metric(descriptor=desc, time_series=ts) + mock_stats.get_metrics.return_value = [mm] + + with MockGetExporterThread() as mget: + exporter = stackdriver.new_stats_exporter( + stackdriver.Options(project_id=1)) + mget.transport.step() + + exporter.client.create_metric_descriptor.assert_called() + self.assertEqual( + exporter.client.create_metric_descriptor.call_count, + 1) + md_call_arg =\ + exporter.client.create_metric_descriptor.call_args[0][1] + self.assertEqual( + md_call_arg.metric_kind, + monitoring_v3.enums.MetricDescriptor.MetricKind.GAUGE + ) + self.assertEqual( + md_call_arg.value_type, + monitoring_v3.enums.MetricDescriptor.ValueType.INT64 + ) + + exporter.client.create_time_series.assert_called() + self.assertEqual( + exporter.client.create_time_series.call_count, + 1) + ts_call_arg = exporter.client.create_time_series.call_args[0][1] + self.assertEqual(len(ts_call_arg), 1) + self.assertEqual(len(ts_call_arg[0].points), 1) + self.assertEqual(ts_call_arg[0].points[0].value.int64_value, 123) + + +class TestCreateTimeseries(unittest.TestCase): + + def setUp(self): + patcher = mock.patch( + 'opencensus.ext.stackdriver.stats_exporter.stats.stats', + stats_module._Stats()) + patcher.start() + self.addCleanup(patcher.stop) + + def check_labels(self, + actual_labels, + expected_labels, + include_opencensus=False): + actual_labels = dict(actual_labels) + if include_opencensus: + opencensus_tag = actual_labels.pop(stackdriver.OPENCENSUS_TASK) + self.assertIsNotNone(opencensus_tag) + self.assertIn("py-", opencensus_tag) + self.assertDictEqual(actual_labels, expected_labels) + + @mock.patch('opencensus.ext.stackdriver.stats_exporter.' + 'monitored_resource.get_instance', + return_value=None) + def test_create_batched_time_series(self, monitor_resource_mock): + client = mock.Mock() + v_data = view_data_module.ViewData( + view=VIDEO_SIZE_VIEW, + start_time=TEST_TIME_STR, + end_time=TEST_TIME_STR) + v_data.record(context=tag_map_module.TagMap(), value=2, timestamp=None) + view_data = [v_data] + + option = stackdriver.Options(project_id="project-test") + exporter = stackdriver.StackdriverStatsExporter( + options=option, client=client) + + view_data = [metric_utils.view_data_to_metric(view_data[0], TEST_TIME)] + + time_series_batches = exporter.create_batched_time_series(view_data, 1) + + self.assertEqual(len(time_series_batches), 1) + [time_series_batch] = time_series_batches + self.assertEqual(len(time_series_batch), 1) + [time_series] = time_series_batch + self.assertEqual( + time_series.metric.type, + 'custom.googleapis.com/opencensus/' + VIDEO_SIZE_VIEW_NAME) + self.check_labels( + time_series.metric.labels, {}, include_opencensus=True) + + @mock.patch('opencensus.ext.stackdriver.stats_exporter.' + 'monitored_resource.get_instance', + return_value=None) + def test_create_batched_time_series_with_many(self, monitor_resource_mock): + client = mock.Mock() + + # First view with 3 + view_name1 = "view-name1" + view1 = view_module.View(view_name1, "test description", ['test'], + VIDEO_SIZE_MEASURE, + aggregation_module.LastValueAggregation()) + v_data1 = view_data_module.ViewData( + view=view1, start_time=TEST_TIME_STR, end_time=TEST_TIME_STR) + v_data1.record(context=tag_map_module.TagMap({'test': '1'}), value=7, + timestamp=None) + v_data1.record(context=tag_map_module.TagMap({'test': '2'}), value=5, + timestamp=None) + v_data1.record(context=tag_map_module.TagMap({'test': '3'}), value=3, + timestamp=None) + + # Second view with 2 + view_name2 = "view-name2" + view2 = view_module.View(view_name2, "test description", ['test'], + VIDEO_SIZE_MEASURE, + aggregation_module.LastValueAggregation()) + v_data2 = view_data_module.ViewData( + view=view2, start_time=TEST_TIME_STR, end_time=TEST_TIME_STR) + v_data2.record(context=tag_map_module.TagMap({'test': '1'}), value=7, + timestamp=None) + v_data2.record(context=tag_map_module.TagMap({'test': '2'}), value=5, + timestamp=None) + + view_data = [v_data1, v_data2] + view_data = [metric_utils.view_data_to_metric(vd, TEST_TIME) + for vd in view_data] + + option = stackdriver.Options(project_id="project-test") + exporter = stackdriver.StackdriverStatsExporter( + options=option, client=client) + + time_series_batches = exporter.create_batched_time_series(view_data, 2) + + self.assertEqual(len(time_series_batches), 3) + [tsb1, tsb2, tsb3] = time_series_batches + self.assertEqual(len(tsb1), 2) + self.assertEqual(len(tsb2), 2) + self.assertEqual(len(tsb3), 1) + + def setup_create_timeseries_test(self): + client = mock.Mock() + execution_context.clear() + + option = stackdriver.Options( + project_id="project-test", resource="global") + exporter = stackdriver.StackdriverStatsExporter( + options=option, client=client) + + stats = stats_module.stats + view_manager = stats.view_manager + stats_recorder = stats.stats_recorder + + if len(view_manager.measure_to_view_map.exporters) > 0: + view_manager.unregister_exporter( + view_manager.measure_to_view_map.exporters[0]) + + view_manager.register_exporter(exporter) + return view_manager, stats_recorder, exporter + + @mock.patch('opencensus.ext.stackdriver.stats_exporter.' + 'monitored_resource.get_instance', + return_value=None) + def test_create_timeseries(self, monitor_resource_mock): + view_manager, stats_recorder, exporter = \ + self.setup_create_timeseries_test() + + view_manager.register_view(VIDEO_SIZE_VIEW) + + tag_value = tag_value_module.TagValue("1200") + tag_map = tag_map_module.TagMap() + tag_map.insert(FRONTEND_KEY, tag_value) + + measure_map = stats_recorder.new_measurement_map() + measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 25 * MiB) + measure_map.record(tag_map) + + v_data = measure_map.measure_to_view_map.get_view( + VIDEO_SIZE_VIEW_NAME, None) + + v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) + + time_series_list = exporter.create_time_series_list(v_data) + + self.assertEqual(len(time_series_list), 1) + time_series = time_series_list[0] + self.assertEqual(time_series.resource.type, "global") + self.assertEqual( + time_series_list[0].metric.type, + "custom.googleapis.com/opencensus/my.org/views/video_size_test2") + self.check_labels( + time_series.metric.labels, {FRONTEND_KEY_CLEAN: "1200"}, + include_opencensus=True) + self.assertIsNotNone(time_series.resource) + + self.assertEqual(len(time_series.points), 1) + value = time_series.points[0].value + + expected_distb = google.api.distribution_pb2.Distribution( + count=1, + mean=26214400.0, + bucket_options=google.api.distribution_pb2.Distribution.BucketOptions( # noqa + explicit_buckets=google.api.distribution_pb2.Distribution.BucketOptions.Explicit( # noqa + bounds=[0.0, 16777216.0, 268435456.0])), + bucket_counts=[0, 0, 1, 0] + ) + self.assertEqual(value.distribution_value, expected_distb) + + @mock.patch('opencensus.ext.stackdriver.stats_exporter.' + 'monitored_resource.get_instance') + def test_create_timeseries_with_resource(self, monitor_resource_mock): + + client = mock.Mock() + execution_context.clear() + + option = stackdriver.Options(project_id="project-test", resource="") + exporter = stackdriver.StackdriverStatsExporter( + options=option, client=client) + + stats = stats_module.stats + view_manager = stats.view_manager + stats_recorder = stats.stats_recorder + + if len(view_manager.measure_to_view_map.exporters) > 0: + view_manager.unregister_exporter( + view_manager.measure_to_view_map.exporters[0]) + + view_manager.register_exporter(exporter) + view_manager.register_view(VIDEO_SIZE_VIEW) + + tag_value = tag_value_module.TagValue("1200") + tag_map = tag_map_module.TagMap() + tag_map.insert(FRONTEND_KEY, tag_value) + + measure_map = stats_recorder.new_measurement_map() + measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 25 * MiB) + measure_map.record(tag_map) + + v_data = measure_map.measure_to_view_map.get_view( + VIDEO_SIZE_VIEW_NAME, None) + + v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) + + # check for gce_instance monitored resource + mocked_labels = { + 'instance_id': 'my-instance', + 'project_id': 'my-project', + 'zone': 'us-east1', + 'k8s.io/pod/name': 'localhost', + 'k8s.io/namespace/name': 'namespace', + } + + mock_resource = mock.Mock() + mock_resource.get_type.return_value = 'gce_instance' + mock_resource.get_labels.return_value = mocked_labels + monitor_resource_mock.return_value = mock_resource + + time_series_list = exporter.create_time_series_list(v_data) + self.assertEqual(len(time_series_list), 1) + time_series = time_series_list[0] + self.assertEqual(time_series.resource.type, "gce_instance") + self.check_labels( + time_series.resource.labels, { + 'instance_id': 'my-instance', + 'project_id': 'my-project', + 'zone': 'us-east1', + }) + self.assertEqual( + time_series.metric.type, + "custom.googleapis.com/opencensus/my.org/views/video_size_test2") + self.assertIsNotNone(time_series) + + time_series_list = exporter.create_time_series_list(v_data) + self.assertEqual(len(time_series_list), 1) + time_series = time_series_list[0] + + self.assertEqual( + time_series.metric.type, + "custom.googleapis.com/opencensus/my.org/views/video_size_test2") + + # check for k8s_container monitored resource + mocked_labels = { + 'instance_id': 'my-instance', + 'project_id': 'my-project', + 'zone': 'us-east1', + 'k8s.io/pod/name': 'localhost', + 'k8s.io/cluster/name': 'cluster', + 'k8s.io/namespace/name': 'namespace', + } + + mock_resource = mock.Mock() + mock_resource.get_type.return_value = 'k8s_container' + mock_resource.get_labels.return_value = mocked_labels + monitor_resource_mock.return_value = mock_resource + + time_series_list = exporter.create_time_series_list(v_data) + self.assertEqual(len(time_series_list), 1) + time_series = time_series_list[0] + self.assertEqual(time_series.resource.type, "k8s_container") + self.check_labels( + time_series.resource.labels, { + 'project_id': 'my-project', + 'location': 'us-east1', + 'cluster_name': 'cluster', + 'pod_name': 'localhost', + 'namespace_name': 'namespace', + }) + self.assertEqual( + time_series.metric.type, + "custom.googleapis.com/opencensus/my.org/views/video_size_test2") + self.assertIsNotNone(time_series) + + # check for aws_ec2_instance monitored resource + mocked_labels = { + 'instance_id': 'my-instance', + 'aws_account': 'my-project', + 'region': 'us-east1', + } + + mock_resource = mock.Mock() + mock_resource.get_type.return_value = 'aws_ec2_instance' + mock_resource.get_labels.return_value = mocked_labels + monitor_resource_mock.return_value = mock_resource + + time_series_list = exporter.create_time_series_list(v_data) + self.assertEqual(len(time_series_list), 1) + time_series = time_series_list[0] + self.assertEqual(time_series.resource.type, "aws_ec2_instance") + self.check_labels( + time_series.resource.labels, { + 'instance_id': 'my-instance', + 'aws_account': 'my-project', + 'region': 'aws:us-east1', + }) + self.assertEqual( + time_series.metric.type, + "custom.googleapis.com/opencensus/my.org/views/video_size_test2") + self.assertIsNotNone(time_series) + + # check for out of box monitored resource + mock_resource = mock.Mock() + mock_resource.get_type.return_value = '' + mock_resource.get_labels.return_value = mock.Mock() + monitor_resource_mock.return_value = mock_resource + + time_series_list = exporter.create_time_series_list(v_data) + self.assertEqual(len(time_series_list), 1) + time_series = time_series_list[0] + self.assertEqual(time_series.resource.type, 'global') + self.check_labels(time_series.resource.labels, {}) + self.assertEqual( + time_series.metric.type, + "custom.googleapis.com/opencensus/my.org/views/video_size_test2") + self.assertIsNotNone(time_series) + + @mock.patch('opencensus.ext.stackdriver.stats_exporter.' + 'monitored_resource.get_instance', + return_value=None) + def test_create_timeseries_str_tagvalue(self, monitor_resource_mock): + view_manager, stats_recorder, exporter = \ + self.setup_create_timeseries_test() + + agg_1 = aggregation_module.LastValueAggregation(value=2) + view_name1 = "view-name1" + new_view1 = view_module.View( + view_name1, "processed video size over time", [FRONTEND_KEY_INT], + VIDEO_SIZE_MEASURE_2, agg_1) + + view_manager.register_view(new_view1) + + tag_value_int = tag_value_module.TagValue("Abc") + tag_map = tag_map_module.TagMap() + tag_map.insert(FRONTEND_KEY_INT, tag_value_int) + + measure_map = stats_recorder.new_measurement_map() + measure_map.measure_int_put(VIDEO_SIZE_MEASURE_2, 25 * MiB) + measure_map.record(tag_map) + + v_data = measure_map.measure_to_view_map.get_view(view_name1, None) + + v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) + + time_series_list = exporter.create_time_series_list(v_data) + self.assertEqual(len(time_series_list), 1) + time_series = time_series_list[0] + + self.check_labels( + time_series.metric.labels, {FRONTEND_KEY_INT_CLEAN: "Abc"}, + include_opencensus=True) + self.assertIsNotNone(time_series.resource) + + self.assertEqual(len(time_series.points), 1) + expected_value = monitoring_v3.types.TypedValue() + # TODO: #565 + expected_value.double_value = 25.0 * MiB + self.assertEqual(time_series.points[0].value, expected_value) + + @mock.patch('opencensus.ext.stackdriver.stats_exporter.' + 'monitored_resource.get_instance', + return_value=None) + def test_create_timeseries_str_tagvalue_count_aggregtation( + self, monitor_resource_mock): + view_manager, stats_recorder, exporter = \ + self.setup_create_timeseries_test() + + agg_1 = aggregation_module.CountAggregation(count=2) + view_name1 = "view-name1" + new_view1 = view_module.View( + view_name1, "processed video size over time", [FRONTEND_KEY_INT], + VIDEO_SIZE_MEASURE_2, agg_1) + + view_manager.register_view(new_view1) + + tag_value_int = tag_value_module.TagValue("Abc") + tag_map = tag_map_module.TagMap() + tag_map.insert(FRONTEND_KEY_INT, tag_value_int) + + measure_map = stats_recorder.new_measurement_map() + measure_map.measure_int_put(VIDEO_SIZE_MEASURE_2, 25 * MiB) + measure_map.record(tag_map) + + v_data = measure_map.measure_to_view_map.get_view(view_name1, None) + + v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) + + time_series_list = exporter.create_time_series_list(v_data) + self.assertEqual(len(time_series_list), 1) + time_series = time_series_list[0] + self.check_labels( + time_series.metric.labels, {FRONTEND_KEY_INT_CLEAN: "Abc"}, + include_opencensus=True) + self.assertIsNotNone(time_series.resource) + + self.assertEqual(len(time_series.points), 1) + expected_value = monitoring_v3.types.TypedValue() + expected_value.int64_value = 3 + self.assertEqual(time_series.points[0].value, expected_value) + + @mock.patch('opencensus.ext.stackdriver.stats_exporter.' + 'monitored_resource.get_instance', + return_value=None) + def test_create_timeseries_last_value_float_tagvalue( + self, monitor_resource_mock): + view_manager, stats_recorder, exporter = \ + self.setup_create_timeseries_test() + + agg_2 = aggregation_module.LastValueAggregation(value=2.2 * MiB) + view_name2 = "view-name2" + new_view2 = view_module.View( + view_name2, "processed video size over time", [FRONTEND_KEY_FLOAT], + VIDEO_SIZE_MEASURE_FLOAT, agg_2) + + view_manager.register_view(new_view2) + + tag_value_float = tag_value_module.TagValue("Abc") + tag_map = tag_map_module.TagMap() + tag_map.insert(FRONTEND_KEY_FLOAT, tag_value_float) + + measure_map = stats_recorder.new_measurement_map() + measure_map.measure_float_put(VIDEO_SIZE_MEASURE_FLOAT, 25.7 * MiB) + measure_map.record(tag_map) + + v_data = measure_map.measure_to_view_map.get_view(view_name2, None) + + v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) + + time_series_list = exporter.create_time_series_list(v_data) + self.assertEqual(len(time_series_list), 1) + time_series = time_series_list[0] + self.check_labels( + time_series.metric.labels, {FRONTEND_KEY_FLOAT_CLEAN: "Abc"}, + include_opencensus=True) + self.assertIsNotNone(time_series.resource) + + self.assertEqual(len(time_series.points), 1) + expected_value = monitoring_v3.types.TypedValue() + expected_value.double_value = 25.7 * MiB + self.assertEqual(time_series.points[0].value, expected_value) + + @mock.patch('opencensus.ext.stackdriver.stats_exporter.' + 'monitored_resource.get_instance', + return_value=None) + def test_create_timeseries_float_tagvalue(self, monitor_resource_mock): + client = mock.Mock() + + option = stackdriver.Options( + project_id="project-test", resource="global") + exporter = stackdriver.StackdriverStatsExporter( + options=option, client=client) + + stats = stats_module.stats + view_manager = stats.view_manager + stats_recorder = stats.stats_recorder + + if len(view_manager.measure_to_view_map.exporters) > 0: + view_manager.unregister_exporter( + view_manager.measure_to_view_map.exporters[0]) + + view_manager.register_exporter(exporter) + + agg_3 = aggregation_module.SumAggregation(sum=2.2) + view_name3 = "view-name3" + new_view3 = view_module.View( + view_name3, "processed video size over time", [FRONTEND_KEY_FLOAT], + VIDEO_SIZE_MEASURE_FLOAT, agg_3) + + view_manager.register_view(new_view3) + + tag_value_float = tag_value_module.TagValue("1200") + tag_map = tag_map_module.TagMap() + tag_map.insert(FRONTEND_KEY_FLOAT, tag_value_float) + + measure_map = stats_recorder.new_measurement_map() + measure_map.measure_float_put(VIDEO_SIZE_MEASURE_FLOAT, 25 * MiB) + measure_map.record(tag_map) + + v_data = measure_map.measure_to_view_map.get_view(view_name3, None) + + v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) + + time_series_list = exporter.create_time_series_list(v_data) + self.assertEqual(len(time_series_list), 1) + [time_series] = time_series_list + self.assertEqual(time_series.metric.type, + "custom.googleapis.com/opencensus/view-name3") + self.check_labels( + time_series.metric.labels, {FRONTEND_KEY_FLOAT_CLEAN: "1200"}, + include_opencensus=True) + self.assertIsNotNone(time_series.resource) + + self.assertEqual(len(time_series.points), 1) + expected_value = monitoring_v3.types.TypedValue() + expected_value.double_value = 2.2 + 25 * MiB + self.assertEqual(time_series.points[0].value, expected_value) + + @mock.patch('opencensus.ext.stackdriver.stats_exporter.' + 'monitored_resource.get_instance', + return_value=None) + def test_create_timeseries_multiple_tag_values(self, + monitoring_resoure_mock): + view_manager, stats_recorder, exporter = \ + self.setup_create_timeseries_test() + + view_manager.register_view(VIDEO_SIZE_VIEW) + + measure_map = stats_recorder.new_measurement_map() + + # Add first point with one tag value + tag_map = tag_map_module.TagMap() + tag_map.insert(FRONTEND_KEY, tag_value_module.TagValue("1200")) + measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 25 * MiB) + measure_map.record(tag_map) + + # Add second point with different tag value + tag_map = tag_map_module.TagMap() + tag_map.insert(FRONTEND_KEY, tag_value_module.TagValue("1400")) + measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 12 * MiB) + measure_map.record(tag_map) + + v_data = measure_map.measure_to_view_map.get_view( + VIDEO_SIZE_VIEW_NAME, None) + + v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) + + time_series_list = exporter.create_time_series_list(v_data) + + self.assertEqual(len(time_series_list), 2) + ts_by_frontend = { + ts.metric.labels.get(FRONTEND_KEY_CLEAN): ts + for ts in time_series_list + } + self.assertEqual(set(ts_by_frontend.keys()), {"1200", "1400"}) + ts1 = ts_by_frontend["1200"] + ts2 = ts_by_frontend["1400"] + + # Verify first time series + self.assertEqual(ts1.resource.type, "global") + self.assertEqual( + ts1.metric.type, + "custom.googleapis.com/opencensus/my.org/views/video_size_test2") + self.assertIsNotNone(ts1.resource) + + self.assertEqual(len(ts1.points), 1) + value1 = ts1.points[0].value + + expected_distb = google.api.distribution_pb2.Distribution( + count=1, + mean=26214400.0, + bucket_options=google.api.distribution_pb2.Distribution.BucketOptions( # noqa + explicit_buckets=google.api.distribution_pb2.Distribution.BucketOptions.Explicit( # noqa + bounds=[0.0, 16777216.0, 268435456.0])), + bucket_counts=[0, 0, 1, 0] + ) + self.assertEqual(value1.distribution_value, expected_distb) + + # Verify second time series + self.assertEqual(ts2.resource.type, "global") + self.assertEqual( + ts2.metric.type, + "custom.googleapis.com/opencensus/my.org/views/video_size_test2") + self.assertIsNotNone(ts2.resource) + + self.assertEqual(len(ts2.points), 1) + value2 = ts2.points[0].value + + expected_distb = google.api.distribution_pb2.Distribution( + count=1, + mean=12582912.0, + bucket_options=google.api.distribution_pb2.Distribution.BucketOptions( # noqa + explicit_buckets=google.api.distribution_pb2.Distribution.BucketOptions.Explicit( # noqa + bounds=[0.0, 16777216.0, 268435456.0])), + bucket_counts=[0, 1, 0, 0] + ) + self.assertEqual(value2.distribution_value, expected_distb) + + @mock.patch('opencensus.ext.stackdriver.stats_exporter.' + 'monitored_resource.get_instance', + return_value=None) + def test_create_timeseries_disjoint_tags(self, monitoring_resoure_mock): + view_manager, stats_recorder, exporter = \ + self.setup_create_timeseries_test() + + # Register view with two tags + view_name = "view-name" + view = view_module.View(view_name, "test description", + [FRONTEND_KEY, FRONTEND_KEY_FLOAT], + VIDEO_SIZE_MEASURE, + aggregation_module.SumAggregation()) + + view_manager.register_view(view) + + # Add point with one tag in common and one different tag + measure_map = stats_recorder.new_measurement_map() + tag_map = tag_map_module.TagMap() + tag_map.insert(FRONTEND_KEY, tag_value_module.TagValue("1200")) + tag_map.insert(FRONTEND_KEY_STR, tag_value_module.TagValue("1800")) + measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 25 * MiB) + measure_map.record(tag_map) + + v_data = measure_map.measure_to_view_map.get_view(view_name, None) + + v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) + + time_series_list = exporter.create_time_series_list(v_data) + + self.assertEqual(len(time_series_list), 1) + [time_series] = time_series_list + + # Verify first time series + self.assertEqual(time_series.resource.type, "global") + self.assertEqual(time_series.metric.type, + "custom.googleapis.com/opencensus/" + view_name) + self.check_labels( + time_series.metric.labels, {FRONTEND_KEY_CLEAN: "1200"}, + include_opencensus=True) + self.assertIsNotNone(time_series.resource) + + self.assertEqual(len(time_series.points), 1) + expected_value = monitoring_v3.types.TypedValue() + # TODO: #565 + expected_value.double_value = 25.0 * MiB + self.assertEqual(time_series.points[0].value, expected_value) + + def test_create_timeseries_from_distribution(self): + """Check for explicit 0-bound bucket for SD export.""" + agg = aggregation_module.DistributionAggregation() + + view = view_module.View( + name="example.org/test_view", + description="example.org/test_view", + columns=['tag_key'], + measure=mock.Mock(), + aggregation=agg, + ) + + v_data = view_data_module.ViewData( + view=view, + start_time=TEST_TIME_STR, + end_time=TEST_TIME_STR, + ) + + # Aggregation over (10 * range(10)) for buckets [2, 4, 6, 8] + dad = aggregation_data_module.DistributionAggregationData( + mean_data=4.5, + count_data=100, + sum_of_sqd_deviations=825, + counts_per_bucket=[20, 20, 20, 20, 20], + bounds=[2, 4, 6, 8], + exemplars={mock.Mock() for ii in range(5)} + ) + v_data._tag_value_aggregation_data_map = {('tag_value',): dad} + + v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) + + exporter = stackdriver.StackdriverStatsExporter() + time_series_list = exporter.create_time_series_list(v_data) + self.assertEqual(len(time_series_list), 1) + [time_series] = time_series_list + + self.check_labels( + time_series.metric.labels, {'tag_key': 'tag_value'}, + include_opencensus=True) + self.assertEqual(len(time_series.points), 1) + [point] = time_series.points + + dv = point.value.distribution_value + expected_distb = google.api.distribution_pb2.Distribution( + count=100, + mean=4.5, + sum_of_squared_deviation=825.0, + bucket_options=google.api.distribution_pb2.Distribution.BucketOptions( # noqa + explicit_buckets=google.api.distribution_pb2.Distribution.BucketOptions.Explicit( # noqa + bounds=[0, 2, 4, 6, 8])), + bucket_counts=[0, 20, 20, 20, 20, 20] + ) + self.assertEqual(dv, expected_distb) + + def test_create_timeseries_multiple_tags(self): + """Check that exporter creates timeseries for multiple tag values. + + create_time_series_list should return a time series for each set of + values in the tag value aggregation map. + """ + agg = aggregation_module.CountAggregation() + + view = view_module.View( + name="example.org/test_view", + description="example.org/test_view", + columns=[tag_key_module.TagKey('color'), + tag_key_module.TagKey('shape')], + measure=mock.Mock(), + aggregation=agg, + ) + + v_data = view_data_module.ViewData( + view=view, + start_time=TEST_TIME_STR, + end_time=TEST_TIME_STR, + ) + + rs_count = aggregation_data_module.CountAggregationData(10) + bc_count = aggregation_data_module.CountAggregationData(20) + v_data._tag_value_aggregation_data_map = { + ('red', 'square'): rs_count, + ('blue', 'circle'): bc_count, + } + + v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) + + exporter = stackdriver.StackdriverStatsExporter() + time_series_list = exporter.create_time_series_list(v_data) + + self.assertEqual(len(time_series_list), 2) + self.assertEqual(len(time_series_list[0].points), 1) + self.assertEqual(len(time_series_list[1].points), 1) + + ts_by_color = {ts.metric.labels.get('color'): ts + for ts in time_series_list} + rs_ts = ts_by_color['red'] + bc_ts = ts_by_color['blue'] + self.assertEqual(rs_ts.metric.labels.get('shape'), 'square') + self.assertEqual(bc_ts.metric.labels.get('shape'), 'circle') + self.assertEqual(rs_ts.points[0].value.int64_value, 10) + self.assertEqual(bc_ts.points[0].value.int64_value, 20) + + def test_create_timeseries_invalid_aggregation(self): + v_data = mock.Mock(spec=view_data_module.ViewData) + v_data.view.name = "example.org/base_view" + v_data.view.columns = [tag_key_module.TagKey('base_key')] + v_data.start_time = TEST_TIME_STR + v_data.end_time = TEST_TIME_STR + + base_data = None + v_data.tag_value_aggregation_data_map = { + (None,): base_data, + } + + exporter = stackdriver.StackdriverStatsExporter( + options=mock.Mock(), + client=mock.Mock(), + ) + self.assertRaises(TypeError, exporter.create_time_series_list, v_data, + "", "") diff --git a/contrib/opencensus-ext-stackdriver/version.py b/contrib/opencensus-ext-stackdriver/version.py index c652125f7..b7a1f8944 100644 --- a/contrib/opencensus-ext-stackdriver/version.py +++ b/contrib/opencensus-ext-stackdriver/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.7.2' +__version__ = '0.7.3' diff --git a/opencensus/common/version/__init__.py b/opencensus/common/version/__init__.py index 338d64e4d..35844f7e9 100644 --- a/opencensus/common/version/__init__.py +++ b/opencensus/common/version/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = '0.7.9' +__version__ = '0.7.10' diff --git a/opencensus/trace/span.py b/opencensus/trace/span.py index 04aa376e7..4e872c060 100644 --- a/opencensus/trace/span.py +++ b/opencensus/trace/span.py @@ -370,7 +370,7 @@ def finish(self): def __iter__(self): """Iterate through the span tree.""" - for span in chain(*(map(iter, self.children))): + for span in chain.from_iterable(map(iter, self.children)): yield span yield self