Release for v0.7.11 (#960)

.github/CODEOWNERS

@@ -2,4 +2,4 @@
 # This file controls who is tagged for review for any given pull request.
 
 # For anything not explicitly taken by someone else:
-*               @census-instrumentation/global-owners @c24t @reyang @songy23
+*       @census-instrumentation/global-owners @aabmass @c24t @hectorhdzg @lzchen @reyang @songy23 @victoraugustolls

CHANGELOG.md

@@ -2,21 +2,27 @@
 
 ## Unreleased
 
+## 0.7.11
+Released 2020-10-13
+
+- PeriodicMetricTask flush on exit
+([#943](https://github.com/census-instrumentation/opencensus-python/pull/943))
+
 ## 0.7.10
 Released 2020-06-29
 
 - Updated `azure` module
-  ([#903](https://github.com/census-instrumentation/opencensus-python/pull/903),
-   [#925](https://github.com/census-instrumentation/opencensus-python/pull/925))
+([#903](https://github.com/census-instrumentation/opencensus-python/pull/903),
+ [#925](https://github.com/census-instrumentation/opencensus-python/pull/925))
 
 - Updated `stackdriver` module
-  ([#919](https://github.com/census-instrumentation/opencensus-python/pull/919))
+([#919](https://github.com/census-instrumentation/opencensus-python/pull/919))
 
 ## 0.7.9
 Released 2020-06-17
 
-- Hotfix
-  ([#915](https://github.com/census-instrumentation/opencensus-python/pull/915))
+- Hotfix for breaking change
+  ([#915](https://github.com/census-instrumentation/opencensus-python/pull/915),
 
 ## 0.7.8
 Released 2020-06-17
@@ -26,13 +32,13 @@ Released 2020-06-17
    [#902](https://github.com/census-instrumentation/opencensus-python/pull/902))
 
 ## 0.7.7
-Released 2020-02-04
+Released 2020-02-03
 
 - Updated `azure` module
-  ([#837](https://github.com/census-instrumentation/opencensus-python/pull/837),
-   [#845](https://github.com/census-instrumentation/opencensus-python/pull/845),
-   [#848](https://github.com/census-instrumentation/opencensus-python/pull/848),
-   [#851](https://github.com/census-instrumentation/opencensus-python/pull/851))
+([#837](https://github.com/census-instrumentation/opencensus-python/pull/837),
+ [#845](https://github.com/census-instrumentation/opencensus-python/pull/845),
+ [#848](https://github.com/census-instrumentation/opencensus-python/pull/848),
+ [#851](https://github.com/census-instrumentation/opencensus-python/pull/851))
 
 ## 0.7.6
 Released 2019-11-26

contrib/opencensus-ext-azure/CHANGELOG.md

@@ -2,6 +2,20 @@
 
 ## Unreleased
 
+## 1.0.5
+Released 2020-10-13
+
+- Attach rate metrics via Heartbeat for Web and Function apps
+  ([#930](https://github.com/census-instrumentation/opencensus-python/pull/930))
+- Attach rate metrics for VM
+  ([#935](https://github.com/census-instrumentation/opencensus-python/pull/935))
+- Add links in properties for trace exporter envelopes
+  ([#936](https://github.com/census-instrumentation/opencensus-python/pull/936))
+- Fix attach rate metrics for VM to only ping data service on retry
+  ([#946](https://github.com/census-instrumentation/opencensus-python/pull/946))
+- Added queue capacity configuration for exporters
+  ([#949](https://github.com/census-instrumentation/opencensus-python/pull/949))
+
 ## 1.0.4
 Released 2020-06-29
 
@@ -18,7 +32,6 @@ Released 2020-06-17
 - Add support to initialize azure exporters with proxies
   ([#902](https://github.com/census-instrumentation/opencensus-python/pull/902))
 
-
 ## 1.0.2
 Released 2020-02-04
 

contrib/opencensus-ext-azure/examples/logs/properties.py

@@ -33,4 +33,4 @@
 except Exception:
     logger.exception('Captured an exception.', extra=properties)
 
-input("...")
+input("...")

contrib/opencensus-ext-azure/examples/logs/simple.py

@@ -23,4 +23,4 @@
 logger.addHandler(AzureLogHandler())
 logger.warning('Hello, World!')
 
-input("...")
+input("...")

contrib/opencensus-ext-azure/examples/traces/client.py

@@ -26,6 +26,6 @@
 tracer = Tracer(exporter=AzureExporter(), sampler=ProbabilitySampler(1.0))
 with tracer.span(name='parent'):
     with tracer.span(name='child'):
-        response = requests.get(url='http://localhost:8080/')
+        response = requests.get(url='http://example.com/')
         print(response.status_code)
         print(response.text)

contrib/opencensus-ext-azure/examples/traces/simple.py

@@ -23,3 +23,4 @@
 
 with tracer.span(name='foo'):
     print('Hello, World!')
+input(...)

contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py

@@ -56,6 +56,7 @@ def process_options(options):
                 TEMPDIR_PREFIX + TEMPDIR_SUFFIX
             )
 
+    # proxies
     if options.proxies is None:
         options.proxies = '{}'
 
@@ -109,6 +110,7 @@ def __init__(self, *args, **kwargs):
         max_batch_size=100,
         minimum_retry_interval=60,  # minimum retry interval in seconds
         proxies=None,  # string maps url schemes to the url of the proxies
+        queue_capacity=8192,
         storage_maintenance_period=60,
         storage_max_size=50*1024*1024,  # 50MiB
         storage_path=None,

contrib/opencensus-ext-azure/opencensus/ext/azure/common/exporter.py

@@ -28,7 +28,7 @@ def __init__(self, **options):
         self.max_batch_size = options.max_batch_size
         # TODO: queue should be moved to tracer
         # too much refactor work, leave to the next PR
-        self._queue = Queue(capacity=8192)  # TODO: make this configurable
+        self._queue = Queue(capacity=options.queue_capacity)
         # TODO: worker should not be created in the base exporter
         self._worker = Worker(self._queue, self)
         self._worker.start()
@@ -61,7 +61,9 @@ def __init__(self, src, dst):
         self.src = src
         self.dst = dst
         self._stopping = False
-        super(Worker, self).__init__()
+        super(Worker, self).__init__(
+            name="AzureExporter Worker"
+        )
 
     def run(self):  # pragma: NO COVER
         # Indicate that this thread is an exporter thread.

contrib/opencensus-ext-azure/opencensus/ext/azure/common/storage.py

@@ -81,6 +81,7 @@ def __init__(
             maintenance_period=60,  # 1 minute
             retention_period=7*24*60*60,  # 7 days
             write_timeout=60,  # 1 minute
+            source=None,
     ):
         self.path = os.path.abspath(path)
         self.max_size = max_size
@@ -92,6 +93,7 @@ def __init__(
         self._maintenance_task = PeriodicTask(
             interval=self.maintenance_period,
             function=self._maintenance_routine,
+            name='{} Storage Worker'.format(source)
         )
         self._maintenance_task.daemon = True
         self._maintenance_task.start()
@@ -131,7 +133,9 @@ def gets(self):
             if path.endswith('.tmp'):
                 if name < timeout_deadline:
                     try:
-                        os.remove(path)  # TODO: log data loss
+                        os.remove(path)
+                        logger.warning(
+                            'File write exceeded timeout. Dropping telemetry')
                     except Exception:
                         pass  # keep silent
             if path.endswith('.lock'):
@@ -146,7 +150,10 @@ def gets(self):
             if path.endswith('.blob'):
                 if name < retention_deadline:
                     try:
-                        os.remove(path)  # TODO: log data loss
+                        os.remove(path)
+                        logger.warning(
+                            'File write exceeded retention.' +
+                            'Dropping telemetry')
                     except Exception:
                         pass  # keep silent
                 else:

contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py

@@ -79,7 +79,6 @@ def _transmit(self, envelopes):
             logger.info('Transmission succeeded: %s.', text)
             return 0
         if response.status_code == 206:  # Partial Content
-            # TODO: store the unsent data
             if data:
                 try:
                     resend_envelopes = []

contrib/opencensus-ext-azure/opencensus/ext/azure/common/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = '1.0.4'
+__version__ = '1.0.5'

contrib/opencensus-ext-azure/opencensus/ext/azure/log_exporter/__init__.py

@@ -30,6 +30,7 @@
 )
 from opencensus.ext.azure.common.storage import LocalFileStorage
 from opencensus.ext.azure.common.transport import TransportMixin
+from opencensus.ext.azure.metrics_exporter import heartbeat_metrics
 from opencensus.trace import execution_context
 
 logger = logging.getLogger(__name__)
@@ -52,12 +53,33 @@ def __init__(self, **options):
             max_size=self.options.storage_max_size,
             maintenance_period=self.options.storage_maintenance_period,
             retention_period=self.options.storage_retention_period,
+            source=self.__class__.__name__,
         )
         self._telemetry_processors = []
         self.addFilter(SamplingFilter(self.options.logging_sampling_rate))
-        self._queue = Queue(capacity=8192)  # TODO: make this configurable
+        self._queue = Queue(capacity=self.options.queue_capacity)
         self._worker = Worker(self._queue, self)
         self._worker.start()
+        heartbeat_metrics.enable_heartbeat_metrics(
+            self.options.connection_string, self.options.instrumentation_key)
+
+    def _export(self, batch, event=None):  # pragma: NO COVER
+        try:
+            if batch:
+                envelopes = [self.log_record_to_envelope(x) for x in batch]
+                envelopes = self.apply_telemetry_processors(envelopes)
+                result = self._transmit(envelopes)
+                if result > 0:
+                    self.storage.put(envelopes, result)
+            if event:
+                if isinstance(event, QueueExitEvent):
+                    self._transmit_from_storage()  # send files before exit
+                return
+            if len(batch) < self.options.max_batch_size:
+                self._transmit_from_storage()
+        finally:
+            if event:
+                event.set()
 
     def _export(self, batch, event=None):  # pragma: NO COVER
         try:

contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py

@@ -52,7 +52,10 @@ def __init__(self, **options):
             max_size=self.options.storage_max_size,
             maintenance_period=self.options.storage_maintenance_period,
             retention_period=self.options.storage_retention_period,
+            source=self.__class__.__name__,
         )
+        self._atexit_handler = atexit.register(self.shutdown)
+        self.exporter_thread = None
         super(MetricsExporter, self).__init__()
 
     def export_metrics(self, metrics):
@@ -133,14 +136,26 @@ def _create_envelope(self, data_point, timestamp, properties):
         envelope.data = Data(baseData=data, baseType="MetricData")
         return envelope
 
+    def shutdown(self):
+        # Flush the exporter thread
+        if self.exporter_thread:
+            self.exporter_thread.close()
+        # Shutsdown storage worker
+        self.storage.close()
+
 
 def new_metrics_exporter(**options):
     exporter = MetricsExporter(**options)
     producers = [stats_module.stats]
     if exporter.options.enable_standard_metrics:
         producers.append(standard_metrics.producer)
-    transport.get_exporter_thread(producers,
-                                  exporter,
-                                  interval=exporter.options.export_interval)
-    atexit.register(exporter.export_metrics, stats_module.stats.get_metrics())
+    exporter.exporter_thread = transport.get_exporter_thread(
+                                    producers,
+                                    exporter,
+                                    interval=exporter.options.export_interval)
+    from opencensus.ext.azure.metrics_exporter import heartbeat_metrics
+    heartbeat_metrics.enable_heartbeat_metrics(
+        exporter.options.connection_string,
+        exporter.options.instrumentation_key
+    )
     return exporter

contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/heartbeat_metrics/__init__.py

@@ -0,0 +1,55 @@
+# Copyright 2020, OpenCensus Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+
+from opencensus.ext.azure.metrics_exporter import MetricsExporter
+from opencensus.ext.azure.metrics_exporter.heartbeat_metrics.heartbeat import (
+    HeartbeatMetric,
+)
+from opencensus.metrics import transport
+from opencensus.metrics.export.metric_producer import MetricProducer
+
+_HEARTBEAT_METRICS = None
+_HEARTBEAT_LOCK = threading.Lock()
+
+
+def enable_heartbeat_metrics(connection_string, ikey):
+    with _HEARTBEAT_LOCK:
+        # Only start heartbeat if did not exist before
+        global _HEARTBEAT_METRICS  # pylint: disable=global-statement
+        if _HEARTBEAT_METRICS is None:
+            exporter = MetricsExporter(
+                connection_string=connection_string,
+                instrumentation_key=ikey,
+                export_interval=900.0,  # Send every 15 minutes
+            )
+            producer = AzureHeartbeatMetricsProducer()
+            _HEARTBEAT_METRICS = producer
+            exporter.exporter_thread = \
+                transport.get_exporter_thread([_HEARTBEAT_METRICS],
+                                              exporter,
+                                              exporter.options.export_interval)
+
+
+class AzureHeartbeatMetricsProducer(MetricProducer):
+    """Implementation of the producer of heartbeat metrics.
+
+    Includes Azure attach rate metrics, implemented using gauges.
+    """
+    def __init__(self):
+        self._heartbeat = HeartbeatMetric()
+
+    def get_metrics(self):
+        return self._heartbeat.get_metrics()