v0.5.9 - graphite-reconnect

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.5.9
+
+* collectors/graphite - feature: reconnect and resend
+
 # 0.5.8
 
 * collectors/memory - calculate percent util

lib/panoptimon/version.rb

@@ -1,5 +1,5 @@
 # Copyright (C) 2012-2014 Cisco, Inc.
 
 module Panoptimon
-  VERSION = "0.5.8"
+  VERSION = "0.5.9"
 end

plugins/graphite/graphite.rb

@@ -14,7 +14,34 @@
 
 prefix.gsub!(/<%= *(host|domain) *%>/) { hostname[$1] }
 
-socket = TCPSocket.open(host, port)
+# attempt to reconnect and resend
+# (but avoid over-zealous connect attempts or memory leaks)
+# (drops metrics if resend buffer is too large)
+writer = ->(){
+  buffer = []
+  defer = ->(line) {
+    buffer.push(line)
+    buffer = buffer.drop(50) if buffer.length > 100
+  }
+  connect = ->() {
+    return if buffer.length > 3 and not(buffer.length % 10 == 0)
+    socket = TCPSocket.open(host, port)
+    socket.write(buffer.slice!(0..buffer.length-1).join('')) \
+      if buffer.length > 0
+    return socket
+  }
+  socket = connect[] # connect early -> fail early
+  ->(line) {
+    begin
+      socket ||= connect[] or return nil.tap { defer.call(line) }
+      socket.write(line)
+    rescue => err
+      warn "graphite error: #{err}"
+      socket = nil
+      defer.call(line)
+    end
+  }
+}[]
 
 ->(metrics) {
   t = Time.now.to_i
@@ -31,6 +58,6 @@
     metric.gsub!(/\|/, '.')
 
     stat = "#{prefix}.#{metric} #{metrics[k]} #{t}"
-    socket.write("#{stat}\n")
+    writer.call("#{stat}\n")
   }
 }