From 692f9da645a5dfadabe2f9d2931b5933910924c7 Mon Sep 17 00:00:00 2001 From: Max Melentyev Date: Tue, 22 Oct 2024 15:23:43 -0400 Subject: [PATCH 1/3] Cache jmx connection, mbeans names and attributes Signed-off-by: Max Melentyev --- .../java/io/prometheus/jmx/JmxCollector.java | 36 ++-- .../prometheus/jmx/JmxMBeanPropertyCache.java | 7 + .../java/io/prometheus/jmx/JmxScraper.java | 176 ++++++++++-------- .../jmx/ObjectNameAttributeFilter.java | 7 + 4 files changed, 130 insertions(+), 96 deletions(-) diff --git a/collector/src/main/java/io/prometheus/jmx/JmxCollector.java b/collector/src/main/java/io/prometheus/jmx/JmxCollector.java index 450083da..cea8d89c 100644 --- a/collector/src/main/java/io/prometheus/jmx/JmxCollector.java +++ b/collector/src/main/java/io/prometheus/jmx/JmxCollector.java @@ -88,9 +88,25 @@ private static class Config { long lastUpdate = 0L; MatchedRulesCache rulesCache; + + private JmxScraper _jmxScraper; + + JmxScraper jmxScraper() { + if (_jmxScraper == null) { + _jmxScraper = + new JmxScraper( + jmxUrl, + username, + password, + ssl, + includeObjectNames, + excludeObjectNames, + objectNameAttributeFilter); + } + return _jmxScraper; + } } - private PrometheusRegistry prometheusRegistry; private Config config; private File configFile; private long createTimeNanoSecs = System.nanoTime(); @@ -101,8 +117,6 @@ private static class Config { private Gauge jmxScrapeError; private Gauge jmxScrapeCachedBeans; - private final JmxMBeanPropertyCache jmxMBeanPropertyCache = new JmxMBeanPropertyCache(); - public JmxCollector(File in) throws IOException, MalformedObjectNameException { this(in, null); } @@ -130,8 +144,6 @@ public JmxCollector register() { } public JmxCollector register(PrometheusRegistry prometheusRegistry) { - this.prometheusRegistry = prometheusRegistry; - configReloadSuccess = Counter.builder() .name("jmx_config_reload_success_total") @@ -716,18 +728,6 @@ public MetricSnapshots collect() { Receiver receiver = new Receiver(config, stalenessTracker); - JmxScraper scraper = - new JmxScraper( - config.jmxUrl, - config.username, - config.password, - config.ssl, - config.includeObjectNames, - config.excludeObjectNames, - config.objectNameAttributeFilter, - receiver, - jmxMBeanPropertyCache); - long start = System.nanoTime(); double error = 0; @@ -736,7 +736,7 @@ public MetricSnapshots collect() { throw new IllegalStateException("JMXCollector waiting for startDelaySeconds"); } try { - scraper.doScrape(); + config.jmxScraper().doScrape(receiver); } catch (Exception e) { error = 1; StringWriter sw = new StringWriter(); diff --git a/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java b/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java index 0b738daf..8f640752 100644 --- a/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java +++ b/collector/src/main/java/io/prometheus/jmx/JmxMBeanPropertyCache.java @@ -64,6 +64,13 @@ public JmxMBeanPropertyCache() { this.keyPropertiesPerBean = new ConcurrentHashMap<>(); } + public JmxMBeanPropertyCache(Set mBeanNames) { + this(); + for (ObjectName mBeanName : mBeanNames) { + getKeyPropertyList(mBeanName); + } + } + Map> getKeyPropertiesPerBean() { return keyPropertiesPerBean; } diff --git a/collector/src/main/java/io/prometheus/jmx/JmxScraper.java b/collector/src/main/java/io/prometheus/jmx/JmxScraper.java index 926c764f..e93b39e8 100644 --- a/collector/src/main/java/io/prometheus/jmx/JmxScraper.java +++ b/collector/src/main/java/io/prometheus/jmx/JmxScraper.java @@ -22,6 +22,7 @@ import io.prometheus.jmx.logger.LoggerFactory; import java.io.IOException; import java.lang.management.ManagementFactory; +import java.lang.ref.Cleaner; import java.util.*; import javax.management.*; import javax.management.openmbean.CompositeData; @@ -38,6 +39,7 @@ class JmxScraper { private static final Logger LOGGER = LoggerFactory.getLogger(JmxScraper.class); + private static final Cleaner CLEANER = Cleaner.create(); public interface MBeanReceiver { void recordBean( @@ -50,14 +52,19 @@ void recordBean( Object value); } - private final MBeanReceiver receiver; private final String jmxUrl; private final String username; private final String password; private final boolean ssl; private final List includeObjectNames, excludeObjectNames; - private final ObjectNameAttributeFilter objectNameAttributeFilter; - private final JmxMBeanPropertyCache jmxMBeanPropertyCache; + // TODO: accept lists of attributes rather than filter object + private final ObjectNameAttributeFilter defaultObjectNameAttributeFilter; + + // Values cached per connection. + private MBeanServerConnection _beanConn; + private Set mBeanNames; + private ObjectNameAttributeFilter objectNameAttributeFilter; + private JmxMBeanPropertyCache jmxMBeanPropertyCache; public JmxScraper( String jmxUrl, @@ -66,87 +73,98 @@ public JmxScraper( boolean ssl, List includeObjectNames, List excludeObjectNames, - ObjectNameAttributeFilter objectNameAttributeFilter, - MBeanReceiver receiver, - JmxMBeanPropertyCache jmxMBeanPropertyCache) { + ObjectNameAttributeFilter objectNameAttributeFilter) { this.jmxUrl = jmxUrl; - this.receiver = receiver; this.username = username; this.password = password; this.ssl = ssl; this.includeObjectNames = includeObjectNames; this.excludeObjectNames = excludeObjectNames; - this.objectNameAttributeFilter = objectNameAttributeFilter; - this.jmxMBeanPropertyCache = jmxMBeanPropertyCache; + this.defaultObjectNameAttributeFilter = objectNameAttributeFilter; } - /** - * Get a list of mbeans on host_port and scrape their values. - * - *

Values are passed to the receiver in a single thread. - */ - public void doScrape() throws Exception { - MBeanServerConnection beanConn; - JMXConnector jmxc = null; + private MBeanServerConnection connectToMBeanServer() throws Exception { if (jmxUrl.isEmpty()) { - beanConn = ManagementFactory.getPlatformMBeanServer(); - } else { - Map environment = new HashMap<>(); - if (username != null - && username.length() != 0 - && password != null - && password.length() != 0) { - String[] credent = new String[] {username, password}; - environment.put(javax.management.remote.JMXConnector.CREDENTIALS, credent); - } - if (ssl) { - environment.put(Context.SECURITY_PROTOCOL, "ssl"); - SslRMIClientSocketFactory clientSocketFactory = new SslRMIClientSocketFactory(); - environment.put( - RMIConnectorServer.RMI_CLIENT_SOCKET_FACTORY_ATTRIBUTE, - clientSocketFactory); - - if (!"true".equalsIgnoreCase(System.getenv("RMI_REGISTRY_SSL_DISABLED"))) { - environment.put("com.sun.jndi.rmi.factory.socket", clientSocketFactory); - } - } + return ManagementFactory.getPlatformMBeanServer(); + } - jmxc = JMXConnectorFactory.connect(new JMXServiceURL(jmxUrl), environment); - beanConn = jmxc.getMBeanServerConnection(); + Map environment = new HashMap<>(); + if (username != null + && username.length() != 0 + && password != null + && password.length() != 0) { + String[] credent = new String[] {username, password}; + environment.put(javax.management.remote.JMXConnector.CREDENTIALS, credent); } - try { - // Query MBean names, see #89 for reasons queryMBeans() is used instead of queryNames() - Set mBeanNames = new HashSet<>(); - for (ObjectName name : includeObjectNames) { - for (ObjectInstance instance : beanConn.queryMBeans(name, null)) { - mBeanNames.add(instance.getObjectName()); - } + if (ssl) { + environment.put(Context.SECURITY_PROTOCOL, "ssl"); + SslRMIClientSocketFactory clientSocketFactory = new SslRMIClientSocketFactory(); + environment.put( + RMIConnectorServer.RMI_CLIENT_SOCKET_FACTORY_ATTRIBUTE, clientSocketFactory); + + if (!"true".equalsIgnoreCase(System.getenv("RMI_REGISTRY_SSL_DISABLED"))) { + environment.put("com.sun.jndi.rmi.factory.socket", clientSocketFactory); } + } + JMXConnector jmxc = JMXConnectorFactory.connect(new JMXServiceURL(jmxUrl), environment); + CLEANER.register( + this, + () -> { + try { + jmxc.close(); + } catch (IOException e) { + LOGGER.log(FINE, "Failed to close JMX connection", e); + } + }); + return jmxc.getMBeanServerConnection(); + } - for (ObjectName name : excludeObjectNames) { - for (ObjectInstance instance : beanConn.queryMBeans(name, null)) { - mBeanNames.remove(instance.getObjectName()); - } + private void loadMBeanNames(MBeanServerConnection beanConn) throws Exception { + // Query MBean names, see #89 for reasons queryMBeans() is used instead of queryNames() + mBeanNames = new HashSet<>(); + for (ObjectName name : includeObjectNames) { + for (ObjectInstance instance : beanConn.queryMBeans(name, null)) { + mBeanNames.add(instance.getObjectName()); } + } - // Now that we have *only* the whitelisted mBeans, remove any old ones from the cache - // and dynamic attribute filter: - jmxMBeanPropertyCache.onlyKeepMBeans(mBeanNames); - objectNameAttributeFilter.onlyKeepMBeans(mBeanNames); - - for (ObjectName objectName : mBeanNames) { - long start = System.nanoTime(); - scrapeBean(beanConn, objectName); - LOGGER.log(FINE, "TIME: %d ns for %s", System.nanoTime() - start, objectName); + for (ObjectName name : excludeObjectNames) { + for (ObjectInstance instance : beanConn.queryMBeans(name, null)) { + mBeanNames.remove(instance.getObjectName()); } - } finally { - if (jmxc != null) { - jmxc.close(); + } + + this.jmxMBeanPropertyCache = new JmxMBeanPropertyCache(mBeanNames); + + this.objectNameAttributeFilter = defaultObjectNameAttributeFilter.dup(); + objectNameAttributeFilter.onlyKeepMBeans(mBeanNames); + } + + /** + * Get a list of mbeans on host_port and scrape their values. + * + *

Values are passed to the receiver in a single thread. + */ + public void doScrape(MBeanReceiver receiver) throws Exception { + synchronized (this) { + if (_beanConn == null) { + _beanConn = connectToMBeanServer(); + loadMBeanNames(_beanConn); } } + MBeanServerConnection beanConn = _beanConn; + + for (ObjectName objectName : mBeanNames) { + long start = System.nanoTime(); + scrapeBean(receiver, beanConn, objectName); + LOGGER.log(FINE, "TIME: %d ns for %s", System.nanoTime() - start, objectName); + } } - private void scrapeBean(MBeanServerConnection beanConn, ObjectName mBeanName) { + private void scrapeBean( + MBeanReceiver receiver, + MBeanServerConnection beanConn, + ObjectName mBeanName) { MBeanInfo mBeanInfo; try { @@ -198,7 +216,7 @@ private void scrapeBean(MBeanServerConnection beanConn, ObjectName mBeanName) { e.getMessage()); // couldn't get them all in one go, try them 1 by 1 - processAttributesOneByOne(beanConn, mBeanName, name2MBeanAttributeInfo); + processAttributesOneByOne(receiver, beanConn, mBeanName, name2MBeanAttributeInfo); return; } @@ -228,6 +246,7 @@ private void scrapeBean(MBeanServerConnection beanConn, ObjectName mBeanName) { name2MBeanAttributeInfo.get(attribute.getName()); LOGGER.log(FINE, "%s_%s process", mBeanName, mBeanAttributeInfo.getName()); processBeanValue( + receiver, mBeanName, mBeanDomain, jmxMBeanPropertyCache.getKeyPropertyList(mBeanName), @@ -252,6 +271,7 @@ private void scrapeBean(MBeanServerConnection beanConn, ObjectName mBeanName) { } private void processAttributesOneByOne( + MBeanReceiver receiver, MBeanServerConnection beanConn, ObjectName mbeanName, Map name2AttrInfo) { @@ -266,6 +286,7 @@ private void processAttributesOneByOne( LOGGER.log(FINE, "%s_%s process", mbeanName, attr.getName()); processBeanValue( + receiver, mbeanName, mbeanName.getDomain(), jmxMBeanPropertyCache.getKeyPropertyList(mbeanName), @@ -283,6 +304,7 @@ private void processAttributesOneByOne( * pass of getting the values/names out in a way it can be processed elsewhere easily. */ private void processBeanValue( + MBeanReceiver receiver, ObjectName objectName, String domain, LinkedHashMap beanProperties, @@ -302,7 +324,7 @@ private void processBeanValue( value = ((java.util.Date) value).getTime() / 1000.0; } LOGGER.log(FINE, "%s%s%s scrape: %s", domain, beanProperties, attrName, value); - this.receiver.recordBean( + receiver.recordBean( domain, beanProperties, attrKeys, attrName, attrType, attrDescription, value); } else if (value instanceof CompositeData) { LOGGER.log(FINE, "%s%s%s scrape: compositedata", domain, beanProperties, attrName); @@ -314,6 +336,7 @@ private void processBeanValue( String typ = type.getType(key).getTypeName(); Object valu = composite.get(key); processBeanValue( + receiver, objectName, domain, beanProperties, @@ -380,6 +403,7 @@ private void processBeanValue( name = attrName; } processBeanValue( + receiver, objectName, domain, l2s, @@ -400,6 +424,7 @@ private void processBeanValue( Optional optional = (Optional) value; if (optional.isPresent()) { processBeanValue( + receiver, objectName, domain, beanProperties, @@ -412,6 +437,7 @@ private void processBeanValue( } else if (value.getClass().isEnum()) { LOGGER.log(FINE, "%s%s%s scrape: %s", domain, beanProperties, attrName, value); processBeanValue( + receiver, objectName, domain, beanProperties, @@ -453,10 +479,8 @@ public static void main(String[] args) throws Exception { (args.length > 3 && "ssl".equalsIgnoreCase(args[3])), objectNames, new LinkedList<>(), - objectNameAttributeFilter, - new StdoutWriter(), - new JmxMBeanPropertyCache()) - .doScrape(); + objectNameAttributeFilter) + .doScrape(new StdoutWriter()); } else if (args.length > 0) { new JmxScraper( args[0], @@ -465,10 +489,8 @@ public static void main(String[] args) throws Exception { false, objectNames, new LinkedList<>(), - objectNameAttributeFilter, - new StdoutWriter(), - new JmxMBeanPropertyCache()) - .doScrape(); + objectNameAttributeFilter) + .doScrape(new StdoutWriter()); } else { new JmxScraper( "", @@ -477,10 +499,8 @@ public static void main(String[] args) throws Exception { false, objectNames, new LinkedList<>(), - objectNameAttributeFilter, - new StdoutWriter(), - new JmxMBeanPropertyCache()) - .doScrape(); + objectNameAttributeFilter) + .doScrape(new StdoutWriter()); } } } diff --git a/collector/src/main/java/io/prometheus/jmx/ObjectNameAttributeFilter.java b/collector/src/main/java/io/prometheus/jmx/ObjectNameAttributeFilter.java index f8317853..d4395a4c 100644 --- a/collector/src/main/java/io/prometheus/jmx/ObjectNameAttributeFilter.java +++ b/collector/src/main/java/io/prometheus/jmx/ObjectNameAttributeFilter.java @@ -52,6 +52,13 @@ private ObjectNameAttributeFilter() { dynamicExcludeObjectNameAttributesMap = new ConcurrentHashMap<>(); } + public ObjectNameAttributeFilter dup() { + ObjectNameAttributeFilter copy = new ObjectNameAttributeFilter(); + copy.configExcludeObjectNameAttributesMap.putAll(configExcludeObjectNameAttributesMap); + copy.dynamicExcludeObjectNameAttributesMap.putAll(dynamicExcludeObjectNameAttributesMap); + return copy; + } + /** * Method to initialize the ObjectNameAttributeFilter * From 32fae25bbc120fce2759de7eeb7a7a309d518753 Mon Sep 17 00:00:00 2001 From: Max Melentyev Date: Wed, 23 Oct 2024 09:48:39 -0400 Subject: [PATCH 2/3] Refresh mbeans cache when they are registered/unregistered Signed-off-by: Max Melentyev --- .../java/io/prometheus/jmx/JmxScraper.java | 90 ++++++++++++++----- 1 file changed, 66 insertions(+), 24 deletions(-) diff --git a/collector/src/main/java/io/prometheus/jmx/JmxScraper.java b/collector/src/main/java/io/prometheus/jmx/JmxScraper.java index e93b39e8..0dac2391 100644 --- a/collector/src/main/java/io/prometheus/jmx/JmxScraper.java +++ b/collector/src/main/java/io/prometheus/jmx/JmxScraper.java @@ -29,6 +29,7 @@ import javax.management.openmbean.CompositeType; import javax.management.openmbean.TabularData; import javax.management.openmbean.TabularType; +import javax.management.relation.MBeanServerNotificationFilter; import javax.management.remote.JMXConnector; import javax.management.remote.JMXConnectorFactory; import javax.management.remote.JMXServiceURL; @@ -62,9 +63,23 @@ void recordBean( // Values cached per connection. private MBeanServerConnection _beanConn; - private Set mBeanNames; - private ObjectNameAttributeFilter objectNameAttributeFilter; - private JmxMBeanPropertyCache jmxMBeanPropertyCache; + private Cache cache; + private boolean cacheIsStale = false; + + private class Cache { + private final Set mBeanNames; + private final ObjectNameAttributeFilter objectNameAttributeFilter; + private final JmxMBeanPropertyCache jmxMBeanPropertyCache; + + private Cache( + Set mBeanNames, + ObjectNameAttributeFilter objectNameAttributeFilter, + JmxMBeanPropertyCache jmxMBeanPropertyCache) { + this.mBeanNames = mBeanNames; + this.objectNameAttributeFilter = objectNameAttributeFilter; + this.jmxMBeanPropertyCache = jmxMBeanPropertyCache; + } + } public JmxScraper( String jmxUrl, @@ -119,9 +134,41 @@ private MBeanServerConnection connectToMBeanServer() throws Exception { return jmxc.getMBeanServerConnection(); } - private void loadMBeanNames(MBeanServerConnection beanConn) throws Exception { + private synchronized MBeanServerConnection getMBeanServerConnection() throws Exception { + if (_beanConn == null) { + cacheIsStale = true; + _beanConn = connectToMBeanServer(); + // Subscribe to MBeans register/unregister events to invalidate cache + MBeanServerNotificationFilter filter = new MBeanServerNotificationFilter(); + filter.enableAllObjectNames(); + _beanConn.addNotificationListener( + MBeanServerDelegate.DELEGATE_NAME, + (notification, handback) -> { + String type = notification.getType(); + if (MBeanServerNotification.REGISTRATION_NOTIFICATION.equals(type) + || MBeanServerNotification.UNREGISTRATION_NOTIFICATION.equals( + type)) { + LOGGER.log(FINE, "Marking cache as stale due to %s", type); + // Mark cache as stale instead of refreshing it immediately + // to debounce multiple notifications. + synchronized (this) { + cacheIsStale = true; + } + } + }, + filter, + null); + } + if (cacheIsStale) { + cache = fetchCache(_beanConn); + cacheIsStale = false; + } + return _beanConn; + } + + private Cache fetchCache(MBeanServerConnection beanConn) throws Exception { // Query MBean names, see #89 for reasons queryMBeans() is used instead of queryNames() - mBeanNames = new HashSet<>(); + Set mBeanNames = new HashSet<>(); for (ObjectName name : includeObjectNames) { for (ObjectInstance instance : beanConn.queryMBeans(name, null)) { mBeanNames.add(instance.getObjectName()); @@ -134,10 +181,10 @@ private void loadMBeanNames(MBeanServerConnection beanConn) throws Exception { } } - this.jmxMBeanPropertyCache = new JmxMBeanPropertyCache(mBeanNames); + ObjectNameAttributeFilter attributeFilter = defaultObjectNameAttributeFilter.dup(); + attributeFilter.onlyKeepMBeans(mBeanNames); - this.objectNameAttributeFilter = defaultObjectNameAttributeFilter.dup(); - objectNameAttributeFilter.onlyKeepMBeans(mBeanNames); + return new Cache(mBeanNames, attributeFilter, new JmxMBeanPropertyCache(mBeanNames)); } /** @@ -145,16 +192,13 @@ private void loadMBeanNames(MBeanServerConnection beanConn) throws Exception { * *

Values are passed to the receiver in a single thread. */ - public void doScrape(MBeanReceiver receiver) throws Exception { - synchronized (this) { - if (_beanConn == null) { - _beanConn = connectToMBeanServer(); - loadMBeanNames(_beanConn); - } - } - MBeanServerConnection beanConn = _beanConn; + public synchronized void doScrape(MBeanReceiver receiver) throws Exception { + // Method is synchronized to avoid multiple scrapes running concurrently + // and let one of them refresh the cache in the middle of the scrape. - for (ObjectName objectName : mBeanNames) { + MBeanServerConnection beanConn = getMBeanServerConnection(); + + for (ObjectName objectName : cache.mBeanNames) { long start = System.nanoTime(); scrapeBean(receiver, beanConn, objectName); LOGGER.log(FINE, "TIME: %d ns for %s", System.nanoTime() - start, objectName); @@ -162,9 +206,7 @@ public void doScrape(MBeanReceiver receiver) throws Exception { } private void scrapeBean( - MBeanReceiver receiver, - MBeanServerConnection beanConn, - ObjectName mBeanName) { + MBeanReceiver receiver, MBeanServerConnection beanConn, ObjectName mBeanName) { MBeanInfo mBeanInfo; try { @@ -186,7 +228,7 @@ private void scrapeBean( continue; } - if (objectNameAttributeFilter.exclude(mBeanName, mBeanAttributeInfo.getName())) { + if (cache.objectNameAttributeFilter.exclude(mBeanName, mBeanAttributeInfo.getName())) { continue; } @@ -249,7 +291,7 @@ private void scrapeBean( receiver, mBeanName, mBeanDomain, - jmxMBeanPropertyCache.getKeyPropertyList(mBeanName), + cache.jmxMBeanPropertyCache.getKeyPropertyList(mBeanName), new LinkedList<>(), mBeanAttributeInfo.getName(), mBeanAttributeInfo.getType(), @@ -289,7 +331,7 @@ private void processAttributesOneByOne( receiver, mbeanName, mbeanName.getDomain(), - jmxMBeanPropertyCache.getKeyPropertyList(mbeanName), + cache.jmxMBeanPropertyCache.getKeyPropertyList(mbeanName), new LinkedList<>(), attr.getName(), attr.getType(), @@ -447,7 +489,7 @@ private void processBeanValue( attrDescription, value.toString()); } else { - objectNameAttributeFilter.add(objectName, attrName); + cache.objectNameAttributeFilter.add(objectName, attrName); LOGGER.log(FINE, "%s%s scrape: %s not exported", domain, beanProperties, attrType); } } From b5119ad516d648f8572a2f831ea41a93fc4307b8 Mon Sep 17 00:00:00 2001 From: Max Melentyev Date: Thu, 24 Oct 2024 12:20:58 -0400 Subject: [PATCH 3/3] Reset connection if scrape failed Signed-off-by: Max Melentyev --- .../main/java/io/prometheus/jmx/JmxScraper.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/collector/src/main/java/io/prometheus/jmx/JmxScraper.java b/collector/src/main/java/io/prometheus/jmx/JmxScraper.java index 0dac2391..13930d93 100644 --- a/collector/src/main/java/io/prometheus/jmx/JmxScraper.java +++ b/collector/src/main/java/io/prometheus/jmx/JmxScraper.java @@ -196,12 +196,18 @@ public synchronized void doScrape(MBeanReceiver receiver) throws Exception { // Method is synchronized to avoid multiple scrapes running concurrently // and let one of them refresh the cache in the middle of the scrape. - MBeanServerConnection beanConn = getMBeanServerConnection(); + try { + MBeanServerConnection beanConn = getMBeanServerConnection(); - for (ObjectName objectName : cache.mBeanNames) { - long start = System.nanoTime(); - scrapeBean(receiver, beanConn, objectName); - LOGGER.log(FINE, "TIME: %d ns for %s", System.nanoTime() - start, objectName); + for (ObjectName objectName : cache.mBeanNames) { + long start = System.nanoTime(); + scrapeBean(receiver, beanConn, objectName); + LOGGER.log(FINE, "TIME: %d ns for %s", System.nanoTime() - start, objectName); + } + } finally { + // reconnect to resolve connection issues + // TODO: should it make a single retry with a new connection? + _beanConn = null; } }