Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added leader/follower check attempt failure metrics #17254

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
+178 −6
Open

Added leader/follower check attempt failure metrics #17254

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b80b110
Added leader/follower check attempt failure metrics
patelsmit32123 Feb 5, 2025
7463a9d
Merge branch 'opensearch-project:main' into leader-follower-check-met…
patelsmit32123 Feb 5, 2025
984cde6
Added tests
patelsmit32123 Feb 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions server/src/main/java/org/opensearch/cluster/ClusterManagerMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ public final class ClusterManagerMetrics {
public final Histogram clusterStatePublishHistogram;

public final Counter leaderCheckFailureCounter;
public final Counter leaderCheckAttemptFailureCounter;
public final Counter followerChecksFailureCounter;
public final Counter followerCheckAttemptFailureCounter;
public final Counter asyncFetchFailureCounter;
public final Counter asyncFetchSuccessCounter;

Expand Down Expand Up @@ -68,11 +70,21 @@ public ClusterManagerMetrics(MetricsRegistry metricsRegistry) {
"Counter for number of failed follower checks",
COUNTER_METRICS_UNIT
);
followerCheckAttemptFailureCounter = metricsRegistry.createCounter(
"followers.checker.attempt.failure.count",
"Counter for number of failed individual follower check attempts",
COUNTER_METRICS_UNIT
);
leaderCheckFailureCounter = metricsRegistry.createCounter(
"leader.checker.failure.count",
"Counter for number of failed leader checks",
COUNTER_METRICS_UNIT
);
leaderCheckAttemptFailureCounter = metricsRegistry.createCounter(
"leader.checker.attempt.failure.count",
"Counter for number of failed individual leader check attempts",
COUNTER_METRICS_UNIT
);
asyncFetchFailureCounter = metricsRegistry.createCounter(
"async.fetch.failure.count",
"Counter for number of failed async fetches",
Expand Down
1 change: 1 addition & 0 deletions server/src/main/java/org/opensearch/cluster/coordination/FollowersChecker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,7 @@ public void handleException(TransportException exp) {
}

failureCountSinceLastSuccess++;
clusterManagerMetrics.incrementCounter(clusterManagerMetrics.followerCheckAttemptFailureCounter, 1.0);

final String reason;
if (exp instanceof ConnectTransportException || exp.getCause() instanceof ConnectTransportException) {
Expand Down
3 changes: 3 additions & 0 deletions server/src/main/java/org/opensearch/cluster/coordination/LeaderChecker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,9 @@ public void handleException(TransportException exp) {
logger.debug("closed check scheduler received a response, doing nothing");
return;
}

clusterManagerMetrics.incrementCounter(clusterManagerMetrics.leaderCheckAttemptFailureCounter, 1.0);

if (exp instanceof ConnectTransportException || exp.getCause() instanceof ConnectTransportException) {
logger.debug(new ParameterizedMessage("leader [{}] disconnected during check", leader), exp);
leaderFailed(new ConnectTransportException(leader, "disconnected during check", exp));
Expand Down