Skip to content

Commit f1899a6

Browse files
author
吴蓬云
committed
add hadoop, yarn and hive yaml config sample , for this rules has been tested passed on hadoop2.4.1 ,yarn2.7.7 and hive2.3.9
Signed-off-by: 吴蓬云 <[email protected]>
1 parent ea03179 commit f1899a6

File tree

5 files changed

+212
-0
lines changed

5 files changed

+212
-0
lines changed

‎example_configs/hadoop_datanode.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
ssl: false
2+
lowercaseOutputName: true
3+
lowercaseOutputLabelNames: true
4+
rules:
5+
- pattern: 'Hadoop<service=DataNode, name=DataNodeInfo, key=(.*), key_=networkErrors><>DatanodeNetworkCounts: (\d\.\-Ee+)'
6+
name: hadoop_datanode_network_errors
7+
value: $2
8+
help: "DataNode networkError counts every host"
9+
type: COUNTER
10+
labels:
11+
"host": "$1"
12+
- pattern: 'Hadoop<service=(\w+), name=(\w+)><>(\w+)'
13+
name: Hadoop_$1_$2_$3
14+
attrNameSnakeCase: true
15+
help: "Common config, parse from jmx bean: {Hadoop:service=$1, name=$2}.$3"

‎example_configs/hadoop_namenode.yml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
lowercaseOutputName: true
2+
rules:
3+
- pattern: "Hadoop<service=NameNode, name=NameNodeStatus><>State:.*?active.*?"
4+
name: hadoop_namenode_status
5+
value: 1
6+
help: "NameNodeStatus status, means active and means standby"
7+
type: COUNTER
8+
- pattern: "Hadoop<service=NameNode, name=NameNodeStatus><>State:.*?standby.*?"
9+
name: hadoop_namenode_status
10+
value: 0
11+
help: "NameNodeStatus status, means active and means standby"
12+
type: COUNTER
13+
- pattern: "Hadoop<service=NameNode, name=FSNamesystemState><>FSState:.*?Operational.*?"
14+
name: hadoop_namenode_fsnamesystem_state
15+
value: 1
16+
help: "FSNameSystem state, means Operational and means Safemode"
17+
type: COUNTER
18+
- pattern: "Hadoop<service=NameNode, name=FSNamesystemState><>FSState:.*?Safemode.*?"
19+
name: hadoop_namenode_fsnamesystem_state
20+
value: 0
21+
help: "FSNameSystem state, means Operational and means Safemode"
22+
type: COUNTER
23+
- pattern: "Hadoop<service=(\w+), name=(\w+)><>(\w+)"
24+
name: hadoop_$1_$2_$3
25+
attrNameSnakeCase: true
26+
help: "Common config, parse from jmx bean: {Hadoop:service=$1, name=$2}.$3"

‎example_configs/hive-2.0.0.yml

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
ssl: false
2+
lowercaseOutputName: true
3+
# the data of these rules comes from hive jmx url: 192.168.205.197:10002/jmx
4+
# Noted that, you must config hive params in hive-site.xml to enable hive monitor itself:
5+
# hive.metastore.metrics.enabled:true
6+
# hive.server2.metrics.enabled :true
7+
# hive.service.metrics.reporter :JMX
8+
rules:
9+
- pattern: 'Hadoop<service=hive, name=MetricsSystem, sub=Stats>(\w+)'
10+
name: hive_metrics_system_$1
11+
attrNameSnakeCase: true
12+
help: "hadoop hive metrics system status"
13+
- pattern: 'Hadoop<service=hive, name=UgiMetrics>(\w+)'
14+
name: hive_ugi_metrics_$1
15+
attrNameSnakeCase: true
16+
help: "hadoop hive metrics system status"
17+
- pattern: 'metrics<name=buffers.mapped.count><>Value'
18+
name: hive_metrics_buffers_mapped_count
19+
- pattern: 'metrics<name=memory.non-heap.usage><>Value'
20+
name: hive_metrics_memory_non_heap_usage
21+
- pattern: 'metrics<name=gc.PS-Scavenge.time><>Value'
22+
name: hive_metrics_gc_PS_Scavenge_time
23+
- pattern: 'metrics<name=memory.non-heap.init><>Value'
24+
name: hive_metrics_memory_non_heap_init
25+
- pattern: 'metrics<name=memory.pools.Code-Cache.usage><>Value'
26+
name: hive_metrics_memory_pools_Code_Cache_usage
27+
- pattern: 'metrics<name=buffers.direct.used><>Value'
28+
name: hive_metrics_buffers_direct_used
29+
- pattern: 'metrics<name=memory.total.committed><>Value'
30+
name: hive_metrics_memory_total_committed
31+
- pattern: 'metrics<name=buffers.mapped.capacity><>Value'
32+
name: hive_metrics_buffers_mapped_capacity
33+
- pattern: 'metrics<name=memory.heap.max><>Value'
34+
name: hive_metrics_memory_heap_max
35+
- pattern: 'metrics<name=threads.timed_waiting.count><>Value'
36+
name: hive_metrics_threads_timed_waiting_count
37+
- pattern: 'metrics<name=memory.heap.usage><>Value'
38+
name: hive_metrics_memory_heap_usage
39+
- pattern: 'metrics<name=memory.non-heap.max><>Value'
40+
name: hive_metrics_memory_non_heap_max
41+
- pattern: 'metrics<name=threads.runnable.count><>Value'
42+
name: hive_metrics_threads_runnable_count
43+
- pattern: 'metrics<name=memory.pools.PS-Eden-Space.usage><>Value'
44+
name: hive_metrics_memory_pools_PS_Eden_Space_usage
45+
- pattern: 'metrics<name=memory.non-heap.used><>Value'
46+
name: hive_metrics_memory_non_heap_used
47+
- pattern: 'metrics<name=gc.PS-MarkSweep.count><>Value'
48+
name: hive_metrics_gc_PS_Mark_Sweep_count
49+
- pattern: 'metrics<name=buffers.mapped.used><>Value'
50+
name: hive_metrics_buffers_mapped_used
51+
- pattern: 'metrics<name=gc.PS-Scavenge.count><>Value'
52+
name: hive_metrics_gc_PS_Scavenge_count
53+
- pattern: 'metrics<name=threads.new.count><>Value'
54+
name: hive_metrics_threads_new_count
55+
- pattern: 'metrics<name=memory.heap.used><>Value'
56+
name: hive_metrics_memory_heap_used
57+
- pattern: 'metrics<name=memory.total.init><>Value'
58+
name: hive_metrics_memory_total_init
59+
- pattern: 'metrics<name=memory.pools.Metaspace.usage><>Value'
60+
name: hive_metrics_memory_pools_Metaspace_usage
61+
- pattern: 'metrics<name=memory.pools.PS-Old-Gen.usage><>Value'
62+
name: hive_metrics_memory_pools_PS_Old_Gen_usage
63+
- pattern: 'metrics<name=threads.daemon.count><>Value'
64+
name: hive_metrics_threads_daemon_count
65+
- pattern: 'metrics<name=memory.total.max><>Value'
66+
name: hive_metrics_memory_total_max
67+
- pattern: 'metrics<name=threads.count><>Value'
68+
name: hive_metrics_threads_count
69+
- pattern: 'metrics<name=threads.deadlock.count><>Value'
70+
name: hive_metrics_threads_deadlock_count
71+
- pattern: 'metrics<name=memory.heap.init><>Value'
72+
name: hive_metrics_memory_heap_init
73+
- pattern: 'metrics<name=threads.terminated.count><>Value'
74+
name: hive_metrics_threads_terminated_count
75+
- pattern: 'metrics<name=buffers.direct.count><>Value'
76+
name: hive_metrics_buffers_direct_count
77+
- pattern: 'metrics<name=memory.total.used><>Value'
78+
name: hive_metrics_memory_total_used
79+
- pattern: 'metrics<name=memory.non-heap.committed><>Value'
80+
name: hive_metrics_memory_non_heap_committed
81+
- pattern: 'metrics<name=memory.pools.PS-Survivor-Space.usage><>Value'
82+
name: hive_metrics_memory_pools_PS_Survivor_Space_usage
83+
- pattern: 'metrics<name=classLoading.loaded><>Value'
84+
name: hive_metrics_class_loading_loaded
85+
- pattern: 'metrics<name=threads.waiting.count><>Value'
86+
name: hive_metrics_threads_waiting_count
87+
- pattern: 'metrics<name=classLoading.unloaded><>Value'
88+
name: hive_metrics_class_loading_unloaded
89+
- pattern: 'metrics<name=memory.heap.committed><>Value'
90+
name: hive_metrics_memory_heap_committed
91+
- pattern: 'metrics<name=gc.PS-MarkSweep.time><>Value'
92+
name: hive_metrics_gc_PS_Mark_Sweep_time
93+
94+

‎example_configs/yarn-nodemanager.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
ssl: false
2+
lowercaseOutputName: true
3+
lowercaseOutputLabelNames: true
4+
rules:
5+
- pattern: 'Hadoop<service=NodeManager, name=RpcActivityForPort(\d+)><>(\w+)'
6+
name: hadoop_nodemanager_rpc_activity_for_port_$2
7+
attrNameSnakeCase: true
8+
labels:
9+
"port": $1
10+
help: "rpc activity for port metric, parse from jmx bean: {Hadoop:service=NodeManager, name=RpcActivityForPort$1}.$2"
11+
- pattern: 'Hadoop<service=NodeManager, name=(\w+)><>(\w+)'
12+
name: hadoop_nodemanager_$1_$2
13+
attrNameSnakeCase: true
14+
help: "Common config, parse from jmx bean: {Hadoop:service=NodeManager, name=$1}.$2"

‎example_configs/yarn-resourcemanager.yml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
ssl: false
2+
lowercaseOutputName: true
3+
lowercaseOutputLabelNames: true
4+
rules:
5+
- pattern: 'Hadoop<service=ResourceManager, name=MetricsSystem, sub=(\w+)><>(\w+)'
6+
name: hadoop_resourcemanager_metrics_system_$1_$2
7+
attrNameSnakeCase: true
8+
help: "rpc activity for port metric, parse from jmx bean: {Hadoop:service=ResourceManager, name=RpcActivityForPort$1}.$2"
9+
- pattern: 'Hadoop<service=ResourceManager, name=RpcActivityForPort(\d+)><>(\w+)'
10+
name: hadoop_resourcemanager_rpc_activity_for_port_$2
11+
attrNameSnakeCase: true
12+
labels:
13+
"port": $1
14+
help: "rpc activity for port metric, parse from jmx bean: {Hadoop:service=ResourceManager, name=RpcActivityForPort$1}.$2"
15+
- pattern: 'Hadoop<service=ResourceManager, name=RpcDetailedActivityForPort(\d+)><>(\w+)'
16+
name: hadoop_resourcemanager_rpc_activity_for_port_$2
17+
attrNameSnakeCase: true
18+
labels:
19+
"port": $1
20+
help: "rpc activity for port metric, parse from jmx bean: {Hadoop:service=ResourceManager, name=RpcActivityForPort$1}.$2"
21+
- pattern: 'Hadoop<service=ResourceManager, name=QueueMetrics, q0=(\w+), q1=(\w+), q2=(\w+), user=(.*)><>(\w+)'
22+
name: hadoop_resourcemanager_queue_metrics_$5
23+
attrNameSnakeCase: true
24+
labels:
25+
"q0": "$1"
26+
"q1": "$2"
27+
"q2": "$3"
28+
"user": "$4"
29+
- pattern: 'Hadoop<service=ResourceManager, name=QueueMetrics, q0=(\w+), q1=(\w+), q2=(\w+)><>(\w+)'
30+
name: hadoop_resourcemanager_queue_metrics_$4
31+
attrNameSnakeCase: true
32+
labels:
33+
"q0": "$1"
34+
"q1": "$2"
35+
"q2": "$3"
36+
- pattern: 'Hadoop<service=ResourceManager, name=QueueMetrics, q0=(\w+), q1=(\w+), user=(.*)><>(\w+)'
37+
name: hadoop_resourcemanager_queue_metrics_$4
38+
attrNameSnakeCase: true
39+
labels:
40+
"q0": "$1"
41+
"q1": "$2"
42+
"user": "$3"
43+
- pattern: 'Hadoop<service=ResourceManager, name=QueueMetrics, q0=(\w+), q1=(\w+)><>(\w+)'
44+
name: hadoop_resourcemanager_queue_metrics_$3
45+
attrNameSnakeCase: true
46+
labels:
47+
"q0": "$1"
48+
"q1": "$2"
49+
- pattern: 'Hadoop<service=ResourceManager, name=QueueMetrics, q0=(\w+), user=(\w+)><>(\w+)'
50+
name: hadoop_resourcemanager_queue_metrics_$3
51+
attrNameSnakeCase: true
52+
labels:
53+
"q0": "$1"
54+
"user": "$2"
55+
- pattern: 'Hadoop<service=ResourceManager, name=QueueMetrics, q0=(\w+)><>(\w+)'
56+
name: hadoop_resourcemanager_queue_metrics_$2
57+
attrNameSnakeCase: true
58+
labels:
59+
"q0": "$1"
60+
- pattern: 'Hadoop<service=ResourceManager, name=(\w+)><>(\w+)'
61+
name: hadoop_resourcemanager_$1_$2
62+
attrNameSnakeCase: true
63+
help: "Common config, parse from jmx bean: {Hadoop:service=ResourceManager, name=$1}.$2"

0 commit comments

Comments
 (0)