Add Network profiling documentation (#49)

docs/en/setup/configuration/profiling.md

@@ -39,6 +39,51 @@ Also, the following protocol are supported for analyzing using OpenSSL library,
 5. Kafka
 6. DNS
 
+#### Metrics
+
+Network profiling uses metrics send data to the backend service.
+
+##### Data Type
+
+The network profiling has customized the following two types of metrics to represent the network data:
+1. **Counter**: Records the total number of data in a certain period of time. Each counter containers the following data:
+   1. **Count**: The count of the execution.
+   2. **Bytes**: The package size of the execution.
+   3. **Exe Time**: The consumed time(nanosecond) of the execution. 
+2. **Histogram**: Records the distribution of the data in the bucket.
+
+##### Labels
+
+Each metric contains the following labels to identify the process relationship:
+
+| Name | Type | Description |
+|------|------|-------------|
+|client_process_id or server_process_id| string | The ID of the current process, which is determined by the role of the current process in the connection as server or client. |
+|client_local or server_local| boolean | The remote process is a local process. |
+|client_address or server_address| string | The remote process address. ex: `IP:port`. |
+|side| enum | The current process is either "client" or "server" in this connection. |
+|protocol| string | Identification the protocol based on the package data content. |
+|is_ssl| bool | Is the current connection using SSL. |
+
+##### Data
+
+Based on the above two data types, the following metrics are provided. 
+
+| Name | Type| Unit | Description |
+|------|-----|------|-------------|
+|write|Counter|nanosecond|The socket write counter|
+|read|Counter|nanosecond|The socket read counter|
+|write RTT|Counter|microsecond|The socket write RTT counter|
+|connect|Counter|nanosecond|The socket connect/accept with other server/client counter|
+|close|Counter|nanosecond|The socket close counter|
+|retransmit|Counter|nanosecond|The socket retransmit package counter|
+|drop|Counter|nanosecond|The socket drop package counter|
+|write RTT|Histogram|microsecond|The socket write RTT execute time histogram|
+|write execute time|Histogram|nanosecond|The socket write data execute time histogram|
+|read execute time|Histogram|nanosecond|The socket read data execute time histogram|
+|connect execute time|Histogram|nanosecond|The socket connect/accept with other server/client execute time histogram|
+|close execute time|Histogram|nanosecond|The socket close execute time histogram|
+
 ## Configuration
 
 | Name | Default | Environment Key | Description |

pkg/profiling/task/network/analyzer.go

@@ -179,11 +179,11 @@ func (t *TrafficAnalyzer) generateOrCombineTraffic(traffic *ProcessTraffic, con
 			CloseCounter:            NewSocketDataCounter(),
 			RetransmitCounter:       NewSocketDataCounter(),
 			DropCounter:             NewSocketDataCounter(),
-			WriteRTTHistogram:       NewSocketDataHistogram(),
-			WriteExeTimeHistogram:   NewSocketDataHistogram(),
-			ReadExeTimeHistogram:    NewSocketDataHistogram(),
-			ConnectExeTimeHistogram: NewSocketDataHistogram(),
-			CloseExeTimeHistogram:   NewSocketDataHistogram(),
+			WriteRTTHistogram:       NewSocketDataHistogram(HistogramDataUnitUS),
+			WriteExeTimeHistogram:   NewSocketDataHistogram(HistogramDataUnitNS),
+			ReadExeTimeHistogram:    NewSocketDataHistogram(HistogramDataUnitNS),
+			ConnectExeTimeHistogram: NewSocketDataHistogram(HistogramDataUnitNS),
+			CloseExeTimeHistogram:   NewSocketDataHistogram(HistogramDataUnitNS),
 		}
 	}
 	if len(traffic.LocalProcesses) == 0 && len(con.LocalProcesses) > 0 {
@@ -401,8 +401,8 @@ func (t *TrafficAnalyzer) findRemotePidWhenMeshEnvironment(con *ConnectionContex
 			}
 			continue
 		}
-		// if current is mesh application, and remote address is not local and dns, them it's must be sent to the MESH_DP
-		if localProcess.Entity().Layer == layerMeshApp && con.RemotePort != 53 &&
+		// if current is mesh application, them it's must be sent to the MESH_DP
+		if localProcess.Entity().Layer == layerMeshApp &&
 			len(t.localAddresses[con.RemoteIP]) == 0 && !tools.IsLocalHostAddress(con.RemoteIP) {
 			if envoyPid := t.findSameInstanceMeshDP(localProcess.Entity()); envoyPid != 0 {
 				log.Debugf("found in the mesh data plane, remote ip: %s", con.RemoteIP)

pkg/profiling/task/network/context.go

@@ -535,9 +535,9 @@ func (c *Context) newConnectionContext(conID, randomID uint64, pid, fd uint32, p
 		WriteCounter:          NewSocketDataCounterWithHistory(),
 		ReadCounter:           NewSocketDataCounterWithHistory(),
 		WriteRTTCounter:       NewSocketDataCounterWithHistory(),
-		WriteRTTHistogram:     NewSocketDataHistogramWithHistory(),
-		WriteExeTimeHistogram: NewSocketDataHistogramWithHistory(),
-		ReadExeTimeHistogram:  NewSocketDataHistogramWithHistory(),
+		WriteRTTHistogram:     NewSocketDataHistogramWithHistory(HistogramDataUnitUS),
+		WriteExeTimeHistogram: NewSocketDataHistogramWithHistory(HistogramDataUnitNS),
+		ReadExeTimeHistogram:  NewSocketDataHistogramWithHistory(HistogramDataUnitNS),
 	}
 }
 

pkg/profiling/task/network/metrics.go

@@ -81,17 +81,24 @@ func (c *SocketDataCounterWithHistory) CalculateIncrease() *SocketDataCounter {
 	}
 }
 
-// SocketHistogramBuckets means the histogram bucket: 0ms, 0.01ms, 0.05ms, 0.1ms, 0.5ms, 1ms, 1.2ms, 1.5ms, 1.7ms, 2ms,
+// SocketHistogramBucketsNs means the histogram bucket: 0ms, 0.01ms, 0.05ms, 0.1ms, 0.5ms, 1ms, 1.2ms, 1.5ms, 1.7ms, 2ms,
 // 2.5ms, 3ms, 5ms, 7ms, 10ms, 13ms, 16ms, 20ms, 25ms, 30ms, 35ms, 40ms, 45ms, 50ms, 70ms, 100ms, 150ms,
 // 200ms, 300ms, 500ms, 1s, 2s, 3s, 5s
-// value unit: us
-var SocketHistogramBuckets = []float64{0, 10, 50, 100, 500, 1000, 1200, 1500, 1700, 2000,
+// value unit: ns
+var SocketHistogramBucketsNs = []float64{0, 10000, 50000, 100000, 500000, 1000000, 1200000, 1500000, 1700000, 2000000,
+	2500000, 3000000, 5000000, 7000000, 10000000, 13000000, 16000000, 20000000, 25000000, 30000000, 35000000, 40000000,
+	45000000, 50000000, 70000000, 100000000, 150000000, 200000000, 300000000, 500000000, 1000000000, 2000000000,
+	3000000000, 5000000000}
+
+// SocketHistogramBucketsUs same with SocketHistogramBucketsNs, but the value unit: us
+var SocketHistogramBucketsUs = []float64{0, 10, 50, 100, 500, 1000, 1200, 1500, 1700, 2000,
 	2500, 3000, 5000, 7000, 10000, 13000, 16000, 20000, 25000, 30000, 35000, 40000,
 	45000, 50000, 70000, 100000, 150000, 200000, 300000, 500000, 1000000, 2000000,
 	3000000, 5000000}
-var SocketHistogramBucketsCount = len(SocketHistogramBuckets)
+var SocketHistogramBucketsCount = len(SocketHistogramBucketsNs)
 
 type SocketDataHistogram struct {
+	Unit    HistogramDataUnit
 	Buckets map[uint64]uint32
 }
 
@@ -113,13 +120,13 @@ func (h *SocketDataHistogram) Increase(other *SocketDataHistogram) {
 
 func (h *SocketDataHistogram) IncreaseByValue(val uint64) {
 	floatVal := float64(val)
-	for inx, curVal := range SocketHistogramBuckets {
+	for inx, curVal := range SocketHistogramBucketsNs {
 		if inx > 0 && curVal > floatVal {
 			h.Buckets[uint64(inx-1)]++
 			return
 		}
 	}
-	h.Buckets[uint64(len(SocketHistogramBuckets)-1)]++
+	h.Buckets[uint64(len(SocketHistogramBucketsNs)-1)]++
 }
 
 func (h *SocketDataHistogram) NotEmpty() bool {
@@ -131,25 +138,33 @@ func (h *SocketDataHistogram) NotEmpty() bool {
 	return false
 }
 
-func NewSocketDataHistogram() *SocketDataHistogram {
+func NewSocketDataHistogram(unit HistogramDataUnit) *SocketDataHistogram {
 	buckets := make(map[uint64]uint32, SocketHistogramBucketsCount)
 	for i := 0; i < SocketHistogramBucketsCount; i++ {
 		buckets[uint64(i)] = 0
 	}
 	return &SocketDataHistogram{
+		Unit:    unit,
 		Buckets: buckets,
 	}
 }
 
+type HistogramDataUnit int
+
+const (
+	HistogramDataUnitNS HistogramDataUnit = 1
+	HistogramDataUnitUS HistogramDataUnit = 2
+)
+
 type SocketDataHistogramWithHistory struct {
 	Pre *SocketDataHistogram
 	Cur *SocketDataHistogram
 }
 
-func NewSocketDataHistogramWithHistory() *SocketDataHistogramWithHistory {
+func NewSocketDataHistogramWithHistory(unit HistogramDataUnit) *SocketDataHistogramWithHistory {
 	return &SocketDataHistogramWithHistory{
-		Pre: NewSocketDataHistogram(),
-		Cur: NewSocketDataHistogram(),
+		Pre: NewSocketDataHistogram(unit),
+		Cur: NewSocketDataHistogram(unit),
 	}
 }
 
@@ -163,7 +178,7 @@ func (h *SocketDataHistogramWithHistory) UpdateToCurrent(bucket uint64, val uint
 }
 
 func (h *SocketDataHistogramWithHistory) CalculateIncrease() *SocketDataHistogram {
-	histogram := NewSocketDataHistogram()
+	histogram := NewSocketDataHistogram(h.Cur.Unit)
 	var increaseVal uint32
 	for curK, curV := range h.Cur.Buckets {
 		if increaseVal = curV - h.Pre.Buckets[curK]; increaseVal > 0 {
@@ -298,8 +313,14 @@ func (r *ProcessTraffic) appendHistogramValue(metrics []*v3.MeterData, metricsPr
 		if bucketInx >= SocketHistogramBucketsCount {
 			bucketInx = SocketHistogramBucketsCount - 1
 		}
+		var buckets []float64
+		if histogram.Unit == HistogramDataUnitUS {
+			buckets = SocketHistogramBucketsUs
+		} else {
+			buckets = SocketHistogramBucketsNs
+		}
 		values = append(values, &v3.MeterBucketValue{
-			Bucket: SocketHistogramBuckets[bucketInx],
+			Bucket: buckets[bucketInx],
 			Count:  int64(count),
 		})
 	}

pkg/profiling/task/network/runner.go

@@ -237,7 +237,7 @@ func (r *Runner) logTheMetricsConnections(traffices []*ProcessTraffic) {
 				traffic.RemoteIP, traffic.RemotePort, traffic.RemotePid)
 		}
 		side := traffic.ConnectionRole.String()
-		log.Debugf("connection analyze result: %s : %s -> %s, protocol: %s, is SSL: %t, read: %d bytes/%d, write: %d bytes/%d",
+		log.Debugf("connection analyze result: %s : %s -> %s, protocol: %s, is SSL: %t, write: %d bytes/%d, read: %d bytes/%d",
 			side, localInfo, remoteInfo, traffic.Protocol.String(), traffic.IsSSL, traffic.WriteCounter.Bytes, traffic.WriteCounter.Count,
 			traffic.ReadCounter.Bytes, traffic.ReadCounter.Count)
 	}