Support Off CPU Profiling E2E test (#36)

* Support Off CPU Profiling E2E test

* Fix C++ Off CPU Profiling

.github/workflows/rover.yaml

@@ -75,14 +75,23 @@ jobs:
       fail-fast: false
       matrix:
         test:
-          - name: Golang Profiling
-            config: test/e2e/cases/profiling/golang/e2e.yaml
-          - name: C++ Profiling
-            config: test/e2e/cases/profiling/c++/e2e.yaml
-          - name: C Profiling
-            config: test/e2e/cases/profiling/c/e2e.yaml
-          - name: Rust Profiling
-            config: test/e2e/cases/profiling/rust/e2e.yaml
+          - name: Golang On CPU Profiling
+            config: test/e2e/cases/profiling/oncpu/golang/e2e.yaml
+          - name: C++ On CPU Profiling
+            config: test/e2e/cases/profiling/oncpu/c++/e2e.yaml
+          - name: C On CPU Profiling
+            config: test/e2e/cases/profiling/oncpu/c/e2e.yaml
+          - name: Rust On CPU Profiling
+            config: test/e2e/cases/profiling/oncpu/rust/e2e.yaml
+
+          - name: Golang Off CPU Profiling
+            config: test/e2e/cases/profiling/offcpu/golang/e2e.yaml
+          - name: C++ Off CPU Profiling
+            config: test/e2e/cases/profiling/offcpu/c++/e2e.yaml
+          - name: C Off CPU Profiling
+            config: test/e2e/cases/profiling/offcpu/c/e2e.yaml
+          - name: Rust Off CPU Profiling
+            config: test/e2e/cases/profiling/offcpu/rust/e2e.yaml
 
           - name: go2sky Agent Sensor
             config: test/e2e/cases/process/agent_sensor/golang/e2e.yaml

pkg/profiling/task/offcpu/runner.go

@@ -61,7 +61,7 @@ type Runner struct {
 	kernelProfiling  *profiling.Info
 
 	// runtime
-	previousStacks  map[ProcessStack]*StackCounter
+	previousStacks  map[ProcessStack]StackCounter
 	bpf             *bpfObjects
 	kprobe          link.Link
 	stopChan        chan bool
@@ -82,7 +82,7 @@ func (r *Runner) Init(task *base.ProfilingTask, process api.ProcessInterface) er
 		log.Warnf("could not analyze kernel profiling stats: %v", err)
 	}
 	r.kernelProfiling = kernelProfiling
-	r.previousStacks = make(map[ProcessStack]*StackCounter)
+	r.previousStacks = make(map[ProcessStack]StackCounter)
 	r.stopChan = make(chan bool, 1)
 	return nil
 }
@@ -153,9 +153,6 @@ func (r *Runner) FlushData() ([]*v3.EBPFProfilingData, error) {
 	if r.bpf == nil {
 		return nil, nil
 	}
-	if r.flushDataNotify == nil {
-		return nil, nil
-	}
 	var stack ProcessStack
 	var counter StackCounter
 	iterate := r.bpf.Counts.Iterate()
@@ -183,11 +180,11 @@ func (r *Runner) FlushData() ([]*v3.EBPFProfilingData, error) {
 		switchCount := int32(counter.Times)
 		duration := int64(counter.Deltas)
 		existCounter := r.previousStacks[stack]
-		if existCounter != nil {
+		if existCounter.Times > 0 && existCounter.Deltas > 0 {
 			switchCount -= int32(existCounter.Times)
 			duration -= int64(existCounter.Deltas)
 		}
-		r.previousStacks[stack] = &counter
+		r.previousStacks[stack] = counter
 
 		result = append(result, &v3.EBPFProfilingData{
 			Profiling: &v3.EBPFProfilingData_OffCPU{

pkg/tools/operator/distribution.go

@@ -132,5 +132,7 @@ func normalizeContent(d string) string {
 	d = strings.TrimSpace(d)
 	d = strings.ToLower(d)
 	d = strings.ReplaceAll(d, "-", "_")
+	d = strings.TrimPrefix(d, "\"")
+	d = strings.TrimSuffix(d, "\"")
 	return d
 }

test/e2e/base/env

@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-SW_CTL_COMMIT=03dbdcf8cecc3abdef661efaa5734c01ac49adea
-SW_OAP_COMMIT=91b94cfc105276897c82e2d0f8d4bb8bd3952660
+SW_CTL_COMMIT=bdcba1c93b1ad702a88f30b798df5e99d963689d
+SW_OAP_COMMIT=47509715f5a76aef33d0987be6302f7b31315bf0
 SW_KUBERNETES_COMMIT_SHA=0f3ec68e5a7e1608cec8688716b848ed15e971e5
 
 SW_AGENT_GO_COMMIT=216f122d942cb683f48578d3014cc5ea83637582

test/e2e/cases/profiling/offcpu/c++/Dockerfile.file

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcc:latest
+
+WORKDIR /
+COPY file.cpp /file.cpp
+RUN  g++ file.cpp -o file
+
+CMD ["/file"]

test/e2e/cases/profiling/offcpu/c++/docker-compose.yml

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '2.1'
+
+services:
+  oap:
+    extends:
+      file: ../../../../base/base-compose.yml
+      service: oap
+    ports:
+      - 12800:12800
+
+  rover:
+    extends:
+      file: ../../../../base/base-compose.yml
+      service: rover
+    environment:
+      ROVER_PROCESS_DISCOVERY_REGEX_SCANNER_MATCH_CMD: /file
+      ROVER_PROCESS_DISCOVERY_REGEX_SCANNER_SERVICE_NAME: file
+    depends_on:
+      oap:
+        condition: service_healthy
+
+  file_c_plus_plus:
+    build:
+      context: .
+      dockerfile: Dockerfile.file
+    networks:
+      - e2e
+
+networks:
+  e2e:

test/e2e/cases/profiling/offcpu/c++/e2e.yaml

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+setup:
+  env: compose
+  file: docker-compose.yml
+  timeout: 20m
+  init-system-environment: ../../../../base/env
+  steps:
+    - name: set PATH
+      command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH
+    - name: install yq
+      command: bash test/e2e/base/scripts/prepare/setup-e2e-shell/install.sh yq
+    - name: install swctl
+      command: bash test/e2e/base/scripts/prepare/setup-e2e-shell/install.sh swctl
+
+verify:
+  # verify with retry strategy
+  retry:
+    # max retry count
+    count: 20
+    # the interval between two retries, in millisecond.
+    interval: 10s
+  cases:
+    - includes:
+        - ../profiling-cases.yaml
+    - query: |
+        taskid=$(swctl --base-url=http://${oap_host}:${oap_12800}/graphql --display yaml profiling ebpf list --service-name file |yq e '.[0].taskid' -)
+        scheduleid=$(swctl --base-url=http://${oap_host}:${oap_12800}/graphql --display yaml profiling ebpf schedules --task-id=$taskid |yq e '.[0].scheduleid' -);
+        start=$(swctl --base-url=http://${oap_host}:${oap_12800}/graphql --display yaml profiling ebpf schedules --task-id=$taskid | yq e '.[0].starttime' -)
+        end=$(swctl --base-url=http://${oap_host}:${oap_12800}/graphql --display yaml profiling ebpf schedules --task-id=$taskid | yq e '.[0].endtime' -)
+        swctl --base-url=http://${oap_host}:${oap_12800}/graphql --display yaml profiling ebpf analysis --schedule-id=$scheduleid --time-ranges=$start-$end
+      expected: profiling-analysis.yml

test/e2e/cases/profiling/offcpu/c++/file.cpp

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iostream>
+#include <fstream>
+using namespace std;
+
+void fileCPlusPlus()
+{
+    for (;;)
+    {
+        ofstream myfile;
+        myfile.open ("/tmp/skywalking-rover-e2e-test");
+        myfile << "e2e test.\n";
+        myfile.close();
+    }
+}
+
+int main()
+{
+    fileCPlusPlus();
+    return 0;
+}

test/e2e/cases/profiling/offcpu/c++/profiling-analysis.yml

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+tip: null
+trees:
+{{- contains .trees }}
+- elements:
+  {{- contains .elements }}
+  - id: "{{ notEmpty .id }}"
+    parentid: "{{ notEmpty .parentid }}"
+    symbol: "fileCPlusPlus()"
+    stacktype: USER_SPACE
+    dumpcount: {{ gt .dumpcount 0 }}
+{{- end }}
+{{- end }}

test/e2e/cases/profiling/offcpu/c/Dockerfile.file

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcc:latest
+
+WORKDIR /
+COPY file.c /file.c
+RUN gcc file.c -o file
+
+CMD ["/file"]

test/e2e/cases/profiling/offcpu/c/docker-compose.yml

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '2.1'
+
+services:
+  oap:
+    extends:
+      file: ../../../../base/base-compose.yml
+      service: oap
+    ports:
+      - 12800:12800
+
+  rover:
+    extends:
+      file: ../../../../base/base-compose.yml
+      service: rover
+    environment:
+      ROVER_PROCESS_DISCOVERY_REGEX_SCANNER_MATCH_CMD: /file
+      ROVER_PROCESS_DISCOVERY_REGEX_SCANNER_SERVICE_NAME: file
+    depends_on:
+      oap:
+        condition: service_healthy
+
+  file_c:
+    build:
+      context: .
+      dockerfile: Dockerfile.file
+    networks:
+      - e2e
+
+networks:
+  e2e:

test/e2e/cases/profiling/offcpu/c/e2e.yaml

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+setup:
+  env: compose
+  file: docker-compose.yml
+  timeout: 20m
+  init-system-environment: ../../../../base/env
+  steps:
+    - name: set PATH
+      command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH
+    - name: install yq
+      command: bash test/e2e/base/scripts/prepare/setup-e2e-shell/install.sh yq
+    - name: install swctl
+      command: bash test/e2e/base/scripts/prepare/setup-e2e-shell/install.sh swctl
+
+verify:
+  # verify with retry strategy
+  retry:
+    # max retry count
+    count: 20
+    # the interval between two retries, in millisecond.
+    interval: 10s
+  cases:
+    - includes:
+        - ../profiling-cases.yaml
+    - query: |
+        taskid=$(swctl --base-url=http://${oap_host}:${oap_12800}/graphql --display yaml profiling ebpf list --service-name file |yq e '.[0].taskid' -)
+        scheduleid=$(swctl --base-url=http://${oap_host}:${oap_12800}/graphql --display yaml profiling ebpf schedules --task-id=$taskid |yq e '.[0].scheduleid' -);
+        start=$(swctl --base-url=http://${oap_host}:${oap_12800}/graphql --display yaml profiling ebpf schedules --task-id=$taskid | yq e '.[0].starttime' -)
+        end=$(swctl --base-url=http://${oap_host}:${oap_12800}/graphql --display yaml profiling ebpf schedules --task-id=$taskid | yq e '.[0].endtime' -)
+        swctl --base-url=http://${oap_host}:${oap_12800}/graphql --display yaml profiling ebpf analysis --schedule-id=$scheduleid --time-ranges=$start-$end
+      expected: profiling-analysis.yml