From 404e93fbdcce108df205472ce2da7301cd602e72 Mon Sep 17 00:00:00 2001 From: mrproliu <741550557@qq.com> Date: Wed, 7 Sep 2022 20:07:08 +0800 Subject: [PATCH] Add HTTP2 protocol identify (#51) --- .github/workflows/rover.yaml | 2 + bpf/profiling/network/protocol_analyze.h | 81 ++++++++++++++++ docs/en/setup/configuration/profiling.md | 13 +-- .../cases/profiling/network/base/nginx.conf | 14 +++ .../cases/profiling/network/http2/Dockerfile | 38 ++++++++ .../network/http2/docker-compose.yml | 64 ++++++++++++ .../cases/profiling/network/http2/e2e.yaml | 45 +++++++++ test/e2e/cases/profiling/network/http2/go.mod | 33 +++++++ .../e2e/cases/profiling/network/http2/grpc.go | 97 +++++++++++++++++++ .../profiling/network/http2/service.proto | 36 +++++++ 10 files changed, 417 insertions(+), 6 deletions(-) create mode 100644 test/e2e/cases/profiling/network/http2/Dockerfile create mode 100644 test/e2e/cases/profiling/network/http2/docker-compose.yml create mode 100644 test/e2e/cases/profiling/network/http2/e2e.yaml create mode 100644 test/e2e/cases/profiling/network/http2/go.mod create mode 100644 test/e2e/cases/profiling/network/http2/grpc.go create mode 100644 test/e2e/cases/profiling/network/http2/service.proto diff --git a/.github/workflows/rover.yaml b/.github/workflows/rover.yaml index 72f1a430..6011c4cc 100644 --- a/.github/workflows/rover.yaml +++ b/.github/workflows/rover.yaml @@ -163,6 +163,8 @@ jobs: config: test/e2e/cases/profiling/network/c_plus_plus/e2e.yaml - name: Nodejs Profiling config: test/e2e/cases/profiling/network/nodejs/e2e.yaml + - name: HTTP2 Profiling + config: test/e2e/cases/profiling/network/http2/e2e.yaml steps: - uses: actions/checkout@v2 with: diff --git a/bpf/profiling/network/protocol_analyze.h b/bpf/profiling/network/protocol_analyze.h index 16f122cc..6f7e2c2f 100644 --- a/bpf/profiling/network/protocol_analyze.h +++ b/bpf/profiling/network/protocol_analyze.h @@ -111,6 +111,85 @@ static __inline __u32 infer_http_message(const char* buf, size_t count) { return kUnknown; } +// frame format: https://www.rfc-editor.org/rfc/rfc7540.html#section-4.1 +static __inline __u32 infer_http2_message(const char* buf_src, size_t count) { +static const uint8_t kFrameBasicSize = 0x9; // including Length, Type, Flags, Reserved, Stream Identity +static const uint8_t kFrameTypeHeader = 0x1; // the type of the frame: https://www.rfc-editor.org/rfc/rfc7540.html#section-6.2 +static const uint8_t kFrameLoopCount = 5; + +static const uint8_t kStaticTableMaxSize = 61;// https://www.rfc-editor.org/rfc/rfc7541#appendix-A +static const uint8_t kStaticTableAuth = 1; +static const uint8_t kStaticTableGet = 2; +static const uint8_t kStaticTablePost = 3; +static const uint8_t kStaticTablePath1 = 4; +static const uint8_t kStaticTablePath2 = 5; + + // the buffer size must bigger than basic frame size + if (count < kFrameBasicSize) { + return kUnknown; + } + + // frame info + __u8 frame[21] = { 0 }; + __u32 frameOffset = 0; + // header info + __u8 staticInx, headerBlockFragmentOffset; + + // each all frame +#pragma unroll + for (__u8 i = 0; i < kFrameLoopCount; i++) { + if (frameOffset >= count) { + break; + } + + // read frame + bpf_probe_read(frame, sizeof(frame), buf_src + frameOffset); + frameOffset += (bpf_ntohl(*(__u32 *) frame) >> 8) + kFrameBasicSize; + + // is header frame + if (frame[3] != kFrameTypeHeader) { + continue; + } + + // validate the header(unset): not HTTP2 protocol + // this frame must is a send request + if ((frame[4] & 0xd2) || frame[5] & 0x01) { + return kUnknown; + } + + // locate the header block fragment offset + headerBlockFragmentOffset = kFrameBasicSize; + if (frame[4] & 0x20) { // PADDED flag is set + headerBlockFragmentOffset += 1; + } + if (frame[4] & 0x20) { // PRIORITY flag is set + headerBlockFragmentOffset += 5; + } + +#pragma unroll + for (__u8 j = 0; j <= kStaticTablePath2; j++) { + if (headerBlockFragmentOffset > count) { + return kUnknown; + } + staticInx = frame[headerBlockFragmentOffset] & 0x7f; + if (staticInx <= kStaticTableMaxSize && staticInx > 0) { + if (staticInx == kStaticTableAuth || + staticInx == kStaticTableGet || + staticInx == kStaticTablePost || + staticInx == kStaticTablePath1 || + staticInx == kStaticTablePath2) { + return kRequest; + } else { + return kResponse; + } + } + headerBlockFragmentOffset++; + } + } + + return kUnknown; +} + // Cassandra frame: // 0 8 16 24 32 40 // +---------+---------+---------+---------+---------+ @@ -678,6 +757,8 @@ static __inline enum message_type_t analyze_protocol(char *buf, __u32 count, str // PROTOCOL_LIST: Requires update on new protocols. if ((inferred_message.type = infer_http_message(buf, count)) != kUnknown) { inferred_message.protocol = kProtocolHTTP; + } else if ((inferred_message.type = infer_http2_message(buf, count)) != kUnknown) { + inferred_message.protocol = kProtocolHTTP2; } else if ((inferred_message.type = infer_cql_message(buf, count)) != kUnknown) { inferred_message.protocol = kProtocolCQL; } else if ((inferred_message.type = infer_mongo_message(buf, count)) != kUnknown) { diff --git a/docs/en/setup/configuration/profiling.md b/docs/en/setup/configuration/profiling.md index 78d6fa63..a63b92b3 100644 --- a/docs/en/setup/configuration/profiling.md +++ b/docs/en/setup/configuration/profiling.md @@ -32,12 +32,13 @@ Off CPU Profiling task is attach the `finish_task_switch` in `krobe` to profilin Network Profiling task is intercept IO-related syscall and `urprobe` in process to identify the network traffic and generate the metrics. Also, the following protocol are supported for analyzing using OpenSSL library, BoringSSL library, GoTLS, NodeTLS or plaintext: -1. HTTP -2. MySQL -3. CQL(The Cassandra Query Language) -4. MongoDB -5. Kafka -6. DNS +1. HTTP/1.x +2. HTTP/2 +3. MySQL +4. CQL(The Cassandra Query Language) +5. MongoDB +6. Kafka +7. DNS #### Metrics diff --git a/test/e2e/cases/profiling/network/base/nginx.conf b/test/e2e/cases/profiling/network/base/nginx.conf index 2de1521e..57acaf56 100644 --- a/test/e2e/cases/profiling/network/base/nginx.conf +++ b/test/e2e/cases/profiling/network/base/nginx.conf @@ -32,4 +32,18 @@ http { proxy_http_version 1.1; } } + + server { + listen 9000 ssl http2; + server_name proxy; + include mime.types; + default_type application/octet-stream; + + ssl_certificate /ssl_data/proxy.crt; + ssl_certificate_key /ssl_data/proxy.key; + + location / { + grpc_pass grpcs://service:9000; + } + } } \ No newline at end of file diff --git a/test/e2e/cases/profiling/network/http2/Dockerfile b/test/e2e/cases/profiling/network/http2/Dockerfile new file mode 100644 index 00000000..b72b7b3e --- /dev/null +++ b/test/e2e/cases/profiling/network/http2/Dockerfile @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.17 + +COPY http2/ /service + +WORKDIR / +RUN apt update && apt install -y zip && \ + mkdir protoc && cd protoc && \ + curl -sL https://github.com/protocolbuffers/protobuf/releases/download/v21.5/protoc-21.5-linux-x86_64.zip -o protoc.zip && \ + unzip protoc.zip + +WORKDIR /service + +RUN go get -u google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0 && \ + go get -u google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0 && \ + go get google.golang.org/grpc/internal/transport@v1.44.0 && \ + /protoc/bin/protoc --go_out=. --go-grpc_out=. service.proto && \ + go build . + +COPY base/ssl /usr/local/share/ca-certificates +RUN update-ca-certificates + +CMD ["/service/test"] diff --git a/test/e2e/cases/profiling/network/http2/docker-compose.yml b/test/e2e/cases/profiling/network/http2/docker-compose.yml new file mode 100644 index 00000000..6b944432 --- /dev/null +++ b/test/e2e/cases/profiling/network/http2/docker-compose.yml @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2.1' + +services: + service: + build: + context: ../ + dockerfile: http2/Dockerfile + volumes: + - ./../base/ssl:/ssl_data + networks: + - e2e + ports: + - 8080:8080 + healthcheck: + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/8080" ] + interval: 5s + timeout: 60s + retries: 120 + + proxy: + extends: + file: ../base/docker-compose.yml + service: proxy + networks: + - e2e + depends_on: + service: + condition: service_healthy + + oap: + extends: + file: ../base/docker-compose.yml + service: oap + ports: + - 12800:12800 + + rover: + extends: + file: ../base/docker-compose.yml + service: rover + environment: + ROVER_PROCESS_DISCOVERY_REGEX_SCANNER_MATCH_CMD2: service/test + ROVER_PROCESS_DISCOVERY_REGEX_SCANNER_PROCESS_NAME2: service + depends_on: + oap: + condition: service_healthy + +networks: + e2e: \ No newline at end of file diff --git a/test/e2e/cases/profiling/network/http2/e2e.yaml b/test/e2e/cases/profiling/network/http2/e2e.yaml new file mode 100644 index 00000000..b6e80b6f --- /dev/null +++ b/test/e2e/cases/profiling/network/http2/e2e.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +setup: + env: compose + file: docker-compose.yml + timeout: 20m + init-system-environment: ../../../../base/env + steps: + - name: set PATH + command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash test/e2e/base/scripts/prepare/setup-e2e-shell/install.sh yq + - name: install swctl + command: bash test/e2e/base/scripts/prepare/setup-e2e-shell/install.sh swctl + +trigger: + action: http + interval: 3s + times: 10 + url: http://${service_host}:${service_8080}/singleCall + method: GET + +verify: + # verify with retry strategy + retry: + # max retry count + count: 20 + # the interval between two retries, in millisecond. + interval: 10s + cases: + - includes: + - ../network-cases.yaml \ No newline at end of file diff --git a/test/e2e/cases/profiling/network/http2/go.mod b/test/e2e/cases/profiling/network/http2/go.mod new file mode 100644 index 00000000..d49a2939 --- /dev/null +++ b/test/e2e/cases/profiling/network/http2/go.mod @@ -0,0 +1,33 @@ +// Licensed to Apache Software Foundation (ASF) under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Apache Software Foundation (ASF) licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module test + +go 1.17 + +require ( + google.golang.org/grpc v1.44.0 + google.golang.org/protobuf v1.25.0 +) + +require ( + github.com/golang/protobuf v1.4.3 // indirect + golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect + golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd // indirect + golang.org/x/text v0.3.0 // indirect + google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect +) diff --git a/test/e2e/cases/profiling/network/http2/grpc.go b/test/e2e/cases/profiling/network/http2/grpc.go new file mode 100644 index 00000000..f6e02370 --- /dev/null +++ b/test/e2e/cases/profiling/network/http2/grpc.go @@ -0,0 +1,97 @@ +// Licensed to Apache Software Foundation (ASF) under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Apache Software Foundation (ASF) licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "context" + "fmt" + "log" + "net" + "net/http" + "test/service" + + "google.golang.org/grpc/credentials" + + "google.golang.org/grpc" +) + +var ( + httpPort = 8080 + gRPCPort = 9000 + gRPCConn *grpc.ClientConn +) + +type Provider struct { + service.UnimplementedServiceServer +} + +func singleCall(w http.ResponseWriter, req *http.Request) { + log.Printf("receive the single call request") + if gRPCConn == nil { + c, err := credentials.NewClientTLSFromFile("/ssl_data/proxy.crt", "proxy") + if err != nil { + log.Fatalf("credentials.NewClientTLSFromFile err: %v", err) + } + + dial, err := grpc.Dial(fmt.Sprintf("proxy:%d", gRPCPort), grpc.WithTransportCredentials(c)) + if err != nil { + log.Printf("init gRPC client failure: %v", err) + _, _ = w.Write([]byte("error")) + return + } + gRPCConn = dial + } + + client := service.NewServiceClient(gRPCConn) + resp, err := client.SingleCall(context.Background(), &service.CallRequest{}) + if err != nil { + log.Printf("send single call request failure: %v", err) + _, _ = w.Write([]byte("error")) + return + } + + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte(resp.Message)) +} + +func (p *Provider) SingleCall(context.Context, *service.CallRequest) (*service.CallReply, error) { + return &service.CallReply{Message: "response success"}, nil +} + +func main() { + c, err := credentials.NewServerTLSFromFile("/ssl_data/service.crt", "/ssl_data/service.key") + if err != nil { + log.Fatalf("credentials.NewServerTLSFromFile err: %v", err) + } + server := grpc.NewServer(grpc.Creds(c)) + service.RegisterServiceServer(server, &Provider{}) + listen, err := net.Listen("tcp", fmt.Sprintf(":%d", gRPCPort)) + if err != nil { + log.Fatalf("listen gRPC port failure: %v", err) + return + } + go func() { + if err := server.Serve(listen); err != nil { + log.Fatalf("startup gRPC server failure") + } + }() + + http.HandleFunc("/singleCall", singleCall) + err1 := http.ListenAndServe(fmt.Sprintf(":%d", httpPort), nil) + log.Fatal(err1) +} diff --git a/test/e2e/cases/profiling/network/http2/service.proto b/test/e2e/cases/profiling/network/http2/service.proto new file mode 100644 index 00000000..88ccc648 --- /dev/null +++ b/test/e2e/cases/profiling/network/http2/service.proto @@ -0,0 +1,36 @@ +// Licensed to Apache Software Foundation (ASF) under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Apache Software Foundation (ASF) licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +syntax = "proto3"; + +option go_package="./service;"; + +// The greeting service definition. +service Service { + // Sends a greeting + rpc SingleCall(CallRequest) returns (CallReply) {} +} + +// The request message containing the user's name. +message CallRequest { + string name = 1; +} + +// The response message containing the greetings +message CallReply { + string message = 1; +} \ No newline at end of file