From fd83eee6a33bf99c7a19691e7ec1a7c4639a5476 Mon Sep 17 00:00:00 2001 From: zbilun Date: Thu, 12 Dec 2024 00:57:52 -0800 Subject: [PATCH 01/45] Improve the rpc soak test and channel soak test to cover concurrency for Go. --- .idea/.gitignore | 8 + .idea/grpc-go.iml | 10 ++ .idea/misc.xml | 6 + .idea/modules.xml | 8 + .idea/vcs.xml | 40 +++++ interop/client/client.go | 13 +- interop/test_utils.go | 253 ++++++++++++++++++++++++------- interop/xds_federation/client.go | 34 ++++- 8 files changed, 309 insertions(+), 63 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/grpc-go.iml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 000000000000..13566b81b018 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/grpc-go.iml b/.idea/grpc-go.iml new file mode 100644 index 000000000000..25ed3f6e7b6e --- /dev/null +++ b/.idea/grpc-go.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 000000000000..639900d13c61 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 000000000000..cd91612713cb --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 000000000000..fd7702ec990b --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,40 @@ + + + + + + + + + \ No newline at end of file diff --git a/interop/client/client.go b/interop/client/client.go index f50139fe6dbb..727ed6159778 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -79,6 +79,7 @@ var ( soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") + numThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") tlsServerName = flag.String("server_host_override", "", "The server name used to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.") additionalMetadata = flag.String("additional_metadata", "", "Additional metadata to send in each request, as a semicolon-separated list of key:value pairs.") testCase = flag.String("test_case", "large_unary", @@ -261,7 +262,7 @@ func main() { } opts = append(opts, grpc.WithUnaryInterceptor(unaryAddMd), grpc.WithStreamInterceptor(streamingAddMd)) } - conn, err := grpc.Dial(serverAddr, opts...) + conn, err := grpc.NewClient(serverAddr, opts...) if err != nil { logger.Fatalf("Fail to dial: %v", err) } @@ -358,10 +359,16 @@ func main() { interop.DoPickFirstUnary(ctx, tc) logger.Infoln("PickFirstUnary done") case "rpc_soak": - interop.DoSoakTest(ctxWithDeadline, tc, serverAddr, opts, false /* resetChannel */, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond) + interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *numThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, + time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, + interop.UseSharedChannel) logger.Infoln("RpcSoak done") case "channel_soak": - interop.DoSoakTest(ctxWithDeadline, tc, serverAddr, opts, true /* resetChannel */, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond) + interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *numThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, + time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, + func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + return interop.CreateNewChannel(currentChannel, serverAddr, opts) + }) logger.Infoln("ChannelSoak done") case "orca_per_rpc": interop.DoORCAPerRPCTest(ctx, tc) diff --git a/interop/test_utils.go b/interop/test_utils.go index 71d0b0f060be..98bbc74d75da 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -28,6 +28,7 @@ import ( "context" "fmt" "io" + "log" "os" "strings" "sync" @@ -684,98 +685,242 @@ func DoPickFirstUnary(ctx context.Context, tc testgrpc.TestServiceClient) { } } -func doOneSoakIteration(ctx context.Context, tc testgrpc.TestServiceClient, resetChannel bool, serverAddr string, soakRequestSize int, soakResponseSize int, dopts []grpc.DialOption, copts []grpc.CallOption) (latency time.Duration, err error) { - start := time.Now() - client := tc - if resetChannel { - var conn *grpc.ClientConn - conn, err = grpc.Dial(serverAddr, dopts...) +type SoakIterationResult struct { + LatencyMs int64 + Status string // The status of the iteration +} + +type ThreadResults struct { + IterationsDone int + Failures int + Latencies *stats.Histogram +} + +// ChannelFunc can be used to customize how the channel is handled (reuse or reset) +type ChannelFunc func(*grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) + +// createChannel Initialize the shared channel (once for all threads) +func createChannel(serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { + conn, err := grpc.NewClient(serverAddr, dopts...) + if err != nil { + log.Fatalf("Failed to create shared channel: %v", err) + } + client := testgrpc.NewTestServiceClient(conn) + return conn, client +} + +func closeChannel(channel *grpc.ClientConn) { + if channel != nil { + err := channel.Close() if err != nil { - return + log.Fatalf("Failed to close channel: %v", err) } - defer conn.Close() - client = testgrpc.NewTestServiceClient(conn) } - // per test spec, don't include channel shutdown in latency measurement - defer func() { latency = time.Since(start) }() - // do a large-unary RPC +} + +// CreateNewChannel Create a new channel by shutting down the current one (for channel soak tests) +func CreateNewChannel(currentChannel *grpc.ClientConn, serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { + closeChannel(currentChannel) + conn, client := createChannel(serverAddr, dopts) + return conn, client +} + +// UseSharedChannel Reuses the provided currentChannel +func UseSharedChannel(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + client := testgrpc.NewTestServiceClient(currentChannel) + return currentChannel, client +} + +func doOneSoakIteration( + ctx context.Context, + client testgrpc.TestServiceClient, + soakRequestSize int, + soakResponseSize int, + copts []grpc.CallOption) (SoakIterationResult, error) { + start := time.Now() + var err error + // Do a large-unary RPC + // Create the request payload pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, soakRequestSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, ResponseSize: int32(soakResponseSize), Payload: pl, } + // Perform the GRPC call var reply *testpb.SimpleResponse reply, err = client.UnaryCall(ctx, req, copts...) if err != nil { err = fmt.Errorf("/TestService/UnaryCall RPC failed: %s", err) - return + return SoakIterationResult{}, err } + // validate response t := reply.GetPayload().GetType() s := len(reply.GetPayload().GetBody()) if t != testpb.PayloadType_COMPRESSABLE || s != soakResponseSize { err = fmt.Errorf("got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, soakResponseSize) - return + return SoakIterationResult{}, err } - return + // Calculate latency and return result + latency := time.Since(start).Milliseconds() + return SoakIterationResult{ + LatencyMs: latency, + Status: "OK", + }, nil } -// DoSoakTest runs large unary RPCs in a loop for a configurable number of times, with configurable failure thresholds. -// If resetChannel is false, then each RPC will be performed on tc. Otherwise, each RPC will be performed on a new -// stub that is created with the provided server address and dial options. -// TODO(mohanli-ml): Create SoakTestOptions as a parameter for this method. -func DoSoakTest(ctx context.Context, tc testgrpc.TestServiceClient, serverAddr string, dopts []grpc.DialOption, resetChannel bool, soakIterations int, maxFailures int, soakRequestSize int, soakResponseSize int, perIterationMaxAcceptableLatency time.Duration, minTimeBetweenRPCs time.Duration) { - start := time.Now() - var elapsedTime float64 - iterationsDone := 0 - totalFailures := 0 - hopts := stats.HistogramOptions{ - NumBuckets: 20, - GrowthFactor: 1, - BaseBucketSize: 1, - MinValue: 0, - } - h := stats.NewHistogram(hopts) - for i := 0; i < soakIterations; i++ { +func executeSoakTestInThread( + soakIterationsPerThread int, + startNs int64, + minTimeBetweenRPCs time.Duration, + soakRequestSize int, + soakResponseSize int, + perIterationMaxAcceptableLatency time.Duration, + overallTimeoutSeconds int, + serverAddr string, + threadResults *ThreadResults, + mu *sync.Mutex, + ctx context.Context, + sharedChannel *grpc.ClientConn, + threadID int, + channelFunc ChannelFunc) { + timeoutDuration := time.Duration(overallTimeoutSeconds) * time.Second + currentChannel := sharedChannel + + for i := 0; i < soakIterationsPerThread; i++ { if ctx.Err() != nil { - elapsedTime = time.Since(start).Seconds() - break + return + } + + if time.Since(time.Unix(0, startNs)) >= timeoutDuration { + fmt.Printf("Test exceeded overall timeout of %d seconds, stopping...\n", overallTimeoutSeconds) + return } earliestNextStart := time.After(minTimeBetweenRPCs) - iterationsDone++ + // Get the channel and client from the provided channelFunc (either shared or new) + _, client := channelFunc(currentChannel) var p peer.Peer - latency, err := doOneSoakIteration(ctx, tc, resetChannel, serverAddr, soakRequestSize, soakResponseSize, dopts, []grpc.CallOption{grpc.Peer(&p)}) - latencyMs := int64(latency / time.Millisecond) - h.Add(latencyMs) + result, err := doOneSoakIteration( + ctx, + client, + soakRequestSize, + soakResponseSize, + []grpc.CallOption{grpc.Peer(&p)}) + addrStr := "nil" + if p.Addr != nil { + addrStr = p.Addr.String() + } else { + fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") + } if err != nil { - totalFailures++ - addrStr := "nil" - if p.Addr != nil { - addrStr = p.Addr.String() - } - fmt.Fprintf(os.Stderr, "soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", i, latencyMs, addrStr, serverAddr, err) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", threadID, i, 0, addrStr, serverAddr, err) + mu.Lock() + threadResults.Failures++ + mu.Unlock() <-earliestNextStart continue } - if latency > perIterationMaxAcceptableLatency { - totalFailures++ - fmt.Fprintf(os.Stderr, "soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", i, latencyMs, p.Addr.String(), serverAddr, perIterationMaxAcceptableLatency.Milliseconds()) + latencyMs := result.LatencyMs + if latencyMs > perIterationMaxAcceptableLatency.Milliseconds() { + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, addrStr, serverAddr, perIterationMaxAcceptableLatency.Milliseconds()) + mu.Lock() + threadResults.Failures++ + mu.Unlock() <-earliestNextStart continue } - fmt.Fprintf(os.Stderr, "soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", i, latencyMs, p.Addr.String(), serverAddr) + // Success: log the details of the iteration + mu.Lock() + threadResults.Latencies.Add(latencyMs) + threadResults.IterationsDone++ + mu.Unlock() + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", threadID, i, latencyMs, addrStr, serverAddr) <-earliestNextStart } +} + +// DoSoakTest runs large unary RPCs in a loop for a configurable number of times, with configurable failure thresholds. +// If resetChannel is false, then each RPC will be performed on tc. Otherwise, each RPC will be performed on a new +// stub that is created with the provided server address and dial options. +// TODO(mohanli-ml): Create SoakTestOptions as a parameter for this method. +func DoSoakTest( + ctx context.Context, + conn *grpc.ClientConn, + serverAddr string, + numThreads int, + soakIterations int, + maxFailures int, + soakRequestSize int, + soakResponseSize int, + perIterationMaxAcceptableLatency time.Duration, + minTimeBetweenRPCs time.Duration, + overallTimeoutSeconds int, + channelFunc ChannelFunc) { + if soakIterations%numThreads != 0 { + fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by numThreads\n") + } + sharedChannel := conn + startNs := time.Now().UnixNano() + var wg sync.WaitGroup + mu := sync.Mutex{} + threadResults := make([]ThreadResults, numThreads) + iterationsPerThread := soakIterations / numThreads + for i := 0; i < numThreads; i++ { + wg.Add(1) + go func(threadID int) { + defer wg.Done() + executeSoakTestInThread( + iterationsPerThread, + startNs, + minTimeBetweenRPCs, + soakRequestSize, + soakResponseSize, + perIterationMaxAcceptableLatency, + overallTimeoutSeconds, + serverAddr, + &threadResults[threadID], + &mu, + ctx, + sharedChannel, + threadID, + channelFunc) + }(i) + } + + // Wait for all goroutines to complete + wg.Wait() + + //Handle results + totalIterations := 0 + totalFailures := 0 + latencies := stats.NewHistogram(stats.HistogramOptions{ + NumBuckets: 20, + GrowthFactor: 1, + BaseBucketSize: 1, + MinValue: 0, + }) + for _, thread := range threadResults { + totalIterations += thread.IterationsDone + totalFailures += thread.Failures + if thread.Latencies != nil { + // Add latencies from the thread's Histogram to the main latencies + latencies.Merge(thread.Latencies) + } + } var b bytes.Buffer - h.Print(&b) - fmt.Fprintf(os.Stderr, "(server_uri: %s) histogram of per-iteration latencies in milliseconds: %s\n", serverAddr, b.String()) - fmt.Fprintf(os.Stderr, "(server_uri: %s) soak test ran: %d / %d iterations. total failures: %d. max failures threshold: %d. See breakdown above for which iterations succeeded, failed, and why for more info.\n", serverAddr, iterationsDone, soakIterations, totalFailures, maxFailures) - if iterationsDone < soakIterations { - logger.Fatalf("(server_uri: %s) soak test consumed all %f seconds of time and quit early, only having ran %d out of desired %d iterations.", serverAddr, elapsedTime, iterationsDone, soakIterations) + latencies.Print(&b) + fmt.Fprintf(os.Stderr, + "(server_uri: %s) soak test ran: %d / %d iterations. Total failures: %d. Latencies in milliseconds: %s\n", + serverAddr, totalIterations, soakIterations, totalFailures, b.String()) + + if totalIterations != soakIterations { + fmt.Fprintf(os.Stderr, "Soak test consumed all %d seconds of time and quit early, ran %d out of %d iterations.\n", overallTimeoutSeconds, totalIterations, soakIterations) } + if totalFailures > maxFailures { - logger.Fatalf("(server_uri: %s) soak test total failures: %d exceeds max failures threshold: %d.", serverAddr, totalFailures, maxFailures) + fmt.Fprintf(os.Stderr, "Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, maxFailures) } + closeChannel(sharedChannel) } type testServer struct { diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index 56572e4a35c3..e9768b44dc6f 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -54,6 +54,7 @@ var ( soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") + numThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") testCase = flag.String("test_case", "rpc_soak", `Configure different test cases. Valid options are: rpc_soak: sends --soak_iterations large_unary RPCs; @@ -63,6 +64,7 @@ var ( ) type clientConfig struct { + conn *grpc.ClientConn tc testgrpc.TestServiceClient opts []grpc.DialOption uri string @@ -81,18 +83,24 @@ func main() { logger.Fatalf("Unsupported credentials type: %v", c) } } - var resetChannel bool + var clients []clientConfig + var channelFunc interop.ChannelFunc switch *testCase { case "rpc_soak": - resetChannel = false + channelFunc = interop.UseSharedChannel case "channel_soak": - resetChannel = true + channelFunc = func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + for _, client := range clients { + return interop.CreateNewChannel(currentChannel, client.uri, client.opts) + } + return nil, nil + } default: logger.Fatal("Unsupported test case: ", *testCase) } // create clients as specified in flags - var clients []clientConfig + //var clients []clientConfig for i := range uris { var opts []grpc.DialOption switch creds[i] { @@ -101,12 +109,13 @@ func main() { case insecureCredsName: opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } - cc, err := grpc.Dial(uris[i], opts...) + cc, err := grpc.NewClient(uris[i], opts...) if err != nil { logger.Fatalf("Fail to dial %v: %v", uris[i], err) } defer cc.Close() clients = append(clients, clientConfig{ + conn: cc, tc: testgrpc.NewTestServiceClient(cc), opts: opts, uri: uris[i], @@ -122,7 +131,20 @@ func main() { go func(c clientConfig) { ctxWithDeadline, cancel := context.WithTimeout(ctx, time.Duration(*soakOverallTimeoutSeconds)*time.Second) defer cancel() - interop.DoSoakTest(ctxWithDeadline, c.tc, c.uri, c.opts, resetChannel, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond) + interop.DoSoakTest( + ctxWithDeadline, + c.conn, + c.uri, + *numThreads, + *soakIterations, + *soakMaxFailures, + *soakRequestSize, + *soakResponseSize, + time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, + time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, + *soakOverallTimeoutSeconds, + channelFunc, + ) logger.Infof("%s test done for server: %s", *testCase, c.uri) wg.Done() }(clients[i]) From 0bbe38befcfe88742b8817ee4ebd7d1af49bcbe0 Mon Sep 17 00:00:00 2001 From: zbilun Date: Thu, 12 Dec 2024 01:12:47 -0800 Subject: [PATCH 02/45] Remove .idea/ directory from Git tracking --- .idea/.gitignore | 8 -------- .idea/grpc-go.iml | 10 ---------- .idea/misc.xml | 6 ------ .idea/modules.xml | 8 -------- .idea/vcs.xml | 40 ---------------------------------------- 5 files changed, 72 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/grpc-go.iml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b81b018..000000000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/grpc-go.iml b/.idea/grpc-go.iml deleted file mode 100644 index 25ed3f6e7b6e..000000000000 --- a/.idea/grpc-go.iml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 639900d13c61..000000000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index cd91612713cb..000000000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index fd7702ec990b..000000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - \ No newline at end of file From 219e17e35b61fd8a85bfe6f586bc24da9c54a9af Mon Sep 17 00:00:00 2001 From: zbilun Date: Thu, 12 Dec 2024 01:33:47 -0800 Subject: [PATCH 03/45] Fix the style issue --- .gitignore | 2 ++ interop/test_utils.go | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000000..13bbe5b9112f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.idea/ + diff --git a/interop/test_utils.go b/interop/test_utils.go index 98bbc74d75da..16f8634dd5d4 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -685,11 +685,13 @@ func DoPickFirstUnary(ctx context.Context, tc testgrpc.TestServiceClient) { } } +// SoakIterationResult represents the result of a single iteration in the soak test. type SoakIterationResult struct { LatencyMs int64 Status string // The status of the iteration } +// ThreadResults stores the aggregated results for a specific thread during the soak test type ThreadResults struct { IterationsDone int Failures int @@ -770,6 +772,7 @@ func doOneSoakIteration( } func executeSoakTestInThread( + ctx context.Context, soakIterationsPerThread int, startNs int64, minTimeBetweenRPCs time.Duration, @@ -780,7 +783,6 @@ func executeSoakTestInThread( serverAddr string, threadResults *ThreadResults, mu *sync.Mutex, - ctx context.Context, sharedChannel *grpc.ClientConn, threadID int, channelFunc ChannelFunc) { @@ -870,6 +872,7 @@ func DoSoakTest( go func(threadID int) { defer wg.Done() executeSoakTestInThread( + ctx, iterationsPerThread, startNs, minTimeBetweenRPCs, @@ -880,7 +883,6 @@ func DoSoakTest( serverAddr, &threadResults[threadID], &mu, - ctx, sharedChannel, threadID, channelFunc) From ead030a3a5c1f2ad93aa8233281c0b6bfabc1dc1 Mon Sep 17 00:00:00 2001 From: zbilun Date: Thu, 12 Dec 2024 01:38:04 -0800 Subject: [PATCH 04/45] Remove .idea/ from .gitignore --- .gitignore | 2 -- .idea/.gitignore | 8 ++++++++ .idea/grpc-go.iml | 10 ++++++++++ .idea/misc.xml | 6 ++++++ .idea/modules.xml | 8 ++++++++ .idea/vcs.xml | 40 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 72 insertions(+), 2 deletions(-) delete mode 100644 .gitignore create mode 100644 .idea/.gitignore create mode 100644 .idea/grpc-go.iml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 13bbe5b9112f..000000000000 --- a/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.idea/ - diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 000000000000..13566b81b018 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/grpc-go.iml b/.idea/grpc-go.iml new file mode 100644 index 000000000000..25ed3f6e7b6e --- /dev/null +++ b/.idea/grpc-go.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 000000000000..639900d13c61 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 000000000000..cd91612713cb --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 000000000000..fd7702ec990b --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,40 @@ + + + + + + + + + \ No newline at end of file From cbddddc4705a3cbbbf70600c796c8d6018ca5bb2 Mon Sep 17 00:00:00 2001 From: zbilun Date: Thu, 12 Dec 2024 01:41:13 -0800 Subject: [PATCH 05/45] Add .gitignore file --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000000..13bbe5b9112f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.idea/ + From be473571db3818d571fd663a976a870731d15be7 Mon Sep 17 00:00:00 2001 From: zbilun Date: Thu, 12 Dec 2024 14:14:43 -0800 Subject: [PATCH 06/45] fix channel close issue --- interop/test_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interop/test_utils.go b/interop/test_utils.go index 16f8634dd5d4..91e53c4b1b8f 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -922,7 +922,7 @@ func DoSoakTest( if totalFailures > maxFailures { fmt.Fprintf(os.Stderr, "Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, maxFailures) } - closeChannel(sharedChannel) + defer closeChannel(sharedChannel) } type testServer struct { From bfd1b8ed547c367beaed6ce57305ccf589e26900 Mon Sep 17 00:00:00 2001 From: zbilun Date: Thu, 12 Dec 2024 14:44:32 -0800 Subject: [PATCH 07/45] Stop tracking .idea directory --- .idea/.gitignore | 8 -------- .idea/grpc-go.iml | 10 ---------- .idea/misc.xml | 6 ------ .idea/modules.xml | 8 -------- .idea/vcs.xml | 40 ---------------------------------------- 5 files changed, 72 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/grpc-go.iml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b81b018..000000000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/grpc-go.iml b/.idea/grpc-go.iml deleted file mode 100644 index 25ed3f6e7b6e..000000000000 --- a/.idea/grpc-go.iml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 639900d13c61..000000000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index cd91612713cb..000000000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index fd7702ec990b..000000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - \ No newline at end of file From 3929ecf8295f56362eee0a68e6d6e3a1803f4d68 Mon Sep 17 00:00:00 2001 From: zbilun Date: Fri, 13 Dec 2024 14:11:03 -0800 Subject: [PATCH 08/45] Fix the comments style. --- interop/test_utils.go | 42 ++++++++++++++++---------------- interop/xds_federation/client.go | 8 +++--- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/interop/test_utils.go b/interop/test_utils.go index 91e53c4b1b8f..023643c5fa98 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -685,23 +685,23 @@ func DoPickFirstUnary(ctx context.Context, tc testgrpc.TestServiceClient) { } } -// SoakIterationResult represents the result of a single iteration in the soak test. +// SoakIterationResult represents the latency and status results of a single iteration in the soak test. type SoakIterationResult struct { LatencyMs int64 Status string // The status of the iteration } -// ThreadResults stores the aggregated results for a specific thread during the soak test +// ThreadResults stores the aggregated results for a specific thread during the soak test. type ThreadResults struct { IterationsDone int Failures int Latencies *stats.Histogram } -// ChannelFunc can be used to customize how the channel is handled (reuse or reset) -type ChannelFunc func(*grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) +// ManagedChannel determines whether a new channel will be created or an existing one reused. +type ManagedChannel func(*grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) -// createChannel Initialize the shared channel (once for all threads) +// createChannel initializes the shared channel (once for all threads). func createChannel(serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { conn, err := grpc.NewClient(serverAddr, dopts...) if err != nil { @@ -720,14 +720,14 @@ func closeChannel(channel *grpc.ClientConn) { } } -// CreateNewChannel Create a new channel by shutting down the current one (for channel soak tests) +// CreateNewChannel creates a new channel by shutting down the current one (for channel soak tests). func CreateNewChannel(currentChannel *grpc.ClientConn, serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { closeChannel(currentChannel) conn, client := createChannel(serverAddr, dopts) return conn, client } -// UseSharedChannel Reuses the provided currentChannel +// UseSharedChannel reuses the provided currentChannel. func UseSharedChannel(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { client := testgrpc.NewTestServiceClient(currentChannel) return currentChannel, client @@ -741,29 +741,29 @@ func doOneSoakIteration( copts []grpc.CallOption) (SoakIterationResult, error) { start := time.Now() var err error - // Do a large-unary RPC - // Create the request payload + // Do a large-unary RPC. + // Create the request payload. pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, soakRequestSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, ResponseSize: int32(soakResponseSize), Payload: pl, } - // Perform the GRPC call + // Perform the GRPC call. var reply *testpb.SimpleResponse reply, err = client.UnaryCall(ctx, req, copts...) if err != nil { err = fmt.Errorf("/TestService/UnaryCall RPC failed: %s", err) return SoakIterationResult{}, err } - // validate response + // Validate response. t := reply.GetPayload().GetType() s := len(reply.GetPayload().GetBody()) if t != testpb.PayloadType_COMPRESSABLE || s != soakResponseSize { err = fmt.Errorf("got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, soakResponseSize) return SoakIterationResult{}, err } - // Calculate latency and return result + // Calculate latency and return result. latency := time.Since(start).Milliseconds() return SoakIterationResult{ LatencyMs: latency, @@ -785,7 +785,7 @@ func executeSoakTestInThread( mu *sync.Mutex, sharedChannel *grpc.ClientConn, threadID int, - channelFunc ChannelFunc) { + MayCreateNewChannel ManagedChannel) { timeoutDuration := time.Duration(overallTimeoutSeconds) * time.Second currentChannel := sharedChannel @@ -799,8 +799,8 @@ func executeSoakTestInThread( return } earliestNextStart := time.After(minTimeBetweenRPCs) - // Get the channel and client from the provided channelFunc (either shared or new) - _, client := channelFunc(currentChannel) + // Get the channel and client from the provided channelFunc (either shared or new). + _, client := MayCreateNewChannel(currentChannel) var p peer.Peer result, err := doOneSoakIteration( ctx, @@ -831,7 +831,7 @@ func executeSoakTestInThread( <-earliestNextStart continue } - // Success: log the details of the iteration + // Success: log the details of the iteration. mu.Lock() threadResults.Latencies.Add(latencyMs) threadResults.IterationsDone++ @@ -857,7 +857,7 @@ func DoSoakTest( perIterationMaxAcceptableLatency time.Duration, minTimeBetweenRPCs time.Duration, overallTimeoutSeconds int, - channelFunc ChannelFunc) { + MayCreateNewChannel ManagedChannel) { if soakIterations%numThreads != 0 { fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by numThreads\n") } @@ -885,14 +885,14 @@ func DoSoakTest( &mu, sharedChannel, threadID, - channelFunc) + MayCreateNewChannel) }(i) } - // Wait for all goroutines to complete + // Wait for all goroutines to complete. wg.Wait() - //Handle results + //Handle results. totalIterations := 0 totalFailures := 0 latencies := stats.NewHistogram(stats.HistogramOptions{ @@ -905,7 +905,7 @@ func DoSoakTest( totalIterations += thread.IterationsDone totalFailures += thread.Failures if thread.Latencies != nil { - // Add latencies from the thread's Histogram to the main latencies + // Add latencies from the thread's Histogram to the main latencies. latencies.Merge(thread.Latencies) } } diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index e9768b44dc6f..11c3947f5356 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -84,12 +84,12 @@ func main() { } } var clients []clientConfig - var channelFunc interop.ChannelFunc + var MayCreateNewChannel interop.ManagedChannel switch *testCase { case "rpc_soak": - channelFunc = interop.UseSharedChannel + MayCreateNewChannel = interop.UseSharedChannel case "channel_soak": - channelFunc = func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + MayCreateNewChannel = func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { for _, client := range clients { return interop.CreateNewChannel(currentChannel, client.uri, client.opts) } @@ -143,7 +143,7 @@ func main() { time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, - channelFunc, + MayCreateNewChannel, ) logger.Infof("%s test done for server: %s", *testCase, c.uri) wg.Done() From 742ed86959b8e283f87082bc85aa6e3a008e8219 Mon Sep 17 00:00:00 2001 From: zbilun Date: Mon, 16 Dec 2024 23:18:12 -0800 Subject: [PATCH 09/45] Fix the go style issues. --- interop/client/client.go | 6 +++--- interop/test_utils.go | 31 ++++++++++++------------------- interop/xds_federation/client.go | 6 ++---- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index 727ed6159778..b25ee86b17a6 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -79,7 +79,7 @@ var ( soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") - numThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") + soakNumThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") tlsServerName = flag.String("server_host_override", "", "The server name used to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.") additionalMetadata = flag.String("additional_metadata", "", "Additional metadata to send in each request, as a semicolon-separated list of key:value pairs.") testCase = flag.String("test_case", "large_unary", @@ -359,12 +359,12 @@ func main() { interop.DoPickFirstUnary(ctx, tc) logger.Infoln("PickFirstUnary done") case "rpc_soak": - interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *numThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, + interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, interop.UseSharedChannel) logger.Infoln("RpcSoak done") case "channel_soak": - interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *numThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, + interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { return interop.CreateNewChannel(currentChannel, serverAddr, opts) diff --git a/interop/test_utils.go b/interop/test_utils.go index 023643c5fa98..cc05b2aa50bc 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -733,14 +733,8 @@ func UseSharedChannel(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgr return currentChannel, client } -func doOneSoakIteration( - ctx context.Context, - client testgrpc.TestServiceClient, - soakRequestSize int, - soakResponseSize int, - copts []grpc.CallOption) (SoakIterationResult, error) { +func doOneSoakIteration(ctx context.Context, client testgrpc.TestServiceClient, soakRequestSize int, soakResponseSize int, copts []grpc.CallOption) (SoakIterationResult, error) { start := time.Now() - var err error // Do a large-unary RPC. // Create the request payload. pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, soakRequestSize) @@ -751,7 +745,7 @@ func doOneSoakIteration( } // Perform the GRPC call. var reply *testpb.SimpleResponse - reply, err = client.UnaryCall(ctx, req, copts...) + reply, err := client.UnaryCall(ctx, req, copts...) if err != nil { err = fmt.Errorf("/TestService/UnaryCall RPC failed: %s", err) return SoakIterationResult{}, err @@ -808,14 +802,13 @@ func executeSoakTestInThread( soakRequestSize, soakResponseSize, []grpc.CallOption{grpc.Peer(&p)}) - addrStr := "nil" if p.Addr != nil { - addrStr = p.Addr.String() + fmt.Fprintf(os.Stderr, "Peer address: %q\n", p.Addr) } else { fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") } if err != nil { - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", threadID, i, 0, addrStr, serverAddr, err) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s failed: %s\n", threadID, i, 0, p.Addr, serverAddr, err) mu.Lock() threadResults.Failures++ mu.Unlock() @@ -824,7 +817,7 @@ func executeSoakTestInThread( } latencyMs := result.LatencyMs if latencyMs > perIterationMaxAcceptableLatency.Milliseconds() { - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, addrStr, serverAddr, perIterationMaxAcceptableLatency.Milliseconds()) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, p.Addr, serverAddr, perIterationMaxAcceptableLatency.Milliseconds()) mu.Lock() threadResults.Failures++ mu.Unlock() @@ -836,7 +829,7 @@ func executeSoakTestInThread( threadResults.Latencies.Add(latencyMs) threadResults.IterationsDone++ mu.Unlock() - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", threadID, i, latencyMs, addrStr, serverAddr) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s succeeded\n", threadID, i, latencyMs, p.Addr, serverAddr) <-earliestNextStart } } @@ -849,7 +842,7 @@ func DoSoakTest( ctx context.Context, conn *grpc.ClientConn, serverAddr string, - numThreads int, + soakNumThreads int, soakIterations int, maxFailures int, soakRequestSize int, @@ -858,16 +851,16 @@ func DoSoakTest( minTimeBetweenRPCs time.Duration, overallTimeoutSeconds int, MayCreateNewChannel ManagedChannel) { - if soakIterations%numThreads != 0 { - fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by numThreads\n") + if soakIterations%soakNumThreads != 0 { + fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumThreads\n") } sharedChannel := conn startNs := time.Now().UnixNano() var wg sync.WaitGroup mu := sync.Mutex{} - threadResults := make([]ThreadResults, numThreads) - iterationsPerThread := soakIterations / numThreads - for i := 0; i < numThreads; i++ { + threadResults := make([]ThreadResults, soakNumThreads) + iterationsPerThread := soakIterations / soakNumThreads + for i := 0; i < soakNumThreads; i++ { wg.Add(1) go func(threadID int) { defer wg.Done() diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index 11c3947f5356..a072c9ef92a6 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -54,7 +54,7 @@ var ( soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") - numThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") + soakNumThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") testCase = flag.String("test_case", "rpc_soak", `Configure different test cases. Valid options are: rpc_soak: sends --soak_iterations large_unary RPCs; @@ -99,8 +99,6 @@ func main() { logger.Fatal("Unsupported test case: ", *testCase) } - // create clients as specified in flags - //var clients []clientConfig for i := range uris { var opts []grpc.DialOption switch creds[i] { @@ -135,7 +133,7 @@ func main() { ctxWithDeadline, c.conn, c.uri, - *numThreads, + *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, From 9fdcd5677284c794f921b05aab489a2c154fe5b9 Mon Sep 17 00:00:00 2001 From: zbilun Date: Mon, 16 Dec 2024 23:31:43 -0800 Subject: [PATCH 10/45] Fix the go style issues about the space. --- interop/client/client.go | 2 +- interop/xds_federation/client.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index b25ee86b17a6..a473dbe129e5 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -79,7 +79,7 @@ var ( soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") - soakNumThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") + soakNumThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") tlsServerName = flag.String("server_host_override", "", "The server name used to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.") additionalMetadata = flag.String("additional_metadata", "", "Additional metadata to send in each request, as a semicolon-separated list of key:value pairs.") testCase = flag.String("test_case", "large_unary", diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index a072c9ef92a6..aefe03d74ecd 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -54,7 +54,7 @@ var ( soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") - soakNumThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") + soakNumThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") testCase = flag.String("test_case", "rpc_soak", `Configure different test cases. Valid options are: rpc_soak: sends --soak_iterations large_unary RPCs; From b7183008c25b9baec195d47eb9a8d25d1c847123 Mon Sep 17 00:00:00 2001 From: zbilun Date: Mon, 16 Dec 2024 23:57:44 -0800 Subject: [PATCH 11/45] Debug for test failure. --- interop/test_utils.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/interop/test_utils.go b/interop/test_utils.go index cc05b2aa50bc..fe6b21876ba8 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -802,13 +802,20 @@ func executeSoakTestInThread( soakRequestSize, soakResponseSize, []grpc.CallOption{grpc.Peer(&p)}) + addrStr := "nil" if p.Addr != nil { - fmt.Fprintf(os.Stderr, "Peer address: %q\n", p.Addr) + addrStr = p.Addr.String() } else { fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") } + //if p.Addr != nil { + // fmt.Fprintf(os.Stderr, "Peer address: %q\n", p.Addr) + //} else { + // fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") + //} if err != nil { - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s failed: %s\n", threadID, i, 0, p.Addr, serverAddr, err) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", threadID, i, 0, addrStr, serverAddr, err) + //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s failed: %s\n", threadID, i, 0, p.Addr, serverAddr, err) mu.Lock() threadResults.Failures++ mu.Unlock() @@ -817,7 +824,8 @@ func executeSoakTestInThread( } latencyMs := result.LatencyMs if latencyMs > perIterationMaxAcceptableLatency.Milliseconds() { - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, p.Addr, serverAddr, perIterationMaxAcceptableLatency.Milliseconds()) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, addrStr, serverAddr, perIterationMaxAcceptableLatency.Milliseconds()) + //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, p.Addr, serverAddr, perIterationMaxAcceptableLatency.Milliseconds()) mu.Lock() threadResults.Failures++ mu.Unlock() @@ -829,7 +837,8 @@ func executeSoakTestInThread( threadResults.Latencies.Add(latencyMs) threadResults.IterationsDone++ mu.Unlock() - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s succeeded\n", threadID, i, latencyMs, p.Addr, serverAddr) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", threadID, i, latencyMs, addrStr, serverAddr) + //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s succeeded\n", threadID, i, latencyMs, p.Addr, serverAddr) <-earliestNextStart } } From 2df82f809b876ecfa32db1e36e997ab1c90ea678 Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 17 Dec 2024 01:29:15 -0800 Subject: [PATCH 12/45] Replace long parameter lists with struct for cleaner code. --- interop/client/client.go | 44 +++++++++-- interop/test_utils.go | 132 +++++++++++++------------------ interop/xds_federation/client.go | 42 ++++++---- 3 files changed, 122 insertions(+), 96 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index a473dbe129e5..f70eb9cff8db 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -359,17 +359,47 @@ func main() { interop.DoPickFirstUnary(ctx, tc) logger.Infoln("PickFirstUnary done") case "rpc_soak": - interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, - time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, - interop.UseSharedChannel) + soakTestConfig := interop.SoakTestConfig{ + SoakRequestSize: *soakRequestSize, + SoakResponseSize: *soakResponseSize, + PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, + MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, + OverallTimeoutSeconds: *soakOverallTimeoutSeconds, + ServerAddr: serverAddr, + SoakNumThreads: *soakNumThreads, + SoakIterations: *soakIterations, + MaxFailures: *soakMaxFailures, + SharedChannel: conn, + MayCreateNewChannel: interop.UseSharedChannel, + } + interop.DoSoakTest(ctxWithDeadline, conn, soakTestConfig) logger.Infoln("RpcSoak done") + //interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, + // time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, + // interop.UseSharedChannel) case "channel_soak": - interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, - time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, - func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + soakTestConfig := interop.SoakTestConfig{ + SoakRequestSize: *soakRequestSize, + SoakResponseSize: *soakResponseSize, + PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, + MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, + OverallTimeoutSeconds: *soakOverallTimeoutSeconds, + ServerAddr: serverAddr, + SoakNumThreads: *soakNumThreads, + SoakIterations: *soakIterations, + MaxFailures: *soakMaxFailures, + SharedChannel: conn, + MayCreateNewChannel: func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { return interop.CreateNewChannel(currentChannel, serverAddr, opts) - }) + }, + } + interop.DoSoakTest(ctxWithDeadline, conn, soakTestConfig) logger.Infoln("ChannelSoak done") + //interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, + // time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, + // func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + // return interop.CreateNewChannel(currentChannel, serverAddr, opts) + // }) case "orca_per_rpc": interop.DoORCAPerRPCTest(ctx, tc) logger.Infoln("ORCAPerRPC done") diff --git a/interop/test_utils.go b/interop/test_utils.go index fe6b21876ba8..2f95bd3ec906 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -701,6 +701,29 @@ type ThreadResults struct { // ManagedChannel determines whether a new channel will be created or an existing one reused. type ManagedChannel func(*grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) +// SoakIterationConfig holds the parameters required for a single soak iteration. +type SoakIterationConfig struct { + SoakRequestSize int // The size of the request payload in bytes. + SoakResponseSize int // The expected size of the response payload in bytes. + Client testgrpc.TestServiceClient // The gRPC client to make the call. + CallOptions []grpc.CallOption // Call options for the RPC. +} + +// SoakTestConfig holds the configuration for the entire soak test. +type SoakTestConfig struct { + SoakRequestSize int + SoakResponseSize int + PerIterationMaxAcceptableLatency time.Duration + MinTimeBetweenRPCs time.Duration + OverallTimeoutSeconds int + ServerAddr string + SoakNumThreads int + SoakIterations int + MaxFailures int + SharedChannel *grpc.ClientConn + MayCreateNewChannel ManagedChannel +} + // createChannel initializes the shared channel (once for all threads). func createChannel(serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { conn, err := grpc.NewClient(serverAddr, dopts...) @@ -733,19 +756,19 @@ func UseSharedChannel(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgr return currentChannel, client } -func doOneSoakIteration(ctx context.Context, client testgrpc.TestServiceClient, soakRequestSize int, soakResponseSize int, copts []grpc.CallOption) (SoakIterationResult, error) { +func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (SoakIterationResult, error) { start := time.Now() // Do a large-unary RPC. // Create the request payload. - pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, soakRequestSize) + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, config.SoakRequestSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseSize: int32(soakResponseSize), + ResponseSize: int32(config.SoakResponseSize), Payload: pl, } // Perform the GRPC call. var reply *testpb.SimpleResponse - reply, err := client.UnaryCall(ctx, req, copts...) + reply, err := config.Client.UnaryCall(ctx, req, config.CallOptions...) if err != nil { err = fmt.Errorf("/TestService/UnaryCall RPC failed: %s", err) return SoakIterationResult{}, err @@ -753,8 +776,8 @@ func doOneSoakIteration(ctx context.Context, client testgrpc.TestServiceClient, // Validate response. t := reply.GetPayload().GetType() s := len(reply.GetPayload().GetBody()) - if t != testpb.PayloadType_COMPRESSABLE || s != soakResponseSize { - err = fmt.Errorf("got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, soakResponseSize) + if t != testpb.PayloadType_COMPRESSABLE || s != config.SoakResponseSize { + err = fmt.Errorf("got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, config.SoakResponseSize) return SoakIterationResult{}, err } // Calculate latency and return result. @@ -765,43 +788,30 @@ func doOneSoakIteration(ctx context.Context, client testgrpc.TestServiceClient, }, nil } -func executeSoakTestInThread( - ctx context.Context, - soakIterationsPerThread int, - startNs int64, - minTimeBetweenRPCs time.Duration, - soakRequestSize int, - soakResponseSize int, - perIterationMaxAcceptableLatency time.Duration, - overallTimeoutSeconds int, - serverAddr string, - threadResults *ThreadResults, - mu *sync.Mutex, - sharedChannel *grpc.ClientConn, - threadID int, - MayCreateNewChannel ManagedChannel) { - timeoutDuration := time.Duration(overallTimeoutSeconds) * time.Second - currentChannel := sharedChannel - +func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs int64, threadID int, threadResults *ThreadResults, mu *sync.Mutex) { + timeoutDuration := time.Duration(config.OverallTimeoutSeconds) * time.Second + currentChannel := config.SharedChannel + soakIterationsPerThread := config.SoakIterations / config.SoakNumThreads for i := 0; i < soakIterationsPerThread; i++ { if ctx.Err() != nil { return } if time.Since(time.Unix(0, startNs)) >= timeoutDuration { - fmt.Printf("Test exceeded overall timeout of %d seconds, stopping...\n", overallTimeoutSeconds) + fmt.Printf("Test exceeded overall timeout of %d seconds, stopping...\n", config.OverallTimeoutSeconds) return } - earliestNextStart := time.After(minTimeBetweenRPCs) + earliestNextStart := time.After(config.MinTimeBetweenRPCs) // Get the channel and client from the provided channelFunc (either shared or new). - _, client := MayCreateNewChannel(currentChannel) + _, client := config.MayCreateNewChannel(currentChannel) var p peer.Peer - result, err := doOneSoakIteration( - ctx, - client, - soakRequestSize, - soakResponseSize, - []grpc.CallOption{grpc.Peer(&p)}) + iterationConfig := SoakIterationConfig{ + SoakRequestSize: config.SoakRequestSize, + SoakResponseSize: config.SoakResponseSize, + Client: client, + CallOptions: []grpc.CallOption{grpc.Peer(&p)}, + } + result, err := doOneSoakIteration(ctx, iterationConfig) addrStr := "nil" if p.Addr != nil { addrStr = p.Addr.String() @@ -814,7 +824,7 @@ func executeSoakTestInThread( // fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") //} if err != nil { - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", threadID, i, 0, addrStr, serverAddr, err) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", threadID, i, 0, addrStr, config.ServerAddr, err) //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s failed: %s\n", threadID, i, 0, p.Addr, serverAddr, err) mu.Lock() threadResults.Failures++ @@ -823,8 +833,8 @@ func executeSoakTestInThread( continue } latencyMs := result.LatencyMs - if latencyMs > perIterationMaxAcceptableLatency.Milliseconds() { - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, addrStr, serverAddr, perIterationMaxAcceptableLatency.Milliseconds()) + if latencyMs > config.PerIterationMaxAcceptableLatency.Milliseconds() { + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, addrStr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, p.Addr, serverAddr, perIterationMaxAcceptableLatency.Milliseconds()) mu.Lock() threadResults.Failures++ @@ -837,7 +847,7 @@ func executeSoakTestInThread( threadResults.Latencies.Add(latencyMs) threadResults.IterationsDone++ mu.Unlock() - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", threadID, i, latencyMs, addrStr, serverAddr) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", threadID, i, latencyMs, addrStr, config.ServerAddr) //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s succeeded\n", threadID, i, latencyMs, p.Addr, serverAddr) <-earliestNextStart } @@ -847,50 +857,22 @@ func executeSoakTestInThread( // If resetChannel is false, then each RPC will be performed on tc. Otherwise, each RPC will be performed on a new // stub that is created with the provided server address and dial options. // TODO(mohanli-ml): Create SoakTestOptions as a parameter for this method. -func DoSoakTest( - ctx context.Context, - conn *grpc.ClientConn, - serverAddr string, - soakNumThreads int, - soakIterations int, - maxFailures int, - soakRequestSize int, - soakResponseSize int, - perIterationMaxAcceptableLatency time.Duration, - minTimeBetweenRPCs time.Duration, - overallTimeoutSeconds int, - MayCreateNewChannel ManagedChannel) { - if soakIterations%soakNumThreads != 0 { +func DoSoakTest(ctx context.Context, conn *grpc.ClientConn, config SoakTestConfig) { + if config.SoakIterations%config.SoakNumThreads != 0 { fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumThreads\n") } sharedChannel := conn startNs := time.Now().UnixNano() var wg sync.WaitGroup mu := sync.Mutex{} - threadResults := make([]ThreadResults, soakNumThreads) - iterationsPerThread := soakIterations / soakNumThreads - for i := 0; i < soakNumThreads; i++ { + threadResults := make([]ThreadResults, config.SoakNumThreads) + for i := 0; i < config.SoakNumThreads; i++ { wg.Add(1) go func(threadID int) { defer wg.Done() - executeSoakTestInThread( - ctx, - iterationsPerThread, - startNs, - minTimeBetweenRPCs, - soakRequestSize, - soakResponseSize, - perIterationMaxAcceptableLatency, - overallTimeoutSeconds, - serverAddr, - &threadResults[threadID], - &mu, - sharedChannel, - threadID, - MayCreateNewChannel) + executeSoakTestInThread(ctx, config, startNs,threadID, &threadResults[threadID], &mu) }(i) } - // Wait for all goroutines to complete. wg.Wait() @@ -915,14 +897,14 @@ func DoSoakTest( latencies.Print(&b) fmt.Fprintf(os.Stderr, "(server_uri: %s) soak test ran: %d / %d iterations. Total failures: %d. Latencies in milliseconds: %s\n", - serverAddr, totalIterations, soakIterations, totalFailures, b.String()) + config.ServerAddr, totalIterations, config.SoakIterations, totalFailures, b.String()) - if totalIterations != soakIterations { - fmt.Fprintf(os.Stderr, "Soak test consumed all %d seconds of time and quit early, ran %d out of %d iterations.\n", overallTimeoutSeconds, totalIterations, soakIterations) + if totalIterations != config.SoakIterations { + fmt.Fprintf(os.Stderr, "Soak test consumed all %d seconds of time and quit early, ran %d out of %d iterations.\n", config.OverallTimeoutSeconds, totalIterations, config.SoakIterations) } - if totalFailures > maxFailures { - fmt.Fprintf(os.Stderr, "Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, maxFailures) + if totalFailures > config.MaxFailures { + fmt.Fprintf(os.Stderr, "Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, config.MaxFailures) } defer closeChannel(sharedChannel) } diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index aefe03d74ecd..5c058d7a490b 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -129,20 +129,34 @@ func main() { go func(c clientConfig) { ctxWithDeadline, cancel := context.WithTimeout(ctx, time.Duration(*soakOverallTimeoutSeconds)*time.Second) defer cancel() - interop.DoSoakTest( - ctxWithDeadline, - c.conn, - c.uri, - *soakNumThreads, - *soakIterations, - *soakMaxFailures, - *soakRequestSize, - *soakResponseSize, - time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, - time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, - *soakOverallTimeoutSeconds, - MayCreateNewChannel, - ) + soakTestConfig := interop.SoakTestConfig{ + SoakRequestSize: *soakRequestSize, + SoakResponseSize: *soakResponseSize, + PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, + MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, + OverallTimeoutSeconds: *soakOverallTimeoutSeconds, + ServerAddr: c.uri, + SoakNumThreads: *soakNumThreads, + SoakIterations: *soakIterations, + MaxFailures: *soakMaxFailures, + SharedChannel: c.conn, + MayCreateNewChannel: MayCreateNewChannel, + } + interop.DoSoakTest(ctxWithDeadline, c.conn, soakTestConfig) + //interop.DoSoakTest( + // ctxWithDeadline, + // c.conn, + // c.uri, + // *soakNumThreads, + // *soakIterations, + // *soakMaxFailures, + // *soakRequestSize, + // *soakResponseSize, + // time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, + // time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, + // *soakOverallTimeoutSeconds, + // MayCreateNewChannel, + //) logger.Infof("%s test done for server: %s", *testCase, c.uri) wg.Done() }(clients[i]) From 2d721731505827e92c89ea76d8516de8f5cd0bd9 Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 17 Dec 2024 01:38:32 -0800 Subject: [PATCH 13/45] Clean code by deleting useless comments. --- interop/client/client.go | 8 -------- interop/xds_federation/client.go | 14 -------------- 2 files changed, 22 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index f70eb9cff8db..0adf42afdcc3 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -374,9 +374,6 @@ func main() { } interop.DoSoakTest(ctxWithDeadline, conn, soakTestConfig) logger.Infoln("RpcSoak done") - //interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, - // time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, - // interop.UseSharedChannel) case "channel_soak": soakTestConfig := interop.SoakTestConfig{ SoakRequestSize: *soakRequestSize, @@ -395,11 +392,6 @@ func main() { } interop.DoSoakTest(ctxWithDeadline, conn, soakTestConfig) logger.Infoln("ChannelSoak done") - //interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, - // time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, - // func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { - // return interop.CreateNewChannel(currentChannel, serverAddr, opts) - // }) case "orca_per_rpc": interop.DoORCAPerRPCTest(ctx, tc) logger.Infoln("ORCAPerRPC done") diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index 5c058d7a490b..04423f508a93 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -143,20 +143,6 @@ func main() { MayCreateNewChannel: MayCreateNewChannel, } interop.DoSoakTest(ctxWithDeadline, c.conn, soakTestConfig) - //interop.DoSoakTest( - // ctxWithDeadline, - // c.conn, - // c.uri, - // *soakNumThreads, - // *soakIterations, - // *soakMaxFailures, - // *soakRequestSize, - // *soakResponseSize, - // time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, - // time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, - // *soakOverallTimeoutSeconds, - // MayCreateNewChannel, - //) logger.Infof("%s test done for server: %s", *testCase, c.uri) wg.Done() }(clients[i]) From a0a3ae2665f891d576c45d8c7bbebb3367038503 Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 17 Dec 2024 12:25:11 -0800 Subject: [PATCH 14/45] Fix the test fail --- interop/client/client.go | 67 ++++++++++++++++++-------------- interop/test_utils.go | 42 +++++++++++++++----- interop/xds_federation/client.go | 40 ++++++++++++------- 3 files changed, 97 insertions(+), 52 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index 0adf42afdcc3..b1ac38bc4eb6 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -359,38 +359,47 @@ func main() { interop.DoPickFirstUnary(ctx, tc) logger.Infoln("PickFirstUnary done") case "rpc_soak": - soakTestConfig := interop.SoakTestConfig{ - SoakRequestSize: *soakRequestSize, - SoakResponseSize: *soakResponseSize, - PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, - MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, - OverallTimeoutSeconds: *soakOverallTimeoutSeconds, - ServerAddr: serverAddr, - SoakNumThreads: *soakNumThreads, - SoakIterations: *soakIterations, - MaxFailures: *soakMaxFailures, - SharedChannel: conn, - MayCreateNewChannel: interop.UseSharedChannel, - } - interop.DoSoakTest(ctxWithDeadline, conn, soakTestConfig) + //soakTestConfig := interop.SoakTestConfig{ + // SoakRequestSize: *soakRequestSize, + // SoakResponseSize: *soakResponseSize, + // PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, + // MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, + // OverallTimeoutSeconds: *soakOverallTimeoutSeconds, + // ServerAddr: serverAddr, + // SoakNumThreads: *soakNumThreads, + // SoakIterations: *soakIterations, + // MaxFailures: *soakMaxFailures, + // SharedChannel: conn, + // MayCreateNewChannel: interop.UseSharedChannel, + //} + //interop.DoSoakTest(ctxWithDeadline, conn, soakTestConfig) + interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, + time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, + interop.UseSharedChannel) logger.Infoln("RpcSoak done") case "channel_soak": - soakTestConfig := interop.SoakTestConfig{ - SoakRequestSize: *soakRequestSize, - SoakResponseSize: *soakResponseSize, - PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, - MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, - OverallTimeoutSeconds: *soakOverallTimeoutSeconds, - ServerAddr: serverAddr, - SoakNumThreads: *soakNumThreads, - SoakIterations: *soakIterations, - MaxFailures: *soakMaxFailures, - SharedChannel: conn, - MayCreateNewChannel: func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + //soakTestConfig := interop.SoakTestConfig{ + // SoakRequestSize: *soakRequestSize, + // SoakResponseSize: *soakResponseSize, + // PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, + // MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, + // OverallTimeoutSeconds: *soakOverallTimeoutSeconds, + // ServerAddr: serverAddr, + // SoakNumThreads: *soakNumThreads, + // SoakIterations: *soakIterations, + // MaxFailures: *soakMaxFailures, + // SharedChannel: conn, + // MayCreateNewChannel: func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + // return interop.CreateNewChannel(currentChannel, serverAddr, opts) + // }, + //} + //interop.DoSoakTest(ctxWithDeadline, conn, soakTestConfig) + + interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, + time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, + func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { return interop.CreateNewChannel(currentChannel, serverAddr, opts) - }, - } - interop.DoSoakTest(ctxWithDeadline, conn, soakTestConfig) + }) logger.Infoln("ChannelSoak done") case "orca_per_rpc": interop.DoORCAPerRPCTest(ctx, tc) diff --git a/interop/test_utils.go b/interop/test_utils.go index 2f95bd3ec906..c76ae1a82c06 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -857,19 +857,43 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs // If resetChannel is false, then each RPC will be performed on tc. Otherwise, each RPC will be performed on a new // stub that is created with the provided server address and dial options. // TODO(mohanli-ml): Create SoakTestOptions as a parameter for this method. -func DoSoakTest(ctx context.Context, conn *grpc.ClientConn, config SoakTestConfig) { - if config.SoakIterations%config.SoakNumThreads != 0 { +func DoSoakTest( ctx context.Context, + conn *grpc.ClientConn, + serverAddr string, + soakNumThreads int, + soakIterations int, + maxFailures int, + soakRequestSize int, + soakResponseSize int, + perIterationMaxAcceptableLatency time.Duration, + minTimeBetweenRPCs time.Duration, + overallTimeoutSeconds int, + channelFunc ManagedChannel) { + if soakIterations%soakNumThreads != 0 { fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumThreads\n") } sharedChannel := conn startNs := time.Now().UnixNano() var wg sync.WaitGroup mu := sync.Mutex{} - threadResults := make([]ThreadResults, config.SoakNumThreads) - for i := 0; i < config.SoakNumThreads; i++ { + threadResults := make([]ThreadResults, soakNumThreads) + for i := 0; i < soakNumThreads; i++ { wg.Add(1) go func(threadID int) { defer wg.Done() + config := SoakTestConfig{ + SoakRequestSize: soakRequestSize, + SoakResponseSize: soakResponseSize, + PerIterationMaxAcceptableLatency: perIterationMaxAcceptableLatency, + MinTimeBetweenRPCs: minTimeBetweenRPCs, + OverallTimeoutSeconds: overallTimeoutSeconds, + ServerAddr: serverAddr, + SoakNumThreads: soakNumThreads, + SoakIterations: soakIterations, + MaxFailures: maxFailures, + SharedChannel: sharedChannel, + MayCreateNewChannel: channelFunc, + } executeSoakTestInThread(ctx, config, startNs,threadID, &threadResults[threadID], &mu) }(i) } @@ -897,14 +921,14 @@ func DoSoakTest(ctx context.Context, conn *grpc.ClientConn, config SoakTestConfi latencies.Print(&b) fmt.Fprintf(os.Stderr, "(server_uri: %s) soak test ran: %d / %d iterations. Total failures: %d. Latencies in milliseconds: %s\n", - config.ServerAddr, totalIterations, config.SoakIterations, totalFailures, b.String()) + serverAddr, totalIterations, soakIterations, totalFailures, b.String()) - if totalIterations != config.SoakIterations { - fmt.Fprintf(os.Stderr, "Soak test consumed all %d seconds of time and quit early, ran %d out of %d iterations.\n", config.OverallTimeoutSeconds, totalIterations, config.SoakIterations) + if totalIterations != soakIterations { + fmt.Fprintf(os.Stderr, "Soak test consumed all %d seconds of time and quit early, ran %d out of %d iterations.\n", overallTimeoutSeconds, totalIterations, soakIterations) } - if totalFailures > config.MaxFailures { - fmt.Fprintf(os.Stderr, "Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, config.MaxFailures) + if totalFailures > maxFailures { + fmt.Fprintf(os.Stderr, "Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, maxFailures) } defer closeChannel(sharedChannel) } diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index 04423f508a93..1fc972bc5e55 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -129,20 +129,32 @@ func main() { go func(c clientConfig) { ctxWithDeadline, cancel := context.WithTimeout(ctx, time.Duration(*soakOverallTimeoutSeconds)*time.Second) defer cancel() - soakTestConfig := interop.SoakTestConfig{ - SoakRequestSize: *soakRequestSize, - SoakResponseSize: *soakResponseSize, - PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, - MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, - OverallTimeoutSeconds: *soakOverallTimeoutSeconds, - ServerAddr: c.uri, - SoakNumThreads: *soakNumThreads, - SoakIterations: *soakIterations, - MaxFailures: *soakMaxFailures, - SharedChannel: c.conn, - MayCreateNewChannel: MayCreateNewChannel, - } - interop.DoSoakTest(ctxWithDeadline, c.conn, soakTestConfig) + //soakTestConfig := interop.SoakTestConfig{ + // SoakRequestSize: *soakRequestSize, + // SoakResponseSize: *soakResponseSize, + // PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, + // MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, + // OverallTimeoutSeconds: *soakOverallTimeoutSeconds, + // ServerAddr: c.uri, + // SoakNumThreads: *soakNumThreads, + // SoakIterations: *soakIterations, + // MaxFailures: *soakMaxFailures, + // SharedChannel: c.conn, + // MayCreateNewChannel: MayCreateNewChannel, + //} + //interop.DoSoakTest(ctxWithDeadline, c.conn, soakTestConfig) + interop.DoSoakTest(ctxWithDeadline, + c.conn, + c.uri, + *soakNumThreads, + *soakIterations, + *soakMaxFailures, + *soakRequestSize, + *soakResponseSize, + time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, + time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, + *soakOverallTimeoutSeconds, + MayCreateNewChannel) logger.Infof("%s test done for server: %s", *testCase, c.uri) wg.Done() }(clients[i]) From 957c1b421aa5a698e0ff56b364433720e5d96a79 Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 17 Dec 2024 14:56:20 -0800 Subject: [PATCH 15/45] Fix the format check --- interop/test_utils.go | 70 +++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/interop/test_utils.go b/interop/test_utils.go index c76ae1a82c06..ff8d7acc66e8 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -711,17 +711,17 @@ type SoakIterationConfig struct { // SoakTestConfig holds the configuration for the entire soak test. type SoakTestConfig struct { - SoakRequestSize int - SoakResponseSize int - PerIterationMaxAcceptableLatency time.Duration - MinTimeBetweenRPCs time.Duration - OverallTimeoutSeconds int - ServerAddr string - SoakNumThreads int - SoakIterations int - MaxFailures int - SharedChannel *grpc.ClientConn - MayCreateNewChannel ManagedChannel + SoakRequestSize int + SoakResponseSize int + PerIterationMaxAcceptableLatency time.Duration + MinTimeBetweenRPCs time.Duration + OverallTimeoutSeconds int + ServerAddr string + SoakNumThreads int + SoakIterations int + MaxFailures int + SharedChannel *grpc.ClientConn + MayCreateNewChannel ManagedChannel } // createChannel initializes the shared channel (once for all threads). @@ -857,18 +857,18 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs // If resetChannel is false, then each RPC will be performed on tc. Otherwise, each RPC will be performed on a new // stub that is created with the provided server address and dial options. // TODO(mohanli-ml): Create SoakTestOptions as a parameter for this method. -func DoSoakTest( ctx context.Context, - conn *grpc.ClientConn, - serverAddr string, - soakNumThreads int, - soakIterations int, - maxFailures int, - soakRequestSize int, - soakResponseSize int, - perIterationMaxAcceptableLatency time.Duration, - minTimeBetweenRPCs time.Duration, - overallTimeoutSeconds int, - channelFunc ManagedChannel) { +func DoSoakTest(ctx context.Context, + conn *grpc.ClientConn, + serverAddr string, + soakNumThreads int, + soakIterations int, + maxFailures int, + soakRequestSize int, + soakResponseSize int, + perIterationMaxAcceptableLatency time.Duration, + minTimeBetweenRPCs time.Duration, + overallTimeoutSeconds int, + channelFunc ManagedChannel) { if soakIterations%soakNumThreads != 0 { fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumThreads\n") } @@ -882,19 +882,19 @@ func DoSoakTest( ctx context.Context, go func(threadID int) { defer wg.Done() config := SoakTestConfig{ - SoakRequestSize: soakRequestSize, - SoakResponseSize: soakResponseSize, - PerIterationMaxAcceptableLatency: perIterationMaxAcceptableLatency, - MinTimeBetweenRPCs: minTimeBetweenRPCs, - OverallTimeoutSeconds: overallTimeoutSeconds, - ServerAddr: serverAddr, - SoakNumThreads: soakNumThreads, - SoakIterations: soakIterations, - MaxFailures: maxFailures, - SharedChannel: sharedChannel, - MayCreateNewChannel: channelFunc, + SoakRequestSize: soakRequestSize, + SoakResponseSize: soakResponseSize, + PerIterationMaxAcceptableLatency: perIterationMaxAcceptableLatency, + MinTimeBetweenRPCs: minTimeBetweenRPCs, + OverallTimeoutSeconds: overallTimeoutSeconds, + ServerAddr: serverAddr, + SoakNumThreads: soakNumThreads, + SoakIterations: soakIterations, + MaxFailures: maxFailures, + SharedChannel: sharedChannel, + MayCreateNewChannel: channelFunc, } - executeSoakTestInThread(ctx, config, startNs,threadID, &threadResults[threadID], &mu) + executeSoakTestInThread(ctx, config, startNs, threadID, &threadResults[threadID], &mu) }(i) } // Wait for all goroutines to complete. From 67948ceee18ee69025a67c0735fba85baeaa672e Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 17 Dec 2024 15:09:34 -0800 Subject: [PATCH 16/45] Fix the p.addr --- interop/test_utils.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/interop/test_utils.go b/interop/test_utils.go index ff8d7acc66e8..5cfba223a709 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -812,20 +812,20 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs CallOptions: []grpc.CallOption{grpc.Peer(&p)}, } result, err := doOneSoakIteration(ctx, iterationConfig) - addrStr := "nil" - if p.Addr != nil { - addrStr = p.Addr.String() - } else { - fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") - } + //addrStr := "nil" //if p.Addr != nil { - // fmt.Fprintf(os.Stderr, "Peer address: %q\n", p.Addr) + // addrStr = p.Addr.String() //} else { // fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") //} + if p.Addr != nil { + fmt.Fprintf(os.Stderr, "Peer address: %v\n", p.Addr) + } else { + fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") + } if err != nil { - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", threadID, i, 0, addrStr, config.ServerAddr, err) - //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s failed: %s\n", threadID, i, 0, p.Addr, serverAddr, err) + //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", threadID, i, 0, addrStr, config.ServerAddr, err) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s failed: %s\n", threadID, i, 0, p.Addr, config.ServerAddr, err) mu.Lock() threadResults.Failures++ mu.Unlock() @@ -834,8 +834,8 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs } latencyMs := result.LatencyMs if latencyMs > config.PerIterationMaxAcceptableLatency.Milliseconds() { - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, addrStr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) - //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, p.Addr, serverAddr, perIterationMaxAcceptableLatency.Milliseconds()) + //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, addrStr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) mu.Lock() threadResults.Failures++ mu.Unlock() @@ -847,8 +847,8 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs threadResults.Latencies.Add(latencyMs) threadResults.IterationsDone++ mu.Unlock() - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", threadID, i, latencyMs, addrStr, config.ServerAddr) - //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %q server_uri: %s succeeded\n", threadID, i, latencyMs, p.Addr, serverAddr) + //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", threadID, i, latencyMs, addrStr, config.ServerAddr) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", threadID, i, latencyMs, p.Addr, config.ServerAddr) <-earliestNextStart } } From f03b6fe7eb16b93bd42f794bda5ed7f58762eb56 Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 17 Dec 2024 15:34:14 -0800 Subject: [PATCH 17/45] Change the addrStr back due to the print type issue. --- interop/test_utils.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/interop/test_utils.go b/interop/test_utils.go index 5cfba223a709..6fd4d6c4d30f 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -812,20 +812,20 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs CallOptions: []grpc.CallOption{grpc.Peer(&p)}, } result, err := doOneSoakIteration(ctx, iterationConfig) - //addrStr := "nil" - //if p.Addr != nil { - // addrStr = p.Addr.String() - //} else { - // fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") - //} + addrStr := "nil" if p.Addr != nil { - fmt.Fprintf(os.Stderr, "Peer address: %v\n", p.Addr) + addrStr = p.Addr.String() } else { fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") } + //if p.Addr != nil { + // fmt.Fprintf(os.Stderr, "Peer address: %v\n", p.Addr) + //} else { + // fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") + //} if err != nil { - //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", threadID, i, 0, addrStr, config.ServerAddr, err) - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s failed: %s\n", threadID, i, 0, p.Addr, config.ServerAddr, err) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", threadID, i, 0, addrStr, config.ServerAddr, err) + //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s failed: %s\n", threadID, i, 0, p.Addr, config.ServerAddr, err) mu.Lock() threadResults.Failures++ mu.Unlock() @@ -834,8 +834,8 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs } latencyMs := result.LatencyMs if latencyMs > config.PerIterationMaxAcceptableLatency.Milliseconds() { - //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, addrStr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, addrStr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) + //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) mu.Lock() threadResults.Failures++ mu.Unlock() @@ -847,8 +847,8 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs threadResults.Latencies.Add(latencyMs) threadResults.IterationsDone++ mu.Unlock() - //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", threadID, i, latencyMs, addrStr, config.ServerAddr) - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", threadID, i, latencyMs, p.Addr, config.ServerAddr) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", threadID, i, latencyMs, addrStr, config.ServerAddr) + //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", threadID, i, latencyMs, p.Addr, config.ServerAddr) <-earliestNextStart } } From f621f31fa2d016e252f0ce4c3355a85af0595c2e Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 17 Dec 2024 15:55:48 -0800 Subject: [PATCH 18/45] Clean comments. --- interop/test_utils.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/interop/test_utils.go b/interop/test_utils.go index 6fd4d6c4d30f..58fdc7abaf11 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -818,14 +818,8 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs } else { fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") } - //if p.Addr != nil { - // fmt.Fprintf(os.Stderr, "Peer address: %v\n", p.Addr) - //} else { - // fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") - //} if err != nil { fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", threadID, i, 0, addrStr, config.ServerAddr, err) - //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s failed: %s\n", threadID, i, 0, p.Addr, config.ServerAddr, err) mu.Lock() threadResults.Failures++ mu.Unlock() @@ -835,7 +829,6 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs latencyMs := result.LatencyMs if latencyMs > config.PerIterationMaxAcceptableLatency.Milliseconds() { fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, addrStr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) - //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) mu.Lock() threadResults.Failures++ mu.Unlock() @@ -848,7 +841,6 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs threadResults.IterationsDone++ mu.Unlock() fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", threadID, i, latencyMs, addrStr, config.ServerAddr) - //fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", threadID, i, latencyMs, p.Addr, config.ServerAddr) <-earliestNextStart } } From 2204ae831fa5b7f0836d03ac7407c1155abb4ce1 Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 17 Dec 2024 15:59:41 -0800 Subject: [PATCH 19/45] Add print message. --- interop/test_utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/interop/test_utils.go b/interop/test_utils.go index 58fdc7abaf11..5fcaa1bcd2a2 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -815,6 +815,7 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs addrStr := "nil" if p.Addr != nil { addrStr = p.Addr.String() + fmt.Fprintf(os.Stderr, "Peer address: %q\n", p.Addr) } else { fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") } From ca86c16150fa6a7ed8af87895b3d917af780aec3 Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 17 Dec 2024 16:11:10 -0800 Subject: [PATCH 20/45] Debug the test fail --- interop/test_utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/interop/test_utils.go b/interop/test_utils.go index 5fcaa1bcd2a2..58fdc7abaf11 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -815,7 +815,6 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs addrStr := "nil" if p.Addr != nil { addrStr = p.Addr.String() - fmt.Fprintf(os.Stderr, "Peer address: %q\n", p.Addr) } else { fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") } From 22c554402f1016fe7d99dd5221fba8be82b24a1c Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 17 Dec 2024 16:34:22 -0800 Subject: [PATCH 21/45] Duplicate print error --- interop/test_utils.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/interop/test_utils.go b/interop/test_utils.go index 58fdc7abaf11..2a517ce8d3ba 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -812,14 +812,19 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs CallOptions: []grpc.CallOption{grpc.Peer(&p)}, } result, err := doOneSoakIteration(ctx, iterationConfig) - addrStr := "nil" + //addrStr := "nil" + //if p.Addr != nil { + // addrStr = p.Addr.String() + //} else { + // fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") + //} if p.Addr != nil { - addrStr = p.Addr.String() + fmt.Fprintf(os.Stderr, "Peer address: %v\n", p.Addr) } else { fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") } if err != nil { - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", threadID, i, 0, addrStr, config.ServerAddr, err) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s failed: %s\n", threadID, i, 0, p.Addr, config.ServerAddr, err) mu.Lock() threadResults.Failures++ mu.Unlock() @@ -828,7 +833,7 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs } latencyMs := result.LatencyMs if latencyMs > config.PerIterationMaxAcceptableLatency.Milliseconds() { - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, addrStr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) mu.Lock() threadResults.Failures++ mu.Unlock() @@ -840,7 +845,7 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs threadResults.Latencies.Add(latencyMs) threadResults.IterationsDone++ mu.Unlock() - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", threadID, i, latencyMs, addrStr, config.ServerAddr) + fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", threadID, i, latencyMs, p.Addr, config.ServerAddr) <-earliestNextStart } } From 04282c5f07371a215876e1f2f30d55a8474fd9d3 Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 17 Dec 2024 17:46:23 -0800 Subject: [PATCH 22/45] Refactor the doSoakTest func. --- interop/client/client.go | 77 ++++++++++++++++---------------- interop/test_utils.go | 67 +++++++++++---------------- interop/xds_federation/client.go | 52 ++++++++++----------- 3 files changed, 90 insertions(+), 106 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index b1ac38bc4eb6..cd8418bb86fb 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -359,47 +359,48 @@ func main() { interop.DoPickFirstUnary(ctx, tc) logger.Infoln("PickFirstUnary done") case "rpc_soak": - //soakTestConfig := interop.SoakTestConfig{ - // SoakRequestSize: *soakRequestSize, - // SoakResponseSize: *soakResponseSize, - // PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, - // MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, - // OverallTimeoutSeconds: *soakOverallTimeoutSeconds, - // ServerAddr: serverAddr, - // SoakNumThreads: *soakNumThreads, - // SoakIterations: *soakIterations, - // MaxFailures: *soakMaxFailures, - // SharedChannel: conn, - // MayCreateNewChannel: interop.UseSharedChannel, - //} - //interop.DoSoakTest(ctxWithDeadline, conn, soakTestConfig) - interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, - time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, - interop.UseSharedChannel) + rpcSoakConfig := interop.SoakTestConfig{ + SoakRequestSize: *soakRequestSize, + SoakResponseSize: *soakResponseSize, + PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, + MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, + OverallTimeoutSeconds: *soakOverallTimeoutSeconds, + ServerAddr: serverAddr, + SoakNumThreads: *soakNumThreads, + SoakIterations: *soakIterations, + MaxFailures: *soakMaxFailures, + SharedChannel: conn, + MayCreateNewChannel: interop.UseSharedChannel, + } + interop.DoSoakTest(ctxWithDeadline, rpcSoakConfig) + + //interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, + // time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, + // interop.UseSharedChannel) logger.Infoln("RpcSoak done") case "channel_soak": - //soakTestConfig := interop.SoakTestConfig{ - // SoakRequestSize: *soakRequestSize, - // SoakResponseSize: *soakResponseSize, - // PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, - // MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, - // OverallTimeoutSeconds: *soakOverallTimeoutSeconds, - // ServerAddr: serverAddr, - // SoakNumThreads: *soakNumThreads, - // SoakIterations: *soakIterations, - // MaxFailures: *soakMaxFailures, - // SharedChannel: conn, - // MayCreateNewChannel: func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { - // return interop.CreateNewChannel(currentChannel, serverAddr, opts) - // }, - //} - //interop.DoSoakTest(ctxWithDeadline, conn, soakTestConfig) - - interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, - time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, - func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + channelSoakConfig := interop.SoakTestConfig{ + SoakRequestSize: *soakRequestSize, + SoakResponseSize: *soakResponseSize, + PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, + MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, + OverallTimeoutSeconds: *soakOverallTimeoutSeconds, + ServerAddr: serverAddr, + SoakNumThreads: *soakNumThreads, + SoakIterations: *soakIterations, + MaxFailures: *soakMaxFailures, + SharedChannel: conn, + MayCreateNewChannel: func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { return interop.CreateNewChannel(currentChannel, serverAddr, opts) - }) + }, + } + interop.DoSoakTest(ctxWithDeadline, channelSoakConfig) + + //interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, + // time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, + // func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + // return interop.CreateNewChannel(currentChannel, serverAddr, opts) + // }) logger.Infoln("ChannelSoak done") case "orca_per_rpc": interop.DoORCAPerRPCTest(ctx, tc) diff --git a/interop/test_utils.go b/interop/test_utils.go index 2a517ce8d3ba..26f8beaa113d 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -812,12 +812,6 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs CallOptions: []grpc.CallOption{grpc.Peer(&p)}, } result, err := doOneSoakIteration(ctx, iterationConfig) - //addrStr := "nil" - //if p.Addr != nil { - // addrStr = p.Addr.String() - //} else { - // fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") - //} if p.Addr != nil { fmt.Fprintf(os.Stderr, "Peer address: %v\n", p.Addr) } else { @@ -854,44 +848,33 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs // If resetChannel is false, then each RPC will be performed on tc. Otherwise, each RPC will be performed on a new // stub that is created with the provided server address and dial options. // TODO(mohanli-ml): Create SoakTestOptions as a parameter for this method. -func DoSoakTest(ctx context.Context, - conn *grpc.ClientConn, - serverAddr string, - soakNumThreads int, - soakIterations int, - maxFailures int, - soakRequestSize int, - soakResponseSize int, - perIterationMaxAcceptableLatency time.Duration, - minTimeBetweenRPCs time.Duration, - overallTimeoutSeconds int, - channelFunc ManagedChannel) { - if soakIterations%soakNumThreads != 0 { +func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { + if soakConfig.SoakIterations%soakConfig.SoakNumThreads != 0 { fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumThreads\n") } - sharedChannel := conn + //sharedChannel := conn startNs := time.Now().UnixNano() var wg sync.WaitGroup mu := sync.Mutex{} - threadResults := make([]ThreadResults, soakNumThreads) - for i := 0; i < soakNumThreads; i++ { + threadResults := make([]ThreadResults, soakConfig.SoakNumThreads) + for i := 0; i < soakConfig.SoakNumThreads; i++ { wg.Add(1) go func(threadID int) { defer wg.Done() - config := SoakTestConfig{ - SoakRequestSize: soakRequestSize, - SoakResponseSize: soakResponseSize, - PerIterationMaxAcceptableLatency: perIterationMaxAcceptableLatency, - MinTimeBetweenRPCs: minTimeBetweenRPCs, - OverallTimeoutSeconds: overallTimeoutSeconds, - ServerAddr: serverAddr, - SoakNumThreads: soakNumThreads, - SoakIterations: soakIterations, - MaxFailures: maxFailures, - SharedChannel: sharedChannel, - MayCreateNewChannel: channelFunc, - } - executeSoakTestInThread(ctx, config, startNs, threadID, &threadResults[threadID], &mu) + //config := SoakTestConfig{ + // SoakRequestSize: soakRequestSize, + // SoakResponseSize: soakResponseSize, + // PerIterationMaxAcceptableLatency: perIterationMaxAcceptableLatency, + // MinTimeBetweenRPCs: minTimeBetweenRPCs, + // OverallTimeoutSeconds: overallTimeoutSeconds, + // ServerAddr: serverAddr, + // SoakNumThreads: soakNumThreads, + // SoakIterations: soakIterations, + // MaxFailures: maxFailures, + // SharedChannel: sharedChannel, + // MayCreateNewChannel: channelFunc, + //} + executeSoakTestInThread(ctx, soakConfig, startNs, threadID, &threadResults[threadID], &mu) }(i) } // Wait for all goroutines to complete. @@ -918,16 +901,16 @@ func DoSoakTest(ctx context.Context, latencies.Print(&b) fmt.Fprintf(os.Stderr, "(server_uri: %s) soak test ran: %d / %d iterations. Total failures: %d. Latencies in milliseconds: %s\n", - serverAddr, totalIterations, soakIterations, totalFailures, b.String()) + soakConfig.ServerAddr, totalIterations, soakConfig.SoakIterations, totalFailures, b.String()) - if totalIterations != soakIterations { - fmt.Fprintf(os.Stderr, "Soak test consumed all %d seconds of time and quit early, ran %d out of %d iterations.\n", overallTimeoutSeconds, totalIterations, soakIterations) + if totalIterations != soakConfig.SoakIterations { + fmt.Fprintf(os.Stderr, "Soak test consumed all %d seconds of time and quit early, ran %d out of %d iterations.\n", soakConfig.OverallTimeoutSeconds, totalIterations, soakConfig.SoakIterations) } - if totalFailures > maxFailures { - fmt.Fprintf(os.Stderr, "Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, maxFailures) + if totalFailures > soakConfig.MaxFailures { + fmt.Fprintf(os.Stderr, "Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, soakConfig.MaxFailures) } - defer closeChannel(sharedChannel) + defer closeChannel(soakConfig.SharedChannel) } type testServer struct { diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index 1fc972bc5e55..5f022f2a94b2 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -129,32 +129,32 @@ func main() { go func(c clientConfig) { ctxWithDeadline, cancel := context.WithTimeout(ctx, time.Duration(*soakOverallTimeoutSeconds)*time.Second) defer cancel() - //soakTestConfig := interop.SoakTestConfig{ - // SoakRequestSize: *soakRequestSize, - // SoakResponseSize: *soakResponseSize, - // PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, - // MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, - // OverallTimeoutSeconds: *soakOverallTimeoutSeconds, - // ServerAddr: c.uri, - // SoakNumThreads: *soakNumThreads, - // SoakIterations: *soakIterations, - // MaxFailures: *soakMaxFailures, - // SharedChannel: c.conn, - // MayCreateNewChannel: MayCreateNewChannel, - //} - //interop.DoSoakTest(ctxWithDeadline, c.conn, soakTestConfig) - interop.DoSoakTest(ctxWithDeadline, - c.conn, - c.uri, - *soakNumThreads, - *soakIterations, - *soakMaxFailures, - *soakRequestSize, - *soakResponseSize, - time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, - time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, - *soakOverallTimeoutSeconds, - MayCreateNewChannel) + soakConfig := interop.SoakTestConfig{ + SoakRequestSize: *soakRequestSize, + SoakResponseSize: *soakResponseSize, + PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, + MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, + OverallTimeoutSeconds: *soakOverallTimeoutSeconds, + ServerAddr: c.uri, + SoakNumThreads: *soakNumThreads, + SoakIterations: *soakIterations, + MaxFailures: *soakMaxFailures, + SharedChannel: c.conn, + MayCreateNewChannel: MayCreateNewChannel, + } + interop.DoSoakTest(ctxWithDeadline, soakConfig) + //interop.DoSoakTest(ctxWithDeadline, + // c.conn, + // c.uri, + // *soakNumThreads, + // *soakIterations, + // *soakMaxFailures, + // *soakRequestSize, + // *soakResponseSize, + // time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, + // time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, + // *soakOverallTimeoutSeconds, + // MayCreateNewChannel) logger.Infof("%s test done for server: %s", *testCase, c.uri) wg.Done() }(clients[i]) From 959c59438ca043025aec144c11e6596accf85df2 Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 17 Dec 2024 17:56:06 -0800 Subject: [PATCH 23/45] Clean comments. --- interop/client/client.go | 10 ---------- interop/test_utils.go | 13 ------------- interop/xds_federation/client.go | 12 ------------ 3 files changed, 35 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index cd8418bb86fb..ae2e74471ddb 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -373,10 +373,6 @@ func main() { MayCreateNewChannel: interop.UseSharedChannel, } interop.DoSoakTest(ctxWithDeadline, rpcSoakConfig) - - //interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, - // time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, - // interop.UseSharedChannel) logger.Infoln("RpcSoak done") case "channel_soak": channelSoakConfig := interop.SoakTestConfig{ @@ -395,12 +391,6 @@ func main() { }, } interop.DoSoakTest(ctxWithDeadline, channelSoakConfig) - - //interop.DoSoakTest(ctxWithDeadline, conn, serverAddr, *soakNumThreads, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, - // time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, *soakOverallTimeoutSeconds, - // func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { - // return interop.CreateNewChannel(currentChannel, serverAddr, opts) - // }) logger.Infoln("ChannelSoak done") case "orca_per_rpc": interop.DoORCAPerRPCTest(ctx, tc) diff --git a/interop/test_utils.go b/interop/test_utils.go index 26f8beaa113d..11a2230f9f27 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -861,19 +861,6 @@ func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { wg.Add(1) go func(threadID int) { defer wg.Done() - //config := SoakTestConfig{ - // SoakRequestSize: soakRequestSize, - // SoakResponseSize: soakResponseSize, - // PerIterationMaxAcceptableLatency: perIterationMaxAcceptableLatency, - // MinTimeBetweenRPCs: minTimeBetweenRPCs, - // OverallTimeoutSeconds: overallTimeoutSeconds, - // ServerAddr: serverAddr, - // SoakNumThreads: soakNumThreads, - // SoakIterations: soakIterations, - // MaxFailures: maxFailures, - // SharedChannel: sharedChannel, - // MayCreateNewChannel: channelFunc, - //} executeSoakTestInThread(ctx, soakConfig, startNs, threadID, &threadResults[threadID], &mu) }(i) } diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index 5f022f2a94b2..6d9dc3e7fc63 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -143,18 +143,6 @@ func main() { MayCreateNewChannel: MayCreateNewChannel, } interop.DoSoakTest(ctxWithDeadline, soakConfig) - //interop.DoSoakTest(ctxWithDeadline, - // c.conn, - // c.uri, - // *soakNumThreads, - // *soakIterations, - // *soakMaxFailures, - // *soakRequestSize, - // *soakResponseSize, - // time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, - // time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, - // *soakOverallTimeoutSeconds, - // MayCreateNewChannel) logger.Infof("%s test done for server: %s", *testCase, c.uri) wg.Done() }(clients[i]) From 895af05f92f1cbac59c091f5a46bafe9aff17459 Mon Sep 17 00:00:00 2001 From: zbilun Date: Wed, 18 Dec 2024 10:15:13 -0800 Subject: [PATCH 24/45] Clean empty line. --- interop/test_utils.go | 1 - 1 file changed, 1 deletion(-) diff --git a/interop/test_utils.go b/interop/test_utils.go index 11a2230f9f27..cee657033e0f 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -796,7 +796,6 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs if ctx.Err() != nil { return } - if time.Since(time.Unix(0, startNs)) >= timeoutDuration { fmt.Printf("Test exceeded overall timeout of %d seconds, stopping...\n", config.OverallTimeoutSeconds) return From 262b98ee3a6fcc9fa1ad25dc445289fd321b7b96 Mon Sep 17 00:00:00 2001 From: zbilun Date: Sat, 28 Dec 2024 20:14:59 -0800 Subject: [PATCH 25/45] Address the second round comments. --- interop/client/client.go | 22 +++--- interop/test_utils.go | 124 +++++++++++++------------------ interop/xds_federation/client.go | 12 +-- 3 files changed, 70 insertions(+), 88 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index ae2e74471ddb..4c0def9ff84e 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -79,7 +79,7 @@ var ( soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") - soakNumThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") + soakNumWorkers = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") tlsServerName = flag.String("server_host_override", "", "The server name used to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.") additionalMetadata = flag.String("additional_metadata", "", "Additional metadata to send in each request, as a semicolon-separated list of key:value pairs.") testCase = flag.String("test_case", "large_unary", @@ -360,14 +360,14 @@ func main() { logger.Infoln("PickFirstUnary done") case "rpc_soak": rpcSoakConfig := interop.SoakTestConfig{ - SoakRequestSize: *soakRequestSize, - SoakResponseSize: *soakResponseSize, + RequestSize: *soakRequestSize, + ResponseSize: *soakResponseSize, PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, - OverallTimeoutSeconds: *soakOverallTimeoutSeconds, + OverallTimeoutSeconds: time.Duration(*soakOverallTimeoutSeconds) * time.Second, ServerAddr: serverAddr, - SoakNumThreads: *soakNumThreads, - SoakIterations: *soakIterations, + NumWorkers: *soakNumWorkers, + Iterations: *soakIterations, MaxFailures: *soakMaxFailures, SharedChannel: conn, MayCreateNewChannel: interop.UseSharedChannel, @@ -376,14 +376,14 @@ func main() { logger.Infoln("RpcSoak done") case "channel_soak": channelSoakConfig := interop.SoakTestConfig{ - SoakRequestSize: *soakRequestSize, - SoakResponseSize: *soakResponseSize, + RequestSize: *soakRequestSize, + ResponseSize: *soakResponseSize, PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, - OverallTimeoutSeconds: *soakOverallTimeoutSeconds, + OverallTimeoutSeconds: time.Duration(*soakOverallTimeoutSeconds) * time.Second, ServerAddr: serverAddr, - SoakNumThreads: *soakNumThreads, - SoakIterations: *soakIterations, + NumWorkers: *soakNumWorkers, + Iterations: *soakIterations, MaxFailures: *soakMaxFailures, SharedChannel: conn, MayCreateNewChannel: func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { diff --git a/interop/test_utils.go b/interop/test_utils.go index cee657033e0f..082af483cdad 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -685,14 +685,8 @@ func DoPickFirstUnary(ctx context.Context, tc testgrpc.TestServiceClient) { } } -// SoakIterationResult represents the latency and status results of a single iteration in the soak test. -type SoakIterationResult struct { - LatencyMs int64 - Status string // The status of the iteration -} - -// ThreadResults stores the aggregated results for a specific thread during the soak test. -type ThreadResults struct { +// WorkerResults stores the aggregated results for a specific worker during the soak test. +type WorkerResults struct { IterationsDone int Failures int Latencies *stats.Histogram @@ -703,28 +697,28 @@ type ManagedChannel func(*grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServi // SoakIterationConfig holds the parameters required for a single soak iteration. type SoakIterationConfig struct { - SoakRequestSize int // The size of the request payload in bytes. - SoakResponseSize int // The expected size of the response payload in bytes. + RequestSize int // The size of the request payload in bytes. + ResponseSize int // The expected size of the response payload in bytes. Client testgrpc.TestServiceClient // The gRPC client to make the call. CallOptions []grpc.CallOption // Call options for the RPC. } // SoakTestConfig holds the configuration for the entire soak test. type SoakTestConfig struct { - SoakRequestSize int - SoakResponseSize int + RequestSize int + ResponseSize int PerIterationMaxAcceptableLatency time.Duration MinTimeBetweenRPCs time.Duration - OverallTimeoutSeconds int + OverallTimeoutSeconds time.Duration ServerAddr string - SoakNumThreads int - SoakIterations int + NumWorkers int + Iterations int MaxFailures int SharedChannel *grpc.ClientConn MayCreateNewChannel ManagedChannel } -// createChannel initializes the shared channel (once for all threads). +// createChannel initializes the shared channel (once for all workers). func createChannel(serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { conn, err := grpc.NewClient(serverAddr, dopts...) if err != nil { @@ -746,8 +740,7 @@ func closeChannel(channel *grpc.ClientConn) { // CreateNewChannel creates a new channel by shutting down the current one (for channel soak tests). func CreateNewChannel(currentChannel *grpc.ClientConn, serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { closeChannel(currentChannel) - conn, client := createChannel(serverAddr, dopts) - return conn, client + return createChannel(serverAddr, dopts) } // UseSharedChannel reuses the provided currentChannel. @@ -756,48 +749,45 @@ func UseSharedChannel(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgr return currentChannel, client } -func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (SoakIterationResult, error) { +func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latencyMs int64, err error) { start := time.Now() // Do a large-unary RPC. // Create the request payload. - pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, config.SoakRequestSize) + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, config.RequestSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseSize: int32(config.SoakResponseSize), + ResponseSize: int32(config.ResponseSize), Payload: pl, } // Perform the GRPC call. var reply *testpb.SimpleResponse - reply, err := config.Client.UnaryCall(ctx, req, config.CallOptions...) + reply, err = config.Client.UnaryCall(ctx, req, config.CallOptions...) if err != nil { err = fmt.Errorf("/TestService/UnaryCall RPC failed: %s", err) - return SoakIterationResult{}, err + return 0, err } // Validate response. t := reply.GetPayload().GetType() s := len(reply.GetPayload().GetBody()) - if t != testpb.PayloadType_COMPRESSABLE || s != config.SoakResponseSize { - err = fmt.Errorf("got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, config.SoakResponseSize) - return SoakIterationResult{}, err + if t != testpb.PayloadType_COMPRESSABLE || s != config.ResponseSize { + err = fmt.Errorf("got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, config.ResponseSize) + return 0, err } // Calculate latency and return result. - latency := time.Since(start).Milliseconds() - return SoakIterationResult{ - LatencyMs: latency, - Status: "OK", - }, nil + latencyMs = time.Since(start).Milliseconds() + return latencyMs, nil } -func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs int64, threadID int, threadResults *ThreadResults, mu *sync.Mutex) { - timeoutDuration := time.Duration(config.OverallTimeoutSeconds) * time.Second +func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTime time.Time, workerID int, workerResults *WorkerResults) { + timeoutDuration := config.OverallTimeoutSeconds currentChannel := config.SharedChannel - soakIterationsPerThread := config.SoakIterations / config.SoakNumThreads - for i := 0; i < soakIterationsPerThread; i++ { + soakIterationsPerWorker := config.Iterations / config.NumWorkers + for i := 0; i < soakIterationsPerWorker; i++ { if ctx.Err() != nil { return } - if time.Since(time.Unix(0, startNs)) >= timeoutDuration { - fmt.Printf("Test exceeded overall timeout of %d seconds, stopping...\n", config.OverallTimeoutSeconds) + if time.Since(startTime) >= timeoutDuration { + fmt.Printf("Test exceeded overall timeout of %v, stopping...\n", config.OverallTimeoutSeconds) return } earliestNextStart := time.After(config.MinTimeBetweenRPCs) @@ -805,40 +795,33 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs _, client := config.MayCreateNewChannel(currentChannel) var p peer.Peer iterationConfig := SoakIterationConfig{ - SoakRequestSize: config.SoakRequestSize, - SoakResponseSize: config.SoakResponseSize, + RequestSize: config.RequestSize, + ResponseSize: config.ResponseSize, Client: client, CallOptions: []grpc.CallOption{grpc.Peer(&p)}, } - result, err := doOneSoakIteration(ctx, iterationConfig) + latencyMs, err := doOneSoakIteration(ctx, iterationConfig) if p.Addr != nil { fmt.Fprintf(os.Stderr, "Peer address: %v\n", p.Addr) } else { fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") } if err != nil { - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s failed: %s\n", threadID, i, 0, p.Addr, config.ServerAddr, err) - mu.Lock() - threadResults.Failures++ - mu.Unlock() + fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s failed: %s\n", workerID, i, 0, p.Addr, config.ServerAddr, err) + workerResults.Failures++ <-earliestNextStart continue } - latencyMs := result.LatencyMs if latencyMs > config.PerIterationMaxAcceptableLatency.Milliseconds() { - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", threadID, i, latencyMs, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) - mu.Lock() - threadResults.Failures++ - mu.Unlock() + fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", workerID, i, latencyMs, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) + workerResults.Failures++ <-earliestNextStart continue } // Success: log the details of the iteration. - mu.Lock() - threadResults.Latencies.Add(latencyMs) - threadResults.IterationsDone++ - mu.Unlock() - fmt.Fprintf(os.Stderr, "Thread %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", threadID, i, latencyMs, p.Addr, config.ServerAddr) + workerResults.Latencies.Add(latencyMs) + workerResults.IterationsDone++ + fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", workerID, i, latencyMs, p.Addr, config.ServerAddr) <-earliestNextStart } } @@ -848,19 +831,18 @@ func executeSoakTestInThread(ctx context.Context, config SoakTestConfig, startNs // stub that is created with the provided server address and dial options. // TODO(mohanli-ml): Create SoakTestOptions as a parameter for this method. func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { - if soakConfig.SoakIterations%soakConfig.SoakNumThreads != 0 { - fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumThreads\n") + if soakConfig.Iterations%soakConfig.NumWorkers != 0 { + fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumWorkers\n") } //sharedChannel := conn - startNs := time.Now().UnixNano() + startTime := time.Now() var wg sync.WaitGroup - mu := sync.Mutex{} - threadResults := make([]ThreadResults, soakConfig.SoakNumThreads) - for i := 0; i < soakConfig.SoakNumThreads; i++ { + workerResults := make([]WorkerResults, soakConfig.NumWorkers) + for i := 0; i < soakConfig.NumWorkers; i++ { wg.Add(1) - go func(threadID int) { + go func(workerID int) { defer wg.Done() - executeSoakTestInThread(ctx, soakConfig, startNs, threadID, &threadResults[threadID], &mu) + executeSoakTestInWorker(ctx, soakConfig, startTime, workerID, &workerResults[workerID]) }(i) } // Wait for all goroutines to complete. @@ -875,22 +857,22 @@ func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { BaseBucketSize: 1, MinValue: 0, }) - for _, thread := range threadResults { - totalIterations += thread.IterationsDone - totalFailures += thread.Failures - if thread.Latencies != nil { - // Add latencies from the thread's Histogram to the main latencies. - latencies.Merge(thread.Latencies) + for _, worker := range workerResults { + totalIterations += worker.IterationsDone + totalFailures += worker.Failures + if worker.Latencies != nil { + // Add latencies from the worker's Histogram to the main latencies. + latencies.Merge(worker.Latencies) } } var b bytes.Buffer latencies.Print(&b) fmt.Fprintf(os.Stderr, "(server_uri: %s) soak test ran: %d / %d iterations. Total failures: %d. Latencies in milliseconds: %s\n", - soakConfig.ServerAddr, totalIterations, soakConfig.SoakIterations, totalFailures, b.String()) + soakConfig.ServerAddr, totalIterations, soakConfig.Iterations, totalFailures, b.String()) - if totalIterations != soakConfig.SoakIterations { - fmt.Fprintf(os.Stderr, "Soak test consumed all %d seconds of time and quit early, ran %d out of %d iterations.\n", soakConfig.OverallTimeoutSeconds, totalIterations, soakConfig.SoakIterations) + if totalIterations != soakConfig.Iterations { + fmt.Fprintf(os.Stderr, "Soak test consumed all %v of time and quit early, ran %d out of %d iterations.\n", soakConfig.OverallTimeoutSeconds, totalIterations, soakConfig.Iterations) } if totalFailures > soakConfig.MaxFailures { diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index 6d9dc3e7fc63..2f6f4d07cce4 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -54,7 +54,7 @@ var ( soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") - soakNumThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") + soakNumWorkers = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") testCase = flag.String("test_case", "rpc_soak", `Configure different test cases. Valid options are: rpc_soak: sends --soak_iterations large_unary RPCs; @@ -130,14 +130,14 @@ func main() { ctxWithDeadline, cancel := context.WithTimeout(ctx, time.Duration(*soakOverallTimeoutSeconds)*time.Second) defer cancel() soakConfig := interop.SoakTestConfig{ - SoakRequestSize: *soakRequestSize, - SoakResponseSize: *soakResponseSize, + RequestSize: *soakRequestSize, + ResponseSize: *soakResponseSize, PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, - OverallTimeoutSeconds: *soakOverallTimeoutSeconds, + OverallTimeoutSeconds: time.Duration(*soakOverallTimeoutSeconds) * time.Second, ServerAddr: c.uri, - SoakNumThreads: *soakNumThreads, - SoakIterations: *soakIterations, + NumWorkers: *soakNumWorkers, + Iterations: *soakIterations, MaxFailures: *soakMaxFailures, SharedChannel: c.conn, MayCreateNewChannel: MayCreateNewChannel, From 1282e49722f6b4d7d76e843996fd21f92a70a858 Mon Sep 17 00:00:00 2001 From: zbilun Date: Sat, 28 Dec 2024 20:18:57 -0800 Subject: [PATCH 26/45] Fix the format issues. --- interop/client/client.go | 16 ++++++++-------- interop/test_utils.go | 10 +++++----- interop/xds_federation/client.go | 8 ++++---- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index 4c0def9ff84e..0c4f1ada8463 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -360,14 +360,14 @@ func main() { logger.Infoln("PickFirstUnary done") case "rpc_soak": rpcSoakConfig := interop.SoakTestConfig{ - RequestSize: *soakRequestSize, - ResponseSize: *soakResponseSize, + RequestSize: *soakRequestSize, + ResponseSize: *soakResponseSize, PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, OverallTimeoutSeconds: time.Duration(*soakOverallTimeoutSeconds) * time.Second, ServerAddr: serverAddr, - NumWorkers: *soakNumWorkers, - Iterations: *soakIterations, + NumWorkers: *soakNumWorkers, + Iterations: *soakIterations, MaxFailures: *soakMaxFailures, SharedChannel: conn, MayCreateNewChannel: interop.UseSharedChannel, @@ -376,14 +376,14 @@ func main() { logger.Infoln("RpcSoak done") case "channel_soak": channelSoakConfig := interop.SoakTestConfig{ - RequestSize: *soakRequestSize, - ResponseSize: *soakResponseSize, + RequestSize: *soakRequestSize, + ResponseSize: *soakResponseSize, PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, OverallTimeoutSeconds: time.Duration(*soakOverallTimeoutSeconds) * time.Second, ServerAddr: serverAddr, - NumWorkers: *soakNumWorkers, - Iterations: *soakIterations, + NumWorkers: *soakNumWorkers, + Iterations: *soakIterations, MaxFailures: *soakMaxFailures, SharedChannel: conn, MayCreateNewChannel: func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { diff --git a/interop/test_utils.go b/interop/test_utils.go index 082af483cdad..e1d8ee356f73 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -699,13 +699,13 @@ type ManagedChannel func(*grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServi type SoakIterationConfig struct { RequestSize int // The size of the request payload in bytes. ResponseSize int // The expected size of the response payload in bytes. - Client testgrpc.TestServiceClient // The gRPC client to make the call. - CallOptions []grpc.CallOption // Call options for the RPC. + Client testgrpc.TestServiceClient // The gRPC client to make the call. + CallOptions []grpc.CallOption // Call options for the RPC. } // SoakTestConfig holds the configuration for the entire soak test. type SoakTestConfig struct { - RequestSize int + RequestSize int ResponseSize int PerIterationMaxAcceptableLatency time.Duration MinTimeBetweenRPCs time.Duration @@ -797,8 +797,8 @@ func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTi iterationConfig := SoakIterationConfig{ RequestSize: config.RequestSize, ResponseSize: config.ResponseSize, - Client: client, - CallOptions: []grpc.CallOption{grpc.Peer(&p)}, + Client: client, + CallOptions: []grpc.CallOption{grpc.Peer(&p)}, } latencyMs, err := doOneSoakIteration(ctx, iterationConfig) if p.Addr != nil { diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index 2f6f4d07cce4..00b352eb1d1f 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -130,14 +130,14 @@ func main() { ctxWithDeadline, cancel := context.WithTimeout(ctx, time.Duration(*soakOverallTimeoutSeconds)*time.Second) defer cancel() soakConfig := interop.SoakTestConfig{ - RequestSize: *soakRequestSize, - ResponseSize: *soakResponseSize, + RequestSize: *soakRequestSize, + ResponseSize: *soakResponseSize, PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, OverallTimeoutSeconds: time.Duration(*soakOverallTimeoutSeconds) * time.Second, ServerAddr: c.uri, - NumWorkers: *soakNumWorkers, - Iterations: *soakIterations, + NumWorkers: *soakNumWorkers, + Iterations: *soakIterations, MaxFailures: *soakMaxFailures, SharedChannel: c.conn, MayCreateNewChannel: MayCreateNewChannel, From 5ef8eca3155696bb31b5b51d235b3285a1ecb5b0 Mon Sep 17 00:00:00 2001 From: zbilun Date: Sat, 28 Dec 2024 20:41:06 -0800 Subject: [PATCH 27/45] Fix the naming check. --- interop/client/client.go | 4 ++-- interop/test_utils.go | 8 ++++---- interop/xds_federation/client.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index 0c4f1ada8463..9918a3bbdf07 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -364,7 +364,7 @@ func main() { ResponseSize: *soakResponseSize, PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, - OverallTimeoutSeconds: time.Duration(*soakOverallTimeoutSeconds) * time.Second, + OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, ServerAddr: serverAddr, NumWorkers: *soakNumWorkers, Iterations: *soakIterations, @@ -380,7 +380,7 @@ func main() { ResponseSize: *soakResponseSize, PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, - OverallTimeoutSeconds: time.Duration(*soakOverallTimeoutSeconds) * time.Second, + OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, ServerAddr: serverAddr, NumWorkers: *soakNumWorkers, Iterations: *soakIterations, diff --git a/interop/test_utils.go b/interop/test_utils.go index e1d8ee356f73..752d04c10bc0 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -709,7 +709,7 @@ type SoakTestConfig struct { ResponseSize int PerIterationMaxAcceptableLatency time.Duration MinTimeBetweenRPCs time.Duration - OverallTimeoutSeconds time.Duration + OverallTimeout time.Duration ServerAddr string NumWorkers int Iterations int @@ -779,7 +779,7 @@ func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latenc } func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTime time.Time, workerID int, workerResults *WorkerResults) { - timeoutDuration := config.OverallTimeoutSeconds + timeoutDuration := config.OverallTimeout currentChannel := config.SharedChannel soakIterationsPerWorker := config.Iterations / config.NumWorkers for i := 0; i < soakIterationsPerWorker; i++ { @@ -787,7 +787,7 @@ func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTi return } if time.Since(startTime) >= timeoutDuration { - fmt.Printf("Test exceeded overall timeout of %v, stopping...\n", config.OverallTimeoutSeconds) + fmt.Printf("Test exceeded overall timeout of %v, stopping...\n", config.OverallTimeout) return } earliestNextStart := time.After(config.MinTimeBetweenRPCs) @@ -872,7 +872,7 @@ func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { soakConfig.ServerAddr, totalIterations, soakConfig.Iterations, totalFailures, b.String()) if totalIterations != soakConfig.Iterations { - fmt.Fprintf(os.Stderr, "Soak test consumed all %v of time and quit early, ran %d out of %d iterations.\n", soakConfig.OverallTimeoutSeconds, totalIterations, soakConfig.Iterations) + fmt.Fprintf(os.Stderr, "Soak test consumed all %v of time and quit early, ran %d out of %d iterations.\n", soakConfig.OverallTimeout, totalIterations, soakConfig.Iterations) } if totalFailures > soakConfig.MaxFailures { diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index 00b352eb1d1f..8c8c84d84ada 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -134,7 +134,7 @@ func main() { ResponseSize: *soakResponseSize, PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, - OverallTimeoutSeconds: time.Duration(*soakOverallTimeoutSeconds) * time.Second, + OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, ServerAddr: c.uri, NumWorkers: *soakNumWorkers, Iterations: *soakIterations, From 64fc618b08d9176a776dd3d6b1ecdf90b1a6a75e Mon Sep 17 00:00:00 2001 From: zbilun Date: Sat, 28 Dec 2024 20:47:54 -0800 Subject: [PATCH 28/45] Remove .gitignore file from repository --- .gitignore | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 13bbe5b9112f..000000000000 --- a/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.idea/ - From 85e243890a59f0667c570556a48cfd4a6799cb33 Mon Sep 17 00:00:00 2001 From: zbilun Date: Sun, 29 Dec 2024 23:36:05 -0800 Subject: [PATCH 29/45] Refactor the common config. --- interop/client/client.go | 76 ++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index 9918a3bbdf07..f6543d28aa12 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -150,6 +150,22 @@ func parseAdditionalMetadataFlag() []string { return addMd } +// createSoakTestConfig creates a shared configuration structure for soak tests. +func createBaseSoakConfig(serverAddr string, conn *grpc.ClientConn) interop.SoakTestConfig { + return interop.SoakTestConfig{ + RequestSize: *soakRequestSize, + ResponseSize: *soakResponseSize, + PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, + MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, + OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, + ServerAddr: serverAddr, + NumWorkers: *soakNumWorkers, + Iterations: *soakIterations, + MaxFailures: *soakMaxFailures, + SharedChannel: conn, + } +} + func main() { flag.Parse() logger.Infof("Client running with test case %q", *testCase) @@ -359,36 +375,42 @@ func main() { interop.DoPickFirstUnary(ctx, tc) logger.Infoln("PickFirstUnary done") case "rpc_soak": - rpcSoakConfig := interop.SoakTestConfig{ - RequestSize: *soakRequestSize, - ResponseSize: *soakResponseSize, - PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, - MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, - OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, - ServerAddr: serverAddr, - NumWorkers: *soakNumWorkers, - Iterations: *soakIterations, - MaxFailures: *soakMaxFailures, - SharedChannel: conn, - MayCreateNewChannel: interop.UseSharedChannel, - } + //rpcSoakConfig := interop.SoakTestConfig{ + // RequestSize: *soakRequestSize, + // ResponseSize: *soakResponseSize, + // PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, + // MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, + // OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, + // ServerAddr: serverAddr, + // NumWorkers: *soakNumWorkers, + // Iterations: *soakIterations, + // MaxFailures: *soakMaxFailures, + // SharedChannel: conn, + // MayCreateNewChannel: interop.UseSharedChannel, + //} + rpcSoakConfig := createBaseSoakConfig(serverAddr, conn) + rpcSoakConfig.MayCreateNewChannel = interop.UseSharedChannel interop.DoSoakTest(ctxWithDeadline, rpcSoakConfig) logger.Infoln("RpcSoak done") case "channel_soak": - channelSoakConfig := interop.SoakTestConfig{ - RequestSize: *soakRequestSize, - ResponseSize: *soakResponseSize, - PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, - MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, - OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, - ServerAddr: serverAddr, - NumWorkers: *soakNumWorkers, - Iterations: *soakIterations, - MaxFailures: *soakMaxFailures, - SharedChannel: conn, - MayCreateNewChannel: func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { - return interop.CreateNewChannel(currentChannel, serverAddr, opts) - }, + //channelSoakConfig := interop.SoakTestConfig{ + // RequestSize: *soakRequestSize, + // ResponseSize: *soakResponseSize, + // PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, + // MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, + // OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, + // ServerAddr: serverAddr, + // NumWorkers: *soakNumWorkers, + // Iterations: *soakIterations, + // MaxFailures: *soakMaxFailures, + // SharedChannel: conn, + // MayCreateNewChannel: func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + // return interop.CreateNewChannel(currentChannel, serverAddr, opts) + // }, + //} + channelSoakConfig := createBaseSoakConfig(serverAddr, conn) + channelSoakConfig.MayCreateNewChannel = func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + return interop.CreateNewChannel(currentChannel, serverAddr, opts) } interop.DoSoakTest(ctxWithDeadline, channelSoakConfig) logger.Infoln("ChannelSoak done") From 0395c79e37e3f730dd3ff4e83b3bea065d75fa86 Mon Sep 17 00:00:00 2001 From: zbilun Date: Sun, 29 Dec 2024 23:49:29 -0800 Subject: [PATCH 30/45] Clean comments. --- interop/client/client.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index f6543d28aa12..1bce0cd5a271 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -375,39 +375,11 @@ func main() { interop.DoPickFirstUnary(ctx, tc) logger.Infoln("PickFirstUnary done") case "rpc_soak": - //rpcSoakConfig := interop.SoakTestConfig{ - // RequestSize: *soakRequestSize, - // ResponseSize: *soakResponseSize, - // PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, - // MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, - // OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, - // ServerAddr: serverAddr, - // NumWorkers: *soakNumWorkers, - // Iterations: *soakIterations, - // MaxFailures: *soakMaxFailures, - // SharedChannel: conn, - // MayCreateNewChannel: interop.UseSharedChannel, - //} rpcSoakConfig := createBaseSoakConfig(serverAddr, conn) rpcSoakConfig.MayCreateNewChannel = interop.UseSharedChannel interop.DoSoakTest(ctxWithDeadline, rpcSoakConfig) logger.Infoln("RpcSoak done") case "channel_soak": - //channelSoakConfig := interop.SoakTestConfig{ - // RequestSize: *soakRequestSize, - // ResponseSize: *soakResponseSize, - // PerIterationMaxAcceptableLatency: time.Duration(*soakPerIterationMaxAcceptableLatencyMs) * time.Millisecond, - // MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, - // OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, - // ServerAddr: serverAddr, - // NumWorkers: *soakNumWorkers, - // Iterations: *soakIterations, - // MaxFailures: *soakMaxFailures, - // SharedChannel: conn, - // MayCreateNewChannel: func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { - // return interop.CreateNewChannel(currentChannel, serverAddr, opts) - // }, - //} channelSoakConfig := createBaseSoakConfig(serverAddr, conn) channelSoakConfig.MayCreateNewChannel = func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { return interop.CreateNewChannel(currentChannel, serverAddr, opts) From 8d9fb6bcf20fe1a0b669d9de3e2b22ce16dcaf35 Mon Sep 17 00:00:00 2001 From: zbilun Date: Tue, 7 Jan 2025 23:56:20 -0800 Subject: [PATCH 31/45] Clean comments. --- interop/client/client.go | 18 +++++-- interop/test_utils.go | 83 +++++++++++++++++--------------- interop/xds_federation/client.go | 27 +++++++---- 3 files changed, 76 insertions(+), 52 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index 1bce0cd5a271..e74dc5258849 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -79,7 +79,7 @@ var ( soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") - soakNumWorkers = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") + soakNumThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") tlsServerName = flag.String("server_host_override", "", "The server name used to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.") additionalMetadata = flag.String("additional_metadata", "", "Additional metadata to send in each request, as a semicolon-separated list of key:value pairs.") testCase = flag.String("test_case", "large_unary", @@ -159,7 +159,7 @@ func createBaseSoakConfig(serverAddr string, conn *grpc.ClientConn) interop.Soak MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, ServerAddr: serverAddr, - NumWorkers: *soakNumWorkers, + NumWorkers: *soakNumThreads, Iterations: *soakIterations, MaxFailures: *soakMaxFailures, SharedChannel: conn, @@ -376,14 +376,22 @@ func main() { logger.Infoln("PickFirstUnary done") case "rpc_soak": rpcSoakConfig := createBaseSoakConfig(serverAddr, conn) - rpcSoakConfig.MayCreateNewChannel = interop.UseSharedChannel + rpcSoakConfig.ChannelForTest = func() (*grpc.ClientConn, func()) { return sharedConn, func() {} } + //rpcSoakConfig.MayCreateNewChannel = interop.UseSharedChannel interop.DoSoakTest(ctxWithDeadline, rpcSoakConfig) logger.Infoln("RpcSoak done") case "channel_soak": channelSoakConfig := createBaseSoakConfig(serverAddr, conn) - channelSoakConfig.MayCreateNewChannel = func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { - return interop.CreateNewChannel(currentChannel, serverAddr, opts) + channelSoakConfig.ChannelForTest = func() (*grpc.ClientConn, func()) { + cc, err := grpc.NewClient(serverAddr, opts...) + if err != nil { + log.Fatal("Failed to create shared channel: %v", err) + } + return cc, func() { cc.Close() /* returns an error, so unfortunately needs wrapping in this closure */ } } + //channelSoakConfig.MayCreateNewChannel = func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + // return interop.CreateNewChannel(currentChannel, serverAddr, opts) + //} interop.DoSoakTest(ctxWithDeadline, channelSoakConfig) logger.Infoln("ChannelSoak done") case "orca_per_rpc": diff --git a/interop/test_utils.go b/interop/test_utils.go index 752d04c10bc0..ecd15e74fab7 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -692,8 +692,8 @@ type WorkerResults struct { Latencies *stats.Histogram } -// ManagedChannel determines whether a new channel will be created or an existing one reused. -type ManagedChannel func(*grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) +//// ManagedChannel determines whether a new channel will be created or an existing one reused. +//type ManagedChannel func(*grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) // SoakIterationConfig holds the parameters required for a single soak iteration. type SoakIterationConfig struct { @@ -714,40 +714,40 @@ type SoakTestConfig struct { NumWorkers int Iterations int MaxFailures int + ChannelForTest func() (*grpc.ClientConn, func()) SharedChannel *grpc.ClientConn - MayCreateNewChannel ManagedChannel } -// createChannel initializes the shared channel (once for all workers). -func createChannel(serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { - conn, err := grpc.NewClient(serverAddr, dopts...) - if err != nil { - log.Fatalf("Failed to create shared channel: %v", err) - } - client := testgrpc.NewTestServiceClient(conn) - return conn, client -} - -func closeChannel(channel *grpc.ClientConn) { - if channel != nil { - err := channel.Close() - if err != nil { - log.Fatalf("Failed to close channel: %v", err) - } - } -} - -// CreateNewChannel creates a new channel by shutting down the current one (for channel soak tests). -func CreateNewChannel(currentChannel *grpc.ClientConn, serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { - closeChannel(currentChannel) - return createChannel(serverAddr, dopts) -} - -// UseSharedChannel reuses the provided currentChannel. -func UseSharedChannel(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { - client := testgrpc.NewTestServiceClient(currentChannel) - return currentChannel, client -} +//// createChannel initializes the shared channel (once for all workers). +//func createChannel(serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { +// conn, err := grpc.NewClient(serverAddr, dopts...) +// if err != nil { +// log.Fatalf("Failed to create shared channel: %v", err) +// } +// client := testgrpc.NewTestServiceClient(conn) +// return conn, client +//} + +//func closeChannel(channel *grpc.ClientConn) { +// if channel != nil { +// err := channel.Close() +// if err != nil { +// log.Fatalf("Failed to close channel: %v", err) +// } +// } +//} + +//// CreateNewChannel creates a new channel by shutting down the current one (for channel soak tests). +//func CreateNewChannel(currentChannel *grpc.ClientConn, serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { +// closeChannel(currentChannel) +// return createChannel(serverAddr, dopts) +//} +// +//// UseSharedChannel reuses the provided currentChannel. +//func UseSharedChannel(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { +// client := testgrpc.NewTestServiceClient(currentChannel) +// return currentChannel, client +//} func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latencyMs int64, err error) { start := time.Now() @@ -780,7 +780,7 @@ func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latenc func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTime time.Time, workerID int, workerResults *WorkerResults) { timeoutDuration := config.OverallTimeout - currentChannel := config.SharedChannel + //currentChannel := config.SharedChannel soakIterationsPerWorker := config.Iterations / config.NumWorkers for i := 0; i < soakIterationsPerWorker; i++ { if ctx.Err() != nil { @@ -791,8 +791,11 @@ func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTi return } earliestNextStart := time.After(config.MinTimeBetweenRPCs) - // Get the channel and client from the provided channelFunc (either shared or new). - _, client := config.MayCreateNewChannel(currentChannel) + //// Get the channel and client from the provided channelFunc (either shared or new). + //_, client := config.MayCreateNewChannel(currentChannel) + currentChannel, cleanup := config.ChannelForTest() + defer cleanup() + client := testgrpc.NewTestServiceClient(currentChannel) var p peer.Peer iterationConfig := SoakIterationConfig{ RequestSize: config.RequestSize, @@ -832,7 +835,7 @@ func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTi // TODO(mohanli-ml): Create SoakTestOptions as a parameter for this method. func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { if soakConfig.Iterations%soakConfig.NumWorkers != 0 { - fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumWorkers\n") + fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumWThreads\n") } //sharedChannel := conn startTime := time.Now() @@ -878,7 +881,11 @@ func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { if totalFailures > soakConfig.MaxFailures { fmt.Fprintf(os.Stderr, "Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, soakConfig.MaxFailures) } - defer closeChannel(soakConfig.SharedChannel) + //defer closeChannel(soakConfig.SharedChannel) + if soakConfig.ChannelForTest != nil { + _, cleanup := soakConfig.ChannelForTest() + defer cleanup() + } } type testServer struct { diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index 8c8c84d84ada..ca6cacc5e270 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -54,7 +54,7 @@ var ( soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") - soakNumWorkers = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") + soakNumThreads = flag.Int("soak_num_threads", 1, "The number of threads for concurrent execution of the soak tests (rpc_soak or channel_soak). The default value is set based on the interop large unary test case.") testCase = flag.String("test_case", "rpc_soak", `Configure different test cases. Valid options are: rpc_soak: sends --soak_iterations large_unary RPCs; @@ -84,16 +84,25 @@ func main() { } } var clients []clientConfig - var MayCreateNewChannel interop.ManagedChannel + //var MayCreateNewChannel interop.ManagedChannel + var ChannelForTest func() (*grpc.ClientConn, func()) switch *testCase { case "rpc_soak": - MayCreateNewChannel = interop.UseSharedChannel + //MayCreateNewChannel = interop.UseSharedChannel + ChannelForTest = func() (*grpc.ClientConn, func()) { return sharedConn, func() {} } case "channel_soak": - MayCreateNewChannel = func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { - for _, client := range clients { - return interop.CreateNewChannel(currentChannel, client.uri, client.opts) + //MayCreateNewChannel = func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { + // for _, client := range clients { + // return interop.CreateNewChannel(currentChannel, client.uri, client.opts) + // } + // return nil, nil + //} + ChannelForTest = func() (*grpc.ClientConn, func()) { + cc, err := grpc.NewClient(serverAddr, opts...) + if err != nil { + log.Fatal("Failed to create shared channel: %v", err) } - return nil, nil + return cc, func() { cc.Close() /* returns an error, so unfortunately needs wrapping in this closure */ } } default: logger.Fatal("Unsupported test case: ", *testCase) @@ -136,11 +145,11 @@ func main() { MinTimeBetweenRPCs: time.Duration(*soakMinTimeMsBetweenRPCs) * time.Millisecond, OverallTimeout: time.Duration(*soakOverallTimeoutSeconds) * time.Second, ServerAddr: c.uri, - NumWorkers: *soakNumWorkers, + NumWorkers: *soakNumThreads, Iterations: *soakIterations, MaxFailures: *soakMaxFailures, SharedChannel: c.conn, - MayCreateNewChannel: MayCreateNewChannel, + ChannelForTest: ChannelForTest, } interop.DoSoakTest(ctxWithDeadline, soakConfig) logger.Infof("%s test done for server: %s", *testCase, c.uri) From 63e82502743900e5b3ec8aa8634203b279060f15 Mon Sep 17 00:00:00 2001 From: zbilun Date: Wed, 8 Jan 2025 01:03:07 -0800 Subject: [PATCH 32/45] Fix check issues. --- interop/client/client.go | 5 ++-- interop/test_utils.go | 1 - interop/xds_federation/client.go | 40 +++++++++++++------------------- 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index e74dc5258849..bf6c6f5b7e9b 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -28,6 +28,7 @@ import ( "crypto/tls" "crypto/x509" "flag" + "log" "net" "os" "strconv" @@ -376,7 +377,7 @@ func main() { logger.Infoln("PickFirstUnary done") case "rpc_soak": rpcSoakConfig := createBaseSoakConfig(serverAddr, conn) - rpcSoakConfig.ChannelForTest = func() (*grpc.ClientConn, func()) { return sharedConn, func() {} } + rpcSoakConfig.ChannelForTest = func() (*grpc.ClientConn, func()) { return conn, func() {} } //rpcSoakConfig.MayCreateNewChannel = interop.UseSharedChannel interop.DoSoakTest(ctxWithDeadline, rpcSoakConfig) logger.Infoln("RpcSoak done") @@ -385,7 +386,7 @@ func main() { channelSoakConfig.ChannelForTest = func() (*grpc.ClientConn, func()) { cc, err := grpc.NewClient(serverAddr, opts...) if err != nil { - log.Fatal("Failed to create shared channel: %v", err) + log.Fatalf("Failed to create shared channel: %v", err) } return cc, func() { cc.Close() /* returns an error, so unfortunately needs wrapping in this closure */ } } diff --git a/interop/test_utils.go b/interop/test_utils.go index ecd15e74fab7..42a87966ded2 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -28,7 +28,6 @@ import ( "context" "fmt" "io" - "log" "os" "strings" "sync" diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index ca6cacc5e270..7f5adad1fe88 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -22,6 +22,7 @@ package main import ( "context" "flag" + "log" "strings" "sync" "time" @@ -84,30 +85,6 @@ func main() { } } var clients []clientConfig - //var MayCreateNewChannel interop.ManagedChannel - var ChannelForTest func() (*grpc.ClientConn, func()) - switch *testCase { - case "rpc_soak": - //MayCreateNewChannel = interop.UseSharedChannel - ChannelForTest = func() (*grpc.ClientConn, func()) { return sharedConn, func() {} } - case "channel_soak": - //MayCreateNewChannel = func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { - // for _, client := range clients { - // return interop.CreateNewChannel(currentChannel, client.uri, client.opts) - // } - // return nil, nil - //} - ChannelForTest = func() (*grpc.ClientConn, func()) { - cc, err := grpc.NewClient(serverAddr, opts...) - if err != nil { - log.Fatal("Failed to create shared channel: %v", err) - } - return cc, func() { cc.Close() /* returns an error, so unfortunately needs wrapping in this closure */ } - } - default: - logger.Fatal("Unsupported test case: ", *testCase) - } - for i := range uris { var opts []grpc.DialOption switch creds[i] { @@ -132,12 +109,27 @@ func main() { // run soak tests with the different clients logger.Infof("Clients running with test case %q", *testCase) var wg sync.WaitGroup + var ChannelForTest func() (*grpc.ClientConn, func()) ctx := context.Background() for i := range clients { wg.Add(1) go func(c clientConfig) { ctxWithDeadline, cancel := context.WithTimeout(ctx, time.Duration(*soakOverallTimeoutSeconds)*time.Second) defer cancel() + switch *testCase { + case "rpc_soak": + ChannelForTest = func() (*grpc.ClientConn, func()) { return c.conn, func() {} } + case "channel_soak": + ChannelForTest = func() (*grpc.ClientConn, func()) { + cc, err := grpc.NewClient(c.uri, c.opts...) + if err != nil { + log.Fatalf("Failed to create shared channel: %v", err) + } + return cc, func() { cc.Close() /* returns an error, so unfortunately needs wrapping in this closure */ } + } + default: + logger.Fatal("Unsupported test case: ", *testCase) + } soakConfig := interop.SoakTestConfig{ RequestSize: *soakRequestSize, ResponseSize: *soakResponseSize, From 9c60ca8129f1b02323db1b745f4e01c8fc63931c Mon Sep 17 00:00:00 2001 From: zbilun Date: Wed, 8 Jan 2025 01:21:31 -0800 Subject: [PATCH 33/45] Improve latency handling and clean comments. --- interop/client/client.go | 4 ---- interop/test_utils.go | 47 ++++------------------------------------ 2 files changed, 4 insertions(+), 47 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index bf6c6f5b7e9b..bc3851b330bc 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -378,7 +378,6 @@ func main() { case "rpc_soak": rpcSoakConfig := createBaseSoakConfig(serverAddr, conn) rpcSoakConfig.ChannelForTest = func() (*grpc.ClientConn, func()) { return conn, func() {} } - //rpcSoakConfig.MayCreateNewChannel = interop.UseSharedChannel interop.DoSoakTest(ctxWithDeadline, rpcSoakConfig) logger.Infoln("RpcSoak done") case "channel_soak": @@ -390,9 +389,6 @@ func main() { } return cc, func() { cc.Close() /* returns an error, so unfortunately needs wrapping in this closure */ } } - //channelSoakConfig.MayCreateNewChannel = func(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { - // return interop.CreateNewChannel(currentChannel, serverAddr, opts) - //} interop.DoSoakTest(ctxWithDeadline, channelSoakConfig) logger.Infoln("ChannelSoak done") case "orca_per_rpc": diff --git a/interop/test_utils.go b/interop/test_utils.go index 42a87966ded2..c1f944b45038 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -691,9 +691,6 @@ type WorkerResults struct { Latencies *stats.Histogram } -//// ManagedChannel determines whether a new channel will be created or an existing one reused. -//type ManagedChannel func(*grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) - // SoakIterationConfig holds the parameters required for a single soak iteration. type SoakIterationConfig struct { RequestSize int // The size of the request payload in bytes. @@ -717,38 +714,7 @@ type SoakTestConfig struct { SharedChannel *grpc.ClientConn } -//// createChannel initializes the shared channel (once for all workers). -//func createChannel(serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { -// conn, err := grpc.NewClient(serverAddr, dopts...) -// if err != nil { -// log.Fatalf("Failed to create shared channel: %v", err) -// } -// client := testgrpc.NewTestServiceClient(conn) -// return conn, client -//} - -//func closeChannel(channel *grpc.ClientConn) { -// if channel != nil { -// err := channel.Close() -// if err != nil { -// log.Fatalf("Failed to close channel: %v", err) -// } -// } -//} - -//// CreateNewChannel creates a new channel by shutting down the current one (for channel soak tests). -//func CreateNewChannel(currentChannel *grpc.ClientConn, serverAddr string, dopts []grpc.DialOption) (*grpc.ClientConn, testgrpc.TestServiceClient) { -// closeChannel(currentChannel) -// return createChannel(serverAddr, dopts) -//} -// -//// UseSharedChannel reuses the provided currentChannel. -//func UseSharedChannel(currentChannel *grpc.ClientConn) (*grpc.ClientConn, testgrpc.TestServiceClient) { -// client := testgrpc.NewTestServiceClient(currentChannel) -// return currentChannel, client -//} - -func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latencyMs int64, err error) { +func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latencyMs time.Duration, err error) { start := time.Now() // Do a large-unary RPC. // Create the request payload. @@ -773,13 +739,12 @@ func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latenc return 0, err } // Calculate latency and return result. - latencyMs = time.Since(start).Milliseconds() + latencyMs = time.Since(start) return latencyMs, nil } func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTime time.Time, workerID int, workerResults *WorkerResults) { timeoutDuration := config.OverallTimeout - //currentChannel := config.SharedChannel soakIterationsPerWorker := config.Iterations / config.NumWorkers for i := 0; i < soakIterationsPerWorker; i++ { if ctx.Err() != nil { @@ -790,8 +755,6 @@ func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTi return } earliestNextStart := time.After(config.MinTimeBetweenRPCs) - //// Get the channel and client from the provided channelFunc (either shared or new). - //_, client := config.MayCreateNewChannel(currentChannel) currentChannel, cleanup := config.ChannelForTest() defer cleanup() client := testgrpc.NewTestServiceClient(currentChannel) @@ -814,14 +777,14 @@ func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTi <-earliestNextStart continue } - if latencyMs > config.PerIterationMaxAcceptableLatency.Milliseconds() { + if latencyMs > config.PerIterationMaxAcceptableLatency { fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", workerID, i, latencyMs, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) workerResults.Failures++ <-earliestNextStart continue } // Success: log the details of the iteration. - workerResults.Latencies.Add(latencyMs) + workerResults.Latencies.Add(latencyMs.Milliseconds()) workerResults.IterationsDone++ fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", workerID, i, latencyMs, p.Addr, config.ServerAddr) <-earliestNextStart @@ -836,7 +799,6 @@ func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { if soakConfig.Iterations%soakConfig.NumWorkers != 0 { fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumWThreads\n") } - //sharedChannel := conn startTime := time.Now() var wg sync.WaitGroup workerResults := make([]WorkerResults, soakConfig.NumWorkers) @@ -880,7 +842,6 @@ func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { if totalFailures > soakConfig.MaxFailures { fmt.Fprintf(os.Stderr, "Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, soakConfig.MaxFailures) } - //defer closeChannel(soakConfig.SharedChannel) if soakConfig.ChannelForTest != nil { _, cleanup := soakConfig.ChannelForTest() defer cleanup() From c0d62385c3c62bae75eaca14cbe9800fd0ab9acb Mon Sep 17 00:00:00 2001 From: zbilun Date: Wed, 8 Jan 2025 01:31:56 -0800 Subject: [PATCH 34/45] Update latency var name. --- interop/test_utils.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/interop/test_utils.go b/interop/test_utils.go index c1f944b45038..e4c6119fea5f 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -714,7 +714,7 @@ type SoakTestConfig struct { SharedChannel *grpc.ClientConn } -func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latencyMs time.Duration, err error) { +func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latency time.Duration, err error) { start := time.Now() // Do a large-unary RPC. // Create the request payload. @@ -739,8 +739,8 @@ func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latenc return 0, err } // Calculate latency and return result. - latencyMs = time.Since(start) - return latencyMs, nil + latency = time.Since(start) + return latency, nil } func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTime time.Time, workerID int, workerResults *WorkerResults) { @@ -765,7 +765,7 @@ func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTi Client: client, CallOptions: []grpc.CallOption{grpc.Peer(&p)}, } - latencyMs, err := doOneSoakIteration(ctx, iterationConfig) + latency, err := doOneSoakIteration(ctx, iterationConfig) if p.Addr != nil { fmt.Fprintf(os.Stderr, "Peer address: %v\n", p.Addr) } else { @@ -777,16 +777,16 @@ func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTi <-earliestNextStart continue } - if latencyMs > config.PerIterationMaxAcceptableLatency { - fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", workerID, i, latencyMs, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) + if latency > config.PerIterationMaxAcceptableLatency { + fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", workerID, i, latency, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) workerResults.Failures++ <-earliestNextStart continue } // Success: log the details of the iteration. - workerResults.Latencies.Add(latencyMs.Milliseconds()) + workerResults.Latencies.Add(latency.Milliseconds()) workerResults.IterationsDone++ - fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", workerID, i, latencyMs, p.Addr, config.ServerAddr) + fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", workerID, i, latency, p.Addr, config.ServerAddr) <-earliestNextStart } } From 24a233aba695442f772bcabbe5176e158cc3903a Mon Sep 17 00:00:00 2001 From: zbilun Date: Wed, 8 Jan 2025 13:46:53 -0800 Subject: [PATCH 35/45] Clean comments. --- interop/client/client.go | 2 +- interop/xds_federation/client.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index bc3851b330bc..cc1c1b02b0fc 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -387,7 +387,7 @@ func main() { if err != nil { log.Fatalf("Failed to create shared channel: %v", err) } - return cc, func() { cc.Close() /* returns an error, so unfortunately needs wrapping in this closure */ } + return cc, func() { cc.Close() } } interop.DoSoakTest(ctxWithDeadline, channelSoakConfig) logger.Infoln("ChannelSoak done") diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index 7f5adad1fe88..c8c3b4f220dc 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -125,7 +125,7 @@ func main() { if err != nil { log.Fatalf("Failed to create shared channel: %v", err) } - return cc, func() { cc.Close() /* returns an error, so unfortunately needs wrapping in this closure */ } + return cc, func() { cc.Close() } } default: logger.Fatal("Unsupported test case: ", *testCase) From 5ef9068ed892b243042bc892af5bc8ca8611722d Mon Sep 17 00:00:00 2001 From: zbilun Date: Wed, 8 Jan 2025 16:30:10 -0800 Subject: [PATCH 36/45] Refactor the big files. --- interop/soak_tests.go | 181 ++++++++++++++++++++++++++++++++++++++++++ interop/test_utils.go | 167 -------------------------------------- 2 files changed, 181 insertions(+), 167 deletions(-) create mode 100644 interop/soak_tests.go diff --git a/interop/soak_tests.go b/interop/soak_tests.go new file mode 100644 index 000000000000..50fe2f5e79fa --- /dev/null +++ b/interop/soak_tests.go @@ -0,0 +1,181 @@ +package interop + +import ( + "bytes" + "context" + "fmt" + "os" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/benchmark/stats" + "google.golang.org/grpc/peer" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// WorkerResults stores the aggregated results for a specific worker during the soak test. +type WorkerResults struct { + IterationsDone int + Failures int + Latencies *stats.Histogram +} + +// SoakIterationConfig holds the parameters required for a single soak iteration. +type SoakIterationConfig struct { + RequestSize int // The size of the request payload in bytes. + ResponseSize int // The expected size of the response payload in bytes. + Client testgrpc.TestServiceClient // The gRPC client to make the call. + CallOptions []grpc.CallOption // Call options for the RPC. +} + +// SoakTestConfig holds the configuration for the entire soak test. +type SoakTestConfig struct { + RequestSize int + ResponseSize int + PerIterationMaxAcceptableLatency time.Duration + MinTimeBetweenRPCs time.Duration + OverallTimeout time.Duration + ServerAddr string + NumWorkers int + Iterations int + MaxFailures int + ChannelForTest func() (*grpc.ClientConn, func()) + SharedChannel *grpc.ClientConn +} + +func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latency time.Duration, err error) { + start := time.Now() + // Do a large-unary RPC. + // Create the request payload. + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, config.RequestSize) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(config.ResponseSize), + Payload: pl, + } + // Perform the GRPC call. + var reply *testpb.SimpleResponse + reply, err = config.Client.UnaryCall(ctx, req, config.CallOptions...) + if err != nil { + err = fmt.Errorf("/TestService/UnaryCall RPC failed: %s", err) + return 0, err + } + // Validate response. + t := reply.GetPayload().GetType() + s := len(reply.GetPayload().GetBody()) + if t != testpb.PayloadType_COMPRESSABLE || s != config.ResponseSize { + err = fmt.Errorf("got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, config.ResponseSize) + return 0, err + } + // Calculate latency and return result. + latency = time.Since(start) + return latency, nil +} + +func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTime time.Time, workerID int, workerResults *WorkerResults) { + timeoutDuration := config.OverallTimeout + soakIterationsPerWorker := config.Iterations / config.NumWorkers + for i := 0; i < soakIterationsPerWorker; i++ { + if ctx.Err() != nil { + return + } + if time.Since(startTime) >= timeoutDuration { + fmt.Printf("Test exceeded overall timeout of %v, stopping...\n", config.OverallTimeout) + return + } + earliestNextStart := time.After(config.MinTimeBetweenRPCs) + currentChannel, cleanup := config.ChannelForTest() + defer cleanup() + client := testgrpc.NewTestServiceClient(currentChannel) + var p peer.Peer + iterationConfig := SoakIterationConfig{ + RequestSize: config.RequestSize, + ResponseSize: config.ResponseSize, + Client: client, + CallOptions: []grpc.CallOption{grpc.Peer(&p)}, + } + latency, err := doOneSoakIteration(ctx, iterationConfig) + if p.Addr != nil { + fmt.Fprintf(os.Stderr, "Peer address: %v\n", p.Addr) + } else { + fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") + } + if err != nil { + fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s failed: %s\n", workerID, i, 0, p.Addr, config.ServerAddr, err) + workerResults.Failures++ + <-earliestNextStart + continue + } + if latency > config.PerIterationMaxAcceptableLatency { + fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", workerID, i, latency, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) + workerResults.Failures++ + <-earliestNextStart + continue + } + // Success: log the details of the iteration. + workerResults.Latencies.Add(latency.Milliseconds()) + workerResults.IterationsDone++ + fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", workerID, i, latency, p.Addr, config.ServerAddr) + <-earliestNextStart + } +} + +// DoSoakTest runs large unary RPCs in a loop for a configurable number of times, with configurable failure thresholds. +// If resetChannel is false, then each RPC will be performed on tc. Otherwise, each RPC will be performed on a new +// stub that is created with the provided server address and dial options. +// TODO(mohanli-ml): Create SoakTestOptions as a parameter for this method. +func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { + if soakConfig.Iterations%soakConfig.NumWorkers != 0 { + fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumWThreads\n") + } + startTime := time.Now() + var wg sync.WaitGroup + workerResults := make([]WorkerResults, soakConfig.NumWorkers) + for i := 0; i < soakConfig.NumWorkers; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + executeSoakTestInWorker(ctx, soakConfig, startTime, workerID, &workerResults[workerID]) + }(i) + } + // Wait for all goroutines to complete. + wg.Wait() + + //Handle results. + totalIterations := 0 + totalFailures := 0 + latencies := stats.NewHistogram(stats.HistogramOptions{ + NumBuckets: 20, + GrowthFactor: 1, + BaseBucketSize: 1, + MinValue: 0, + }) + for _, worker := range workerResults { + totalIterations += worker.IterationsDone + totalFailures += worker.Failures + if worker.Latencies != nil { + // Add latencies from the worker's Histogram to the main latencies. + latencies.Merge(worker.Latencies) + } + } + var b bytes.Buffer + latencies.Print(&b) + fmt.Fprintf(os.Stderr, + "(server_uri: %s) soak test ran: %d / %d iterations. Total failures: %d. Latencies in milliseconds: %s\n", + soakConfig.ServerAddr, totalIterations, soakConfig.Iterations, totalFailures, b.String()) + + if totalIterations != soakConfig.Iterations { + fmt.Fprintf(os.Stderr, "Soak test consumed all %v of time and quit early, ran %d out of %d iterations.\n", soakConfig.OverallTimeout, totalIterations, soakConfig.Iterations) + } + + if totalFailures > soakConfig.MaxFailures { + fmt.Fprintf(os.Stderr, "Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, soakConfig.MaxFailures) + } + if soakConfig.ChannelForTest != nil { + _, cleanup := soakConfig.ChannelForTest() + defer cleanup() + } +} diff --git a/interop/test_utils.go b/interop/test_utils.go index e4c6119fea5f..3aa7bd307a9b 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -24,7 +24,6 @@ package interop import ( - "bytes" "context" "fmt" "io" @@ -36,12 +35,10 @@ import ( "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/grpc" - "google.golang.org/grpc/benchmark/stats" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/metadata" "google.golang.org/grpc/orca" - "google.golang.org/grpc/peer" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" @@ -684,170 +681,6 @@ func DoPickFirstUnary(ctx context.Context, tc testgrpc.TestServiceClient) { } } -// WorkerResults stores the aggregated results for a specific worker during the soak test. -type WorkerResults struct { - IterationsDone int - Failures int - Latencies *stats.Histogram -} - -// SoakIterationConfig holds the parameters required for a single soak iteration. -type SoakIterationConfig struct { - RequestSize int // The size of the request payload in bytes. - ResponseSize int // The expected size of the response payload in bytes. - Client testgrpc.TestServiceClient // The gRPC client to make the call. - CallOptions []grpc.CallOption // Call options for the RPC. -} - -// SoakTestConfig holds the configuration for the entire soak test. -type SoakTestConfig struct { - RequestSize int - ResponseSize int - PerIterationMaxAcceptableLatency time.Duration - MinTimeBetweenRPCs time.Duration - OverallTimeout time.Duration - ServerAddr string - NumWorkers int - Iterations int - MaxFailures int - ChannelForTest func() (*grpc.ClientConn, func()) - SharedChannel *grpc.ClientConn -} - -func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latency time.Duration, err error) { - start := time.Now() - // Do a large-unary RPC. - // Create the request payload. - pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, config.RequestSize) - req := &testpb.SimpleRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseSize: int32(config.ResponseSize), - Payload: pl, - } - // Perform the GRPC call. - var reply *testpb.SimpleResponse - reply, err = config.Client.UnaryCall(ctx, req, config.CallOptions...) - if err != nil { - err = fmt.Errorf("/TestService/UnaryCall RPC failed: %s", err) - return 0, err - } - // Validate response. - t := reply.GetPayload().GetType() - s := len(reply.GetPayload().GetBody()) - if t != testpb.PayloadType_COMPRESSABLE || s != config.ResponseSize { - err = fmt.Errorf("got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, config.ResponseSize) - return 0, err - } - // Calculate latency and return result. - latency = time.Since(start) - return latency, nil -} - -func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTime time.Time, workerID int, workerResults *WorkerResults) { - timeoutDuration := config.OverallTimeout - soakIterationsPerWorker := config.Iterations / config.NumWorkers - for i := 0; i < soakIterationsPerWorker; i++ { - if ctx.Err() != nil { - return - } - if time.Since(startTime) >= timeoutDuration { - fmt.Printf("Test exceeded overall timeout of %v, stopping...\n", config.OverallTimeout) - return - } - earliestNextStart := time.After(config.MinTimeBetweenRPCs) - currentChannel, cleanup := config.ChannelForTest() - defer cleanup() - client := testgrpc.NewTestServiceClient(currentChannel) - var p peer.Peer - iterationConfig := SoakIterationConfig{ - RequestSize: config.RequestSize, - ResponseSize: config.ResponseSize, - Client: client, - CallOptions: []grpc.CallOption{grpc.Peer(&p)}, - } - latency, err := doOneSoakIteration(ctx, iterationConfig) - if p.Addr != nil { - fmt.Fprintf(os.Stderr, "Peer address: %v\n", p.Addr) - } else { - fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") - } - if err != nil { - fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s failed: %s\n", workerID, i, 0, p.Addr, config.ServerAddr, err) - workerResults.Failures++ - <-earliestNextStart - continue - } - if latency > config.PerIterationMaxAcceptableLatency { - fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", workerID, i, latency, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) - workerResults.Failures++ - <-earliestNextStart - continue - } - // Success: log the details of the iteration. - workerResults.Latencies.Add(latency.Milliseconds()) - workerResults.IterationsDone++ - fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", workerID, i, latency, p.Addr, config.ServerAddr) - <-earliestNextStart - } -} - -// DoSoakTest runs large unary RPCs in a loop for a configurable number of times, with configurable failure thresholds. -// If resetChannel is false, then each RPC will be performed on tc. Otherwise, each RPC will be performed on a new -// stub that is created with the provided server address and dial options. -// TODO(mohanli-ml): Create SoakTestOptions as a parameter for this method. -func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { - if soakConfig.Iterations%soakConfig.NumWorkers != 0 { - fmt.Fprintf(os.Stderr, "soakIterations must be evenly divisible by soakNumWThreads\n") - } - startTime := time.Now() - var wg sync.WaitGroup - workerResults := make([]WorkerResults, soakConfig.NumWorkers) - for i := 0; i < soakConfig.NumWorkers; i++ { - wg.Add(1) - go func(workerID int) { - defer wg.Done() - executeSoakTestInWorker(ctx, soakConfig, startTime, workerID, &workerResults[workerID]) - }(i) - } - // Wait for all goroutines to complete. - wg.Wait() - - //Handle results. - totalIterations := 0 - totalFailures := 0 - latencies := stats.NewHistogram(stats.HistogramOptions{ - NumBuckets: 20, - GrowthFactor: 1, - BaseBucketSize: 1, - MinValue: 0, - }) - for _, worker := range workerResults { - totalIterations += worker.IterationsDone - totalFailures += worker.Failures - if worker.Latencies != nil { - // Add latencies from the worker's Histogram to the main latencies. - latencies.Merge(worker.Latencies) - } - } - var b bytes.Buffer - latencies.Print(&b) - fmt.Fprintf(os.Stderr, - "(server_uri: %s) soak test ran: %d / %d iterations. Total failures: %d. Latencies in milliseconds: %s\n", - soakConfig.ServerAddr, totalIterations, soakConfig.Iterations, totalFailures, b.String()) - - if totalIterations != soakConfig.Iterations { - fmt.Fprintf(os.Stderr, "Soak test consumed all %v of time and quit early, ran %d out of %d iterations.\n", soakConfig.OverallTimeout, totalIterations, soakConfig.Iterations) - } - - if totalFailures > soakConfig.MaxFailures { - fmt.Fprintf(os.Stderr, "Soak test total failures: %d exceeded max failures threshold: %d\n", totalFailures, soakConfig.MaxFailures) - } - if soakConfig.ChannelForTest != nil { - _, cleanup := soakConfig.ChannelForTest() - defer cleanup() - } -} - type testServer struct { testgrpc.UnimplementedTestServiceServer From f680c8f559800a8cb104bbfaaa9ef010bb8315e6 Mon Sep 17 00:00:00 2001 From: zbilun Date: Wed, 8 Jan 2025 16:34:44 -0800 Subject: [PATCH 37/45] Add copyrights. --- interop/soak_tests.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/interop/soak_tests.go b/interop/soak_tests.go index 50fe2f5e79fa..2423be683cb8 100644 --- a/interop/soak_tests.go +++ b/interop/soak_tests.go @@ -1,3 +1,27 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package interop contains functions used by interop client/server. +// +// See interop test case descriptions [here]. +// +// [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md +// client starts an interop client to do rpc_soak test and channel_soak test package interop import ( From f4cfaf938259c1300825ba23781ba94d96763cfc Mon Sep 17 00:00:00 2001 From: zbilun Date: Wed, 8 Jan 2025 16:47:05 -0800 Subject: [PATCH 38/45] Modify copyrights. --- interop/soak_tests.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interop/soak_tests.go b/interop/soak_tests.go index 2423be683cb8..fedfeb0f07d9 100644 --- a/interop/soak_tests.go +++ b/interop/soak_tests.go @@ -17,11 +17,11 @@ */ // Package interop contains functions used by interop client/server. +// client starts an interop client to do rpc_soak test and channel_soak test. // // See interop test case descriptions [here]. // // [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md -// client starts an interop client to do rpc_soak test and channel_soak test package interop import ( From eacc6e02ef6270fc2fef5de92fd7faabd338668d Mon Sep 17 00:00:00 2001 From: zbilun Date: Wed, 8 Jan 2025 17:00:21 -0800 Subject: [PATCH 39/45] Modify imports. --- interop/soak_tests.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/interop/soak_tests.go b/interop/soak_tests.go index fedfeb0f07d9..f47d34f2c818 100644 --- a/interop/soak_tests.go +++ b/interop/soak_tests.go @@ -24,6 +24,21 @@ // [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md package interop +//import ( +// "bytes" +// "context" +// "fmt" +// "os" +// "sync" +// "time" +// +// "google.golang.org/grpc" +// "google.golang.org/grpc/benchmark/stats" +// "google.golang.org/grpc/peer" +// +// testgrpc "google.golang.org/grpc/interop/grpc_testing" +// testpb "google.golang.org/grpc/interop/grpc_testing" +//) import ( "bytes" "context" From ca01d13ee3dc3391f6198eda8ebbb36d9bb8bf92 Mon Sep 17 00:00:00 2001 From: zbilun Date: Wed, 8 Jan 2025 17:11:25 -0800 Subject: [PATCH 40/45] Clean comments. --- interop/soak_tests.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/interop/soak_tests.go b/interop/soak_tests.go index f47d34f2c818..fedfeb0f07d9 100644 --- a/interop/soak_tests.go +++ b/interop/soak_tests.go @@ -24,21 +24,6 @@ // [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md package interop -//import ( -// "bytes" -// "context" -// "fmt" -// "os" -// "sync" -// "time" -// -// "google.golang.org/grpc" -// "google.golang.org/grpc/benchmark/stats" -// "google.golang.org/grpc/peer" -// -// testgrpc "google.golang.org/grpc/interop/grpc_testing" -// testpb "google.golang.org/grpc/interop/grpc_testing" -//) import ( "bytes" "context" From 8265b0b452398221a0f5c1efbcc5159f5940bc89 Mon Sep 17 00:00:00 2001 From: zbilun Date: Wed, 8 Jan 2025 17:16:01 -0800 Subject: [PATCH 41/45] Clean comments. --- interop/soak_tests.go | 1 - 1 file changed, 1 deletion(-) diff --git a/interop/soak_tests.go b/interop/soak_tests.go index fedfeb0f07d9..007a804517a8 100644 --- a/interop/soak_tests.go +++ b/interop/soak_tests.go @@ -16,7 +16,6 @@ * */ -// Package interop contains functions used by interop client/server. // client starts an interop client to do rpc_soak test and channel_soak test. // // See interop test case descriptions [here]. From 76bfa6538ddc070772597223cb7092c188919491 Mon Sep 17 00:00:00 2001 From: zbilun Date: Thu, 9 Jan 2025 13:17:19 -0800 Subject: [PATCH 42/45] Address comments. --- interop/client/client.go | 1 - interop/soak_tests.go | 31 ++++++++++--------------------- interop/xds_federation/client.go | 9 ++++----- 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index cc1c1b02b0fc..b1b96d5f3bb9 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -163,7 +163,6 @@ func createBaseSoakConfig(serverAddr string, conn *grpc.ClientConn) interop.Soak NumWorkers: *soakNumThreads, Iterations: *soakIterations, MaxFailures: *soakMaxFailures, - SharedChannel: conn, } } diff --git a/interop/soak_tests.go b/interop/soak_tests.go index 007a804517a8..b667d3678a80 100644 --- a/interop/soak_tests.go +++ b/interop/soak_tests.go @@ -16,11 +16,6 @@ * */ -// client starts an interop client to do rpc_soak test and channel_soak test. -// -// See interop test case descriptions [here]. -// -// [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md package interop import ( @@ -39,8 +34,8 @@ import ( testpb "google.golang.org/grpc/interop/grpc_testing" ) -// WorkerResults stores the aggregated results for a specific worker during the soak test. -type WorkerResults struct { +// SoakWorkerResults stores the aggregated results for a specific worker during the soak test. +type SoakWorkerResults struct { IterationsDone int Failures int Latencies *stats.Histogram @@ -66,7 +61,6 @@ type SoakTestConfig struct { Iterations int MaxFailures int ChannelForTest func() (*grpc.ClientConn, func()) - SharedChannel *grpc.ClientConn } func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latency time.Duration, err error) { @@ -98,7 +92,7 @@ func doOneSoakIteration(ctx context.Context, config SoakIterationConfig) (latenc return latency, nil } -func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTime time.Time, workerID int, workerResults *WorkerResults) { +func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTime time.Time, workerID int, soakWorkerResults *SoakWorkerResults) { timeoutDuration := config.OverallTimeout soakIterationsPerWorker := config.Iterations / config.NumWorkers for i := 0; i < soakIterationsPerWorker; i++ { @@ -121,26 +115,21 @@ func executeSoakTestInWorker(ctx context.Context, config SoakTestConfig, startTi CallOptions: []grpc.CallOption{grpc.Peer(&p)}, } latency, err := doOneSoakIteration(ctx, iterationConfig) - if p.Addr != nil { - fmt.Fprintf(os.Stderr, "Peer address: %v\n", p.Addr) - } else { - fmt.Fprintf(os.Stderr, "No peer address available for this RPC.\n") - } if err != nil { fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s failed: %s\n", workerID, i, 0, p.Addr, config.ServerAddr, err) - workerResults.Failures++ + soakWorkerResults.Failures++ <-earliestNextStart continue } if latency > config.PerIterationMaxAcceptableLatency { fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s exceeds max acceptable latency: %d\n", workerID, i, latency, p.Addr, config.ServerAddr, config.PerIterationMaxAcceptableLatency.Milliseconds()) - workerResults.Failures++ + soakWorkerResults.Failures++ <-earliestNextStart continue } // Success: log the details of the iteration. - workerResults.Latencies.Add(latency.Milliseconds()) - workerResults.IterationsDone++ + soakWorkerResults.Latencies.Add(latency.Milliseconds()) + soakWorkerResults.IterationsDone++ fmt.Fprintf(os.Stderr, "Worker %d: soak iteration: %d elapsed_ms: %d peer: %v server_uri: %s succeeded\n", workerID, i, latency, p.Addr, config.ServerAddr) <-earliestNextStart } @@ -156,12 +145,12 @@ func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { } startTime := time.Now() var wg sync.WaitGroup - workerResults := make([]WorkerResults, soakConfig.NumWorkers) + soakWorkerResults := make([]SoakWorkerResults, soakConfig.NumWorkers) for i := 0; i < soakConfig.NumWorkers; i++ { wg.Add(1) go func(workerID int) { defer wg.Done() - executeSoakTestInWorker(ctx, soakConfig, startTime, workerID, &workerResults[workerID]) + executeSoakTestInWorker(ctx, soakConfig, startTime, workerID, &soakWorkerResults[workerID]) }(i) } // Wait for all goroutines to complete. @@ -176,7 +165,7 @@ func DoSoakTest(ctx context.Context, soakConfig SoakTestConfig) { BaseBucketSize: 1, MinValue: 0, }) - for _, worker := range workerResults { + for _, worker := range soakWorkerResults { totalIterations += worker.IterationsDone totalFailures += worker.Failures if worker.Latencies != nil { diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go index c8c3b4f220dc..1d210369f089 100644 --- a/interop/xds_federation/client.go +++ b/interop/xds_federation/client.go @@ -109,7 +109,7 @@ func main() { // run soak tests with the different clients logger.Infof("Clients running with test case %q", *testCase) var wg sync.WaitGroup - var ChannelForTest func() (*grpc.ClientConn, func()) + var channelForTest func() (*grpc.ClientConn, func()) ctx := context.Background() for i := range clients { wg.Add(1) @@ -118,9 +118,9 @@ func main() { defer cancel() switch *testCase { case "rpc_soak": - ChannelForTest = func() (*grpc.ClientConn, func()) { return c.conn, func() {} } + channelForTest = func() (*grpc.ClientConn, func()) { return c.conn, func() {} } case "channel_soak": - ChannelForTest = func() (*grpc.ClientConn, func()) { + channelForTest = func() (*grpc.ClientConn, func()) { cc, err := grpc.NewClient(c.uri, c.opts...) if err != nil { log.Fatalf("Failed to create shared channel: %v", err) @@ -140,8 +140,7 @@ func main() { NumWorkers: *soakNumThreads, Iterations: *soakIterations, MaxFailures: *soakMaxFailures, - SharedChannel: c.conn, - ChannelForTest: ChannelForTest, + ChannelForTest: channelForTest, } interop.DoSoakTest(ctxWithDeadline, soakConfig) logger.Infof("%s test done for server: %s", *testCase, c.uri) From 51b457dd97f6fce76439ea6b627845da11627fe1 Mon Sep 17 00:00:00 2001 From: zbilun Date: Thu, 9 Jan 2025 13:30:37 -0800 Subject: [PATCH 43/45] Modify time.After limitaion. --- scripts/vet.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/vet.sh b/scripts/vet.sh index 4867e8c677b6..24c7c2cec51d 100755 --- a/scripts/vet.sh +++ b/scripts/vet.sh @@ -49,15 +49,15 @@ git grep 'func [A-Z]' -- "*_test.go" | not grep -v 'func Test\|Benchmark\|Exampl # - Do not use time.After except in tests. It has the potential to leak the # timer since there is no way to stop it early. -git grep -l 'time.After(' -- "*.go" | not grep -v '_test.go\|test_utils\|testutils' +git grep -l 'time.After(' -- "*.go" | not grep -v '_test.go\|soak_tests\|testutils' # - Do not use "interface{}"; use "any" instead. -git grep -l 'interface{}' -- "*.go" 2>&1 | not grep -v '\.pb\.go\|protoc-gen-go-grpc\|grpc_testing_not_regenerated' +git grep -l 'interface{}' -- "*.go" 2>&1 | not grep -v '\.pb\.go\|protoc-gen-go-grpc\|grpc_testing_not_regenerated" # - Do not call grpclog directly. Use grpclog.Component instead. git grep -l -e 'grpclog.I' --or -e 'grpclog.W' --or -e 'grpclog.E' --or -e 'grpclog.F' --or -e 'grpclog.V' -- "*.go" | not grep -v '^grpclog/component.go\|^internal/grpctest/tlogger_test.go\|^internal/grpclog/prefix_logger.go' -# - Ensure that the deprecated protobuf dependency is not used. +# - Ensure that the deprecated protobuf dependency is not used not git grep "\"github.com/golang/protobuf/*" -- "*.go" ':(exclude)testdata/grpc_testing_not_regenerated/*' # - Ensure all usages of grpc_testing package are renamed when importing. From 3c8b2f48d503a795e906f7feb73f592441818d10 Mon Sep 17 00:00:00 2001 From: zbilun Date: Thu, 9 Jan 2025 13:33:37 -0800 Subject: [PATCH 44/45] Revert wrong changes. --- scripts/vet.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/vet.sh b/scripts/vet.sh index 24c7c2cec51d..98e15998ad00 100755 --- a/scripts/vet.sh +++ b/scripts/vet.sh @@ -52,12 +52,12 @@ git grep 'func [A-Z]' -- "*_test.go" | not grep -v 'func Test\|Benchmark\|Exampl git grep -l 'time.After(' -- "*.go" | not grep -v '_test.go\|soak_tests\|testutils' # - Do not use "interface{}"; use "any" instead. -git grep -l 'interface{}' -- "*.go" 2>&1 | not grep -v '\.pb\.go\|protoc-gen-go-grpc\|grpc_testing_not_regenerated" +git grep -l 'interface{}' -- "*.go" 2>&1 | not grep -v '\.pb\.go\|protoc-gen-go-grpc\|grpc_testing_not_regenerated' # - Do not call grpclog directly. Use grpclog.Component instead. git grep -l -e 'grpclog.I' --or -e 'grpclog.W' --or -e 'grpclog.E' --or -e 'grpclog.F' --or -e 'grpclog.V' -- "*.go" | not grep -v '^grpclog/component.go\|^internal/grpctest/tlogger_test.go\|^internal/grpclog/prefix_logger.go' -# - Ensure that the deprecated protobuf dependency is not used +# - Ensure that the deprecated protobuf dependency is not used. not git grep "\"github.com/golang/protobuf/*" -- "*.go" ':(exclude)testdata/grpc_testing_not_regenerated/*' # - Ensure all usages of grpc_testing package are renamed when importing. From 823ae79a6a232d8fd87e2f04a8b6796a14c05f80 Mon Sep 17 00:00:00 2001 From: zbilun Date: Thu, 9 Jan 2025 13:51:18 -0800 Subject: [PATCH 45/45] Clean code. --- interop/client/client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/interop/client/client.go b/interop/client/client.go index b1b96d5f3bb9..b088451cd203 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -152,7 +152,7 @@ func parseAdditionalMetadataFlag() []string { } // createSoakTestConfig creates a shared configuration structure for soak tests. -func createBaseSoakConfig(serverAddr string, conn *grpc.ClientConn) interop.SoakTestConfig { +func createBaseSoakConfig(serverAddr string) interop.SoakTestConfig { return interop.SoakTestConfig{ RequestSize: *soakRequestSize, ResponseSize: *soakResponseSize, @@ -375,12 +375,12 @@ func main() { interop.DoPickFirstUnary(ctx, tc) logger.Infoln("PickFirstUnary done") case "rpc_soak": - rpcSoakConfig := createBaseSoakConfig(serverAddr, conn) + rpcSoakConfig := createBaseSoakConfig(serverAddr) rpcSoakConfig.ChannelForTest = func() (*grpc.ClientConn, func()) { return conn, func() {} } interop.DoSoakTest(ctxWithDeadline, rpcSoakConfig) logger.Infoln("RpcSoak done") case "channel_soak": - channelSoakConfig := createBaseSoakConfig(serverAddr, conn) + channelSoakConfig := createBaseSoakConfig(serverAddr) channelSoakConfig.ChannelForTest = func() (*grpc.ClientConn, func()) { cc, err := grpc.NewClient(serverAddr, opts...) if err != nil {