Apply shutdown timeout to http server to limit reload delay (#14339)

httpServer.stop may block indefinitely, potentially due to misbehaving connections.

Apply shutdown timeout to http server so that in a hot reload, the old and new server overlapping time is bounded.

Author: carsonip

changelogs/head.asciidoc

@@ -8,6 +8,7 @@ https://github.com/elastic/apm-server/compare/8.15\...main[View commits]
 
 - Track all bulk request response status codes {pull}13574[13574]
 - Fix a concurrent map write panic in monitoring middleware {pull}14335[14335]
+- Apply shutdown timeout to http server {pull}14339[14339]
 
 [float]
 ==== Breaking Changes

internal/beater/http.go

@@ -106,9 +106,9 @@ func (h *httpServer) start() error {
 	return h.Serve(h.httpListener)
 }
 
-func (h *httpServer) stop() {
+func (h *httpServer) stop(ctx context.Context) {
 	h.logger.Infof("Stop listening on: %s", h.Server.Addr)
-	if err := h.Shutdown(context.Background()); err != nil {
+	if err := h.Shutdown(ctx); err != nil {
 		h.logger.Errorf("error stopping http server: %s", err.Error())
 		if err := h.Close(); err != nil {
 			h.logger.Errorf("error closing http server: %s", err.Error())

internal/beater/server.go

@@ -222,11 +222,10 @@ func (s server) run(ctx context.Context) error {
 	})
 	g.Go(func() error {
 		<-ctx.Done()
-		// httpServer should stop before grpcServer to avoid a panic caused by placing a new connection into
-		// a closed grpc connection channel during shutdown.
-		// See https://github.com/elastic/gmux/issues/13
-		s.httpServer.stop()
 		s.grpcServer.GracefulStop()
+		stopctx, cancel := context.WithTimeout(context.Background(), s.cfg.ShutdownTimeout)
+		defer cancel()
+		s.httpServer.stop(stopctx)
 		return nil
 	})
 	if err := g.Wait(); err != http.ErrServerClosed {