From 267f415e1f95d72a4c39da845a30c6ac160dba8a Mon Sep 17 00:00:00 2001 From: yamatcha Date: Wed, 20 Nov 2024 02:50:21 +0000 Subject: [PATCH 1/2] Add log-rotation-size option --- cmd/moco-agent/cmd/root.go | 20 +++++++++- docs/moco-agent.md | 1 + server/rotate.go | 50 +++++++++++++++++++++--- server/rotate_test.go | 78 +++++++++++++++++++++++++++++++++++--- 4 files changed, 137 insertions(+), 12 deletions(-) diff --git a/cmd/moco-agent/cmd/root.go b/cmd/moco-agent/cmd/root.go index 09be0e3..88103c5 100644 --- a/cmd/moco-agent/cmd/root.go +++ b/cmd/moco-agent/cmd/root.go @@ -52,6 +52,7 @@ var config struct { connIdleTime time.Duration connectionTimeout time.Duration logRotationSchedule string + logRotationSize int64 readTimeout time.Duration maxDelayThreshold time.Duration socketPath string @@ -127,7 +128,7 @@ var rootCmd = &cobra.Command{ metrics.Init(registry, clusterName, index) c := cron.New(cron.WithLogger(rLogger.WithName("cron"))) - if _, err := c.AddFunc(config.logRotationSchedule, agent.RotateLog); err != nil { + if _, err := c.AddFunc(config.logRotationSchedule, func() { agent.RotateErrorLog(); agent.RotateSlowLog() }); err != nil { rLogger.Error(err, "failed to parse the cron spec", "spec", config.logRotationSchedule) return err } @@ -142,6 +143,22 @@ var rootCmd = &cobra.Command{ } }() + // Rotate if the file size exceeds logRotationSize + if config.logRotationSize > 0 { + ticker := time.NewTicker(time.Second) + go func() { + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + agent.RotateLogIfSizeExceeded(config.logRotationSize) + } + } + }() + defer ticker.Stop() + } + reloader, err := cert.NewReloader(config.grpcCertDir, rLogger.WithName("cert-reloader")) if err != nil { return err @@ -239,6 +256,7 @@ func init() { fs.DurationVar(&config.connIdleTime, "max-idle-time", 30*time.Second, "The maximum amount of time a connection may be idle") fs.DurationVar(&config.connectionTimeout, "connection-timeout", 5*time.Second, "Dial timeout") fs.StringVar(&config.logRotationSchedule, "log-rotation-schedule", logRotationScheduleDefault, "Cron format schedule for MySQL log rotation") + fs.Int64Var(&config.logRotationSize, "log-rotation-size", 0, "Rotate MySQL log files when it exceeds the specified size in bytes.") fs.DurationVar(&config.readTimeout, "read-timeout", 30*time.Second, "I/O read timeout") fs.DurationVar(&config.maxDelayThreshold, "max-delay", time.Minute, "Acceptable max commit delay considering as ready; the zero value accepts any delay") fs.StringVar(&config.socketPath, "socket-path", socketPathDefault, "Path of mysqld socket file.") diff --git a/docs/moco-agent.md b/docs/moco-agent.md index d0c17d5..017b5d8 100644 --- a/docs/moco-agent.md +++ b/docs/moco-agent.md @@ -11,6 +11,7 @@ Flags: --grpc-cert-dir string gRPC certificate directory (default "/grpc-cert") -h, --help help for moco-agent --log-rotation-schedule string Cron format schedule for MySQL log rotation (default "*/5 * * * *") + --log-rotation-size int Rotate MySQL log files when it exceeds the specified size in bytes. --logfile string Log filename --logformat string Log format [plain,logfmt,json] --loglevel string Log level [critical,error,warning,info,debug] diff --git a/server/rotate.go b/server/rotate.go index cd1ced0..f4db7c0 100644 --- a/server/rotate.go +++ b/server/rotate.go @@ -10,8 +10,8 @@ import ( "github.com/cybozu-go/moco-agent/metrics" ) -// RotateLog rotates log files -func (a *Agent) RotateLog() { +// RotateLog rotates error log files +func (a *Agent) RotateErrorLog() { ctx := context.Background() metrics.LogRotationCount.Inc() @@ -25,16 +25,33 @@ func (a *Agent) RotateLog() { return } + if _, err := a.db.ExecContext(ctx, "FLUSH LOCAL ERROR LOGS"); err != nil { + a.logger.Error(err, "failed to exec FLUSH LOCAL ERROR LOGS") + metrics.LogRotationFailureCount.Inc() + return + } + + durationSeconds := time.Since(startTime).Seconds() + metrics.LogRotationDurationSeconds.Observe(durationSeconds) +} + +// RotateLog rotates slow log files +func (a *Agent) RotateSlowLog() { + ctx := context.Background() + + metrics.LogRotationCount.Inc() + startTime := time.Now() + slowFile := filepath.Join(a.logDir, mocoagent.MySQLSlowLogName) - err = os.Rename(slowFile, slowFile+".0") + err := os.Rename(slowFile, slowFile+".0") if err != nil && !os.IsNotExist(err) { a.logger.Error(err, "failed to rotate slow query log file") metrics.LogRotationFailureCount.Inc() return } - if _, err := a.db.ExecContext(ctx, "FLUSH LOCAL ERROR LOGS, SLOW LOGS"); err != nil { - a.logger.Error(err, "failed to exec FLUSH LOCAL LOGS") + if _, err := a.db.ExecContext(ctx, "FLUSH LOCAL SLOW LOGS"); err != nil { + a.logger.Error(err, "failed to exec FLUSH LOCAL SLOW LOGS") metrics.LogRotationFailureCount.Inc() return } @@ -42,3 +59,26 @@ func (a *Agent) RotateLog() { durationSeconds := time.Since(startTime).Seconds() metrics.LogRotationDurationSeconds.Observe(durationSeconds) } + +// RotateLogIfSizeExceeded rotates log files if it exceeds rotationSize +func (a *Agent) RotateLogIfSizeExceeded(rotationSize int64) { + errFile := filepath.Join(a.logDir, mocoagent.MySQLErrorLogName) + errFileStat, err := os.Stat(errFile) + if err != nil { + a.logger.Error(err, "failed to get stat of error log file") + return + } + if errFileStat.Size() > rotationSize { + a.RotateErrorLog() + } + + slowFile := filepath.Join(a.logDir, mocoagent.MySQLSlowLogName) + slowFileStat, err := os.Stat(slowFile) + if err != nil { + a.logger.Error(err, "failed to get stat of slow query log file") + return + } + if slowFileStat.Size() > rotationSize { + a.RotateSlowLog() + } +} diff --git a/server/rotate_test.go b/server/rotate_test.go index a1a9695..8dca622 100644 --- a/server/rotate_test.go +++ b/server/rotate_test.go @@ -1,6 +1,7 @@ package server import ( + "bytes" "os" "path/filepath" "time" @@ -12,7 +13,7 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" ) -var _ = Describe("log rotation", func() { +var _ = Describe("log rotation", Ordered, func() { It("should rotate logs", func() { By("starting MySQLd") StartMySQLD(replicaHost, replicaPort, replicaServerID) @@ -45,13 +46,14 @@ var _ = Describe("log rotation", func() { Expect(err).ShouldNot(HaveOccurred()) } - agent.RotateLog() + agent.RotateErrorLog() + agent.RotateSlowLog() for _, file := range logFiles { _, err := os.Stat(file + ".0") Expect(err).ShouldNot(HaveOccurred()) } - Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 1)) + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 2)) Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 0)) By("creating the same name directory") @@ -62,9 +64,73 @@ var _ = Describe("log rotation", func() { Expect(err).ShouldNot(HaveOccurred()) } - agent.RotateLog() + agent.RotateErrorLog() + agent.RotateSlowLog() - Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 2)) - Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 1)) + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 4)) + Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 2)) + }) + + It("should rotate logs by RotateLogIfSizeExceeded if size exceeds", func() { + By("starting MySQLd") + StartMySQLD(replicaHost, replicaPort, replicaServerID) + defer StopAndRemoveMySQLD(replicaHost) + + sockFile := filepath.Join(socketDir(replicaHost), "mysqld.sock") + tmpDir, err := os.MkdirTemp("", "moco-test-agent-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tmpDir) + + conf := MySQLAccessorConfig{ + Host: "localhost", + Port: replicaPort, + Password: agentUserPassword, + ConnMaxIdleTime: 30 * time.Minute, + ConnectionTimeout: 3 * time.Second, + ReadTimeout: 30 * time.Second, + } + agent, err := New(conf, testClusterName, sockFile, tmpDir, maxDelayThreshold, time.Second, testLogger) + Expect(err).ShouldNot(HaveOccurred()) + defer agent.CloseDB() + + By("preparing log files for testing") + slowFile := filepath.Join(tmpDir, mocoagent.MySQLSlowLogName) + errFile := filepath.Join(tmpDir, mocoagent.MySQLErrorLogName) + logFiles := []string{slowFile, errFile} + + logDataSize := 512 + data := bytes.Repeat([]byte("a"), logDataSize) + for _, file := range logFiles { + f, err := os.Create(file) + Expect(err).ShouldNot(HaveOccurred()) + f.Write(data) + } + + agent.RotateLogIfSizeExceeded(int64(logDataSize) + 1) + + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 4)) + Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 2)) + + agent.RotateLogIfSizeExceeded(int64(logDataSize) - 1) + + for _, file := range logFiles { + _, err := os.Stat(file + ".0") + Expect(err).ShouldNot(HaveOccurred()) + } + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 6)) + Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 2)) + + By("creating the same name directory") + for _, file := range logFiles { + err := os.Rename(file+".0", file) + Expect(err).ShouldNot(HaveOccurred()) + err = os.Mkdir(file+".0", 0777) + Expect(err).ShouldNot(HaveOccurred()) + } + + agent.RotateLogIfSizeExceeded(int64(logDataSize) - 1) + + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 8)) + Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 4)) }) }) From 9fad8c1e2c20a6ec9f3754dc4a01df94ef91c860 Mon Sep 17 00:00:00 2001 From: yamatcha Date: Wed, 27 Nov 2024 00:32:38 +0000 Subject: [PATCH 2/2] Remove rotation for error log --- cmd/moco-agent/cmd/root.go | 4 +- constants.go | 3 -- docs/moco-agent.md | 2 +- server/rotate.go | 51 ++++------------------- server/rotate_test.go | 84 +++++++++++++++----------------------- 5 files changed, 44 insertions(+), 100 deletions(-) diff --git a/cmd/moco-agent/cmd/root.go b/cmd/moco-agent/cmd/root.go index 88103c5..43fd540 100644 --- a/cmd/moco-agent/cmd/root.go +++ b/cmd/moco-agent/cmd/root.go @@ -128,7 +128,7 @@ var rootCmd = &cobra.Command{ metrics.Init(registry, clusterName, index) c := cron.New(cron.WithLogger(rLogger.WithName("cron"))) - if _, err := c.AddFunc(config.logRotationSchedule, func() { agent.RotateErrorLog(); agent.RotateSlowLog() }); err != nil { + if _, err := c.AddFunc(config.logRotationSchedule, agent.RotateLog); err != nil { rLogger.Error(err, "failed to parse the cron spec", "spec", config.logRotationSchedule) return err } @@ -256,7 +256,7 @@ func init() { fs.DurationVar(&config.connIdleTime, "max-idle-time", 30*time.Second, "The maximum amount of time a connection may be idle") fs.DurationVar(&config.connectionTimeout, "connection-timeout", 5*time.Second, "Dial timeout") fs.StringVar(&config.logRotationSchedule, "log-rotation-schedule", logRotationScheduleDefault, "Cron format schedule for MySQL log rotation") - fs.Int64Var(&config.logRotationSize, "log-rotation-size", 0, "Rotate MySQL log files when it exceeds the specified size in bytes.") + fs.Int64Var(&config.logRotationSize, "log-rotation-size", 0, "Rotate MySQL log file when it exceeds the specified size in bytes.") fs.DurationVar(&config.readTimeout, "read-timeout", 30*time.Second, "I/O read timeout") fs.DurationVar(&config.maxDelayThreshold, "max-delay", time.Minute, "Acceptable max commit delay considering as ready; the zero value accepts any delay") fs.StringVar(&config.socketPath, "socket-path", socketPathDefault, "Path of mysqld socket file.") diff --git a/constants.go b/constants.go index 84dbc8e..8e5a35e 100644 --- a/constants.go +++ b/constants.go @@ -40,9 +40,6 @@ const ( // MySQLAdminPort is a port number for MySQL Admin MySQLAdminPort = 33062 - // MySQLErrorLogName is a filekey of error log for MySQL. - MySQLErrorLogName = "mysql.err" - // MySQLSlowLogName is a filekey of slow query log for MySQL. MySQLSlowLogName = "mysql.slow" ) diff --git a/docs/moco-agent.md b/docs/moco-agent.md index 017b5d8..f768b9a 100644 --- a/docs/moco-agent.md +++ b/docs/moco-agent.md @@ -11,7 +11,7 @@ Flags: --grpc-cert-dir string gRPC certificate directory (default "/grpc-cert") -h, --help help for moco-agent --log-rotation-schedule string Cron format schedule for MySQL log rotation (default "*/5 * * * *") - --log-rotation-size int Rotate MySQL log files when it exceeds the specified size in bytes. + --log-rotation-size int Rotate MySQL log file when it exceeds the specified size in bytes. --logfile string Log filename --logformat string Log format [plain,logfmt,json] --loglevel string Log level [critical,error,warning,info,debug] diff --git a/server/rotate.go b/server/rotate.go index f4db7c0..465b5f3 100644 --- a/server/rotate.go +++ b/server/rotate.go @@ -10,33 +10,8 @@ import ( "github.com/cybozu-go/moco-agent/metrics" ) -// RotateLog rotates error log files -func (a *Agent) RotateErrorLog() { - ctx := context.Background() - - metrics.LogRotationCount.Inc() - startTime := time.Now() - - errFile := filepath.Join(a.logDir, mocoagent.MySQLErrorLogName) - err := os.Rename(errFile, errFile+".0") - if err != nil && !os.IsNotExist(err) { - a.logger.Error(err, "failed to rotate err log file") - metrics.LogRotationFailureCount.Inc() - return - } - - if _, err := a.db.ExecContext(ctx, "FLUSH LOCAL ERROR LOGS"); err != nil { - a.logger.Error(err, "failed to exec FLUSH LOCAL ERROR LOGS") - metrics.LogRotationFailureCount.Inc() - return - } - - durationSeconds := time.Since(startTime).Seconds() - metrics.LogRotationDurationSeconds.Observe(durationSeconds) -} - -// RotateLog rotates slow log files -func (a *Agent) RotateSlowLog() { +// RotateLog rotates log file +func (a *Agent) RotateLog() { ctx := context.Background() metrics.LogRotationCount.Inc() @@ -60,25 +35,15 @@ func (a *Agent) RotateSlowLog() { metrics.LogRotationDurationSeconds.Observe(durationSeconds) } -// RotateLogIfSizeExceeded rotates log files if it exceeds rotationSize +// RotateLogIfSizeExceeded rotates log file if it exceeds rotationSize func (a *Agent) RotateLogIfSizeExceeded(rotationSize int64) { - errFile := filepath.Join(a.logDir, mocoagent.MySQLErrorLogName) - errFileStat, err := os.Stat(errFile) - if err != nil { - a.logger.Error(err, "failed to get stat of error log file") - return - } - if errFileStat.Size() > rotationSize { - a.RotateErrorLog() - } - - slowFile := filepath.Join(a.logDir, mocoagent.MySQLSlowLogName) - slowFileStat, err := os.Stat(slowFile) + file := filepath.Join(a.logDir, mocoagent.MySQLSlowLogName) + fileStat, err := os.Stat(file) if err != nil { - a.logger.Error(err, "failed to get stat of slow query log file") + a.logger.Error(err, "failed to get stat of log file") return } - if slowFileStat.Size() > rotationSize { - a.RotateSlowLog() + if fileStat.Size() > rotationSize { + a.RotateLog() } } diff --git a/server/rotate_test.go b/server/rotate_test.go index 8dca622..bd23c01 100644 --- a/server/rotate_test.go +++ b/server/rotate_test.go @@ -36,39 +36,29 @@ var _ = Describe("log rotation", Ordered, func() { Expect(err).ShouldNot(HaveOccurred()) defer agent.CloseDB() - By("preparing log files for testing") - slowFile := filepath.Join(tmpDir, mocoagent.MySQLSlowLogName) - errFile := filepath.Join(tmpDir, mocoagent.MySQLErrorLogName) - logFiles := []string{slowFile, errFile} - - for _, file := range logFiles { - _, err := os.Create(file) - Expect(err).ShouldNot(HaveOccurred()) - } + By("preparing log file for testing") + logFile := filepath.Join(tmpDir, mocoagent.MySQLSlowLogName) - agent.RotateErrorLog() - agent.RotateSlowLog() + _, err = os.Create(logFile) + Expect(err).ShouldNot(HaveOccurred()) - for _, file := range logFiles { - _, err := os.Stat(file + ".0") - Expect(err).ShouldNot(HaveOccurred()) - } - Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 2)) + agent.RotateLog() + + _, err = os.Stat(logFile + ".0") + Expect(err).ShouldNot(HaveOccurred()) + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 1)) Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 0)) By("creating the same name directory") - for _, file := range logFiles { - err := os.Rename(file+".0", file) - Expect(err).ShouldNot(HaveOccurred()) - err = os.Mkdir(file+".0", 0777) - Expect(err).ShouldNot(HaveOccurred()) - } + err = os.Rename(logFile+".0", logFile) + Expect(err).ShouldNot(HaveOccurred()) + err = os.Mkdir(logFile+".0", 0777) + Expect(err).ShouldNot(HaveOccurred()) - agent.RotateErrorLog() - agent.RotateSlowLog() + agent.RotateLog() - Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 4)) - Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 2)) + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 2)) + Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 1)) }) It("should rotate logs by RotateLogIfSizeExceeded if size exceeds", func() { @@ -93,44 +83,36 @@ var _ = Describe("log rotation", Ordered, func() { Expect(err).ShouldNot(HaveOccurred()) defer agent.CloseDB() - By("preparing log files for testing") - slowFile := filepath.Join(tmpDir, mocoagent.MySQLSlowLogName) - errFile := filepath.Join(tmpDir, mocoagent.MySQLErrorLogName) - logFiles := []string{slowFile, errFile} + By("preparing log file for testing") + logFile := filepath.Join(tmpDir, mocoagent.MySQLSlowLogName) logDataSize := 512 data := bytes.Repeat([]byte("a"), logDataSize) - for _, file := range logFiles { - f, err := os.Create(file) - Expect(err).ShouldNot(HaveOccurred()) - f.Write(data) - } + f, err := os.Create(logFile) + Expect(err).ShouldNot(HaveOccurred()) + f.Write(data) agent.RotateLogIfSizeExceeded(int64(logDataSize) + 1) - Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 4)) - Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 2)) + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 2)) + Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 1)) agent.RotateLogIfSizeExceeded(int64(logDataSize) - 1) - for _, file := range logFiles { - _, err := os.Stat(file + ".0") - Expect(err).ShouldNot(HaveOccurred()) - } - Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 6)) - Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 2)) + _, err = os.Stat(logFile + ".0") + Expect(err).ShouldNot(HaveOccurred()) + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 3)) + Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 1)) By("creating the same name directory") - for _, file := range logFiles { - err := os.Rename(file+".0", file) - Expect(err).ShouldNot(HaveOccurred()) - err = os.Mkdir(file+".0", 0777) - Expect(err).ShouldNot(HaveOccurred()) - } + err = os.Rename(logFile+".0", logFile) + Expect(err).ShouldNot(HaveOccurred()) + err = os.Mkdir(logFile+".0", 0777) + Expect(err).ShouldNot(HaveOccurred()) agent.RotateLogIfSizeExceeded(int64(logDataSize) - 1) - Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 8)) - Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 4)) + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 4)) + Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 2)) }) })