diff --git a/cmd/moco-agent/cmd/root.go b/cmd/moco-agent/cmd/root.go index 09be0e3..43fd540 100644 --- a/cmd/moco-agent/cmd/root.go +++ b/cmd/moco-agent/cmd/root.go @@ -52,6 +52,7 @@ var config struct { connIdleTime time.Duration connectionTimeout time.Duration logRotationSchedule string + logRotationSize int64 readTimeout time.Duration maxDelayThreshold time.Duration socketPath string @@ -142,6 +143,22 @@ var rootCmd = &cobra.Command{ } }() + // Rotate if the file size exceeds logRotationSize + if config.logRotationSize > 0 { + ticker := time.NewTicker(time.Second) + go func() { + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + agent.RotateLogIfSizeExceeded(config.logRotationSize) + } + } + }() + defer ticker.Stop() + } + reloader, err := cert.NewReloader(config.grpcCertDir, rLogger.WithName("cert-reloader")) if err != nil { return err @@ -239,6 +256,7 @@ func init() { fs.DurationVar(&config.connIdleTime, "max-idle-time", 30*time.Second, "The maximum amount of time a connection may be idle") fs.DurationVar(&config.connectionTimeout, "connection-timeout", 5*time.Second, "Dial timeout") fs.StringVar(&config.logRotationSchedule, "log-rotation-schedule", logRotationScheduleDefault, "Cron format schedule for MySQL log rotation") + fs.Int64Var(&config.logRotationSize, "log-rotation-size", 0, "Rotate MySQL log file when it exceeds the specified size in bytes.") fs.DurationVar(&config.readTimeout, "read-timeout", 30*time.Second, "I/O read timeout") fs.DurationVar(&config.maxDelayThreshold, "max-delay", time.Minute, "Acceptable max commit delay considering as ready; the zero value accepts any delay") fs.StringVar(&config.socketPath, "socket-path", socketPathDefault, "Path of mysqld socket file.") diff --git a/constants.go b/constants.go index 84dbc8e..8e5a35e 100644 --- a/constants.go +++ b/constants.go @@ -40,9 +40,6 @@ const ( // MySQLAdminPort is a port number for MySQL Admin MySQLAdminPort = 33062 - // MySQLErrorLogName is a filekey of error log for MySQL. - MySQLErrorLogName = "mysql.err" - // MySQLSlowLogName is a filekey of slow query log for MySQL. MySQLSlowLogName = "mysql.slow" ) diff --git a/docs/moco-agent.md b/docs/moco-agent.md index d0c17d5..f768b9a 100644 --- a/docs/moco-agent.md +++ b/docs/moco-agent.md @@ -11,6 +11,7 @@ Flags: --grpc-cert-dir string gRPC certificate directory (default "/grpc-cert") -h, --help help for moco-agent --log-rotation-schedule string Cron format schedule for MySQL log rotation (default "*/5 * * * *") + --log-rotation-size int Rotate MySQL log file when it exceeds the specified size in bytes. --logfile string Log filename --logformat string Log format [plain,logfmt,json] --loglevel string Log level [critical,error,warning,info,debug] diff --git a/server/rotate.go b/server/rotate.go index cd1ced0..465b5f3 100644 --- a/server/rotate.go +++ b/server/rotate.go @@ -10,31 +10,23 @@ import ( "github.com/cybozu-go/moco-agent/metrics" ) -// RotateLog rotates log files +// RotateLog rotates log file func (a *Agent) RotateLog() { ctx := context.Background() metrics.LogRotationCount.Inc() startTime := time.Now() - errFile := filepath.Join(a.logDir, mocoagent.MySQLErrorLogName) - err := os.Rename(errFile, errFile+".0") - if err != nil && !os.IsNotExist(err) { - a.logger.Error(err, "failed to rotate err log file") - metrics.LogRotationFailureCount.Inc() - return - } - slowFile := filepath.Join(a.logDir, mocoagent.MySQLSlowLogName) - err = os.Rename(slowFile, slowFile+".0") + err := os.Rename(slowFile, slowFile+".0") if err != nil && !os.IsNotExist(err) { a.logger.Error(err, "failed to rotate slow query log file") metrics.LogRotationFailureCount.Inc() return } - if _, err := a.db.ExecContext(ctx, "FLUSH LOCAL ERROR LOGS, SLOW LOGS"); err != nil { - a.logger.Error(err, "failed to exec FLUSH LOCAL LOGS") + if _, err := a.db.ExecContext(ctx, "FLUSH LOCAL SLOW LOGS"); err != nil { + a.logger.Error(err, "failed to exec FLUSH LOCAL SLOW LOGS") metrics.LogRotationFailureCount.Inc() return } @@ -42,3 +34,16 @@ func (a *Agent) RotateLog() { durationSeconds := time.Since(startTime).Seconds() metrics.LogRotationDurationSeconds.Observe(durationSeconds) } + +// RotateLogIfSizeExceeded rotates log file if it exceeds rotationSize +func (a *Agent) RotateLogIfSizeExceeded(rotationSize int64) { + file := filepath.Join(a.logDir, mocoagent.MySQLSlowLogName) + fileStat, err := os.Stat(file) + if err != nil { + a.logger.Error(err, "failed to get stat of log file") + return + } + if fileStat.Size() > rotationSize { + a.RotateLog() + } +} diff --git a/server/rotate_test.go b/server/rotate_test.go index a1a9695..bd23c01 100644 --- a/server/rotate_test.go +++ b/server/rotate_test.go @@ -1,6 +1,7 @@ package server import ( + "bytes" "os" "path/filepath" "time" @@ -12,7 +13,7 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" ) -var _ = Describe("log rotation", func() { +var _ = Describe("log rotation", Ordered, func() { It("should rotate logs", func() { By("starting MySQLd") StartMySQLD(replicaHost, replicaPort, replicaServerID) @@ -35,36 +36,83 @@ var _ = Describe("log rotation", func() { Expect(err).ShouldNot(HaveOccurred()) defer agent.CloseDB() - By("preparing log files for testing") - slowFile := filepath.Join(tmpDir, mocoagent.MySQLSlowLogName) - errFile := filepath.Join(tmpDir, mocoagent.MySQLErrorLogName) - logFiles := []string{slowFile, errFile} + By("preparing log file for testing") + logFile := filepath.Join(tmpDir, mocoagent.MySQLSlowLogName) - for _, file := range logFiles { - _, err := os.Create(file) - Expect(err).ShouldNot(HaveOccurred()) - } + _, err = os.Create(logFile) + Expect(err).ShouldNot(HaveOccurred()) agent.RotateLog() - for _, file := range logFiles { - _, err := os.Stat(file + ".0") - Expect(err).ShouldNot(HaveOccurred()) - } + _, err = os.Stat(logFile + ".0") + Expect(err).ShouldNot(HaveOccurred()) Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 1)) Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 0)) By("creating the same name directory") - for _, file := range logFiles { - err := os.Rename(file+".0", file) - Expect(err).ShouldNot(HaveOccurred()) - err = os.Mkdir(file+".0", 0777) - Expect(err).ShouldNot(HaveOccurred()) - } + err = os.Rename(logFile+".0", logFile) + Expect(err).ShouldNot(HaveOccurred()) + err = os.Mkdir(logFile+".0", 0777) + Expect(err).ShouldNot(HaveOccurred()) agent.RotateLog() Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 2)) Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 1)) }) + + It("should rotate logs by RotateLogIfSizeExceeded if size exceeds", func() { + By("starting MySQLd") + StartMySQLD(replicaHost, replicaPort, replicaServerID) + defer StopAndRemoveMySQLD(replicaHost) + + sockFile := filepath.Join(socketDir(replicaHost), "mysqld.sock") + tmpDir, err := os.MkdirTemp("", "moco-test-agent-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tmpDir) + + conf := MySQLAccessorConfig{ + Host: "localhost", + Port: replicaPort, + Password: agentUserPassword, + ConnMaxIdleTime: 30 * time.Minute, + ConnectionTimeout: 3 * time.Second, + ReadTimeout: 30 * time.Second, + } + agent, err := New(conf, testClusterName, sockFile, tmpDir, maxDelayThreshold, time.Second, testLogger) + Expect(err).ShouldNot(HaveOccurred()) + defer agent.CloseDB() + + By("preparing log file for testing") + logFile := filepath.Join(tmpDir, mocoagent.MySQLSlowLogName) + + logDataSize := 512 + data := bytes.Repeat([]byte("a"), logDataSize) + f, err := os.Create(logFile) + Expect(err).ShouldNot(HaveOccurred()) + f.Write(data) + + agent.RotateLogIfSizeExceeded(int64(logDataSize) + 1) + + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 2)) + Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 1)) + + agent.RotateLogIfSizeExceeded(int64(logDataSize) - 1) + + _, err = os.Stat(logFile + ".0") + Expect(err).ShouldNot(HaveOccurred()) + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 3)) + Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 1)) + + By("creating the same name directory") + err = os.Rename(logFile+".0", logFile) + Expect(err).ShouldNot(HaveOccurred()) + err = os.Mkdir(logFile+".0", 0777) + Expect(err).ShouldNot(HaveOccurred()) + + agent.RotateLogIfSizeExceeded(int64(logDataSize) - 1) + + Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 4)) + Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 2)) + }) })