Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add log-rotation-size option #105

Merged
merged 2 commits into from
Nov 27, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions cmd/moco-agent/cmd/root.go
Original file line number Diff line number Diff line change
@@ -52,6 +52,7 @@ var config struct {
connIdleTime time.Duration
connectionTimeout time.Duration
logRotationSchedule string
logRotationSize int64
readTimeout time.Duration
maxDelayThreshold time.Duration
socketPath string
@@ -142,6 +143,22 @@ var rootCmd = &cobra.Command{
}
}()

// Rotate if the file size exceeds logRotationSize
if config.logRotationSize > 0 {
ticker := time.NewTicker(time.Second)
go func() {
shunki-fujita marked this conversation as resolved.
Show resolved Hide resolved
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
agent.RotateLogIfSizeExceeded(config.logRotationSize)
}
}
}()
defer ticker.Stop()
}

reloader, err := cert.NewReloader(config.grpcCertDir, rLogger.WithName("cert-reloader"))
if err != nil {
return err
@@ -239,6 +256,7 @@ func init() {
fs.DurationVar(&config.connIdleTime, "max-idle-time", 30*time.Second, "The maximum amount of time a connection may be idle")
fs.DurationVar(&config.connectionTimeout, "connection-timeout", 5*time.Second, "Dial timeout")
fs.StringVar(&config.logRotationSchedule, "log-rotation-schedule", logRotationScheduleDefault, "Cron format schedule for MySQL log rotation")
fs.Int64Var(&config.logRotationSize, "log-rotation-size", 0, "Rotate MySQL log file when it exceeds the specified size in bytes.")
fs.DurationVar(&config.readTimeout, "read-timeout", 30*time.Second, "I/O read timeout")
fs.DurationVar(&config.maxDelayThreshold, "max-delay", time.Minute, "Acceptable max commit delay considering as ready; the zero value accepts any delay")
fs.StringVar(&config.socketPath, "socket-path", socketPathDefault, "Path of mysqld socket file.")
3 changes: 0 additions & 3 deletions constants.go
Original file line number Diff line number Diff line change
@@ -40,9 +40,6 @@ const (
// MySQLAdminPort is a port number for MySQL Admin
MySQLAdminPort = 33062

// MySQLErrorLogName is a filekey of error log for MySQL.
MySQLErrorLogName = "mysql.err"

// MySQLSlowLogName is a filekey of slow query log for MySQL.
MySQLSlowLogName = "mysql.slow"
)
1 change: 1 addition & 0 deletions docs/moco-agent.md
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@ Flags:
--grpc-cert-dir string gRPC certificate directory (default "/grpc-cert")
-h, --help help for moco-agent
--log-rotation-schedule string Cron format schedule for MySQL log rotation (default "*/5 * * * *")
--log-rotation-size int Rotate MySQL log file when it exceeds the specified size in bytes.
--logfile string Log filename
--logformat string Log format [plain,logfmt,json]
--loglevel string Log level [critical,error,warning,info,debug]
29 changes: 17 additions & 12 deletions server/rotate.go
Original file line number Diff line number Diff line change
@@ -10,35 +10,40 @@ import (
"github.com/cybozu-go/moco-agent/metrics"
)

// RotateLog rotates log files
// RotateLog rotates log file
func (a *Agent) RotateLog() {
ctx := context.Background()

metrics.LogRotationCount.Inc()
startTime := time.Now()

errFile := filepath.Join(a.logDir, mocoagent.MySQLErrorLogName)
err := os.Rename(errFile, errFile+".0")
if err != nil && !os.IsNotExist(err) {
a.logger.Error(err, "failed to rotate err log file")
metrics.LogRotationFailureCount.Inc()
return
}

slowFile := filepath.Join(a.logDir, mocoagent.MySQLSlowLogName)
err = os.Rename(slowFile, slowFile+".0")
err := os.Rename(slowFile, slowFile+".0")
if err != nil && !os.IsNotExist(err) {
a.logger.Error(err, "failed to rotate slow query log file")
metrics.LogRotationFailureCount.Inc()
return
}

if _, err := a.db.ExecContext(ctx, "FLUSH LOCAL ERROR LOGS, SLOW LOGS"); err != nil {
a.logger.Error(err, "failed to exec FLUSH LOCAL LOGS")
if _, err := a.db.ExecContext(ctx, "FLUSH LOCAL SLOW LOGS"); err != nil {
a.logger.Error(err, "failed to exec FLUSH LOCAL SLOW LOGS")
metrics.LogRotationFailureCount.Inc()
return
}

durationSeconds := time.Since(startTime).Seconds()
metrics.LogRotationDurationSeconds.Observe(durationSeconds)
}

// RotateLogIfSizeExceeded rotates log file if it exceeds rotationSize
func (a *Agent) RotateLogIfSizeExceeded(rotationSize int64) {
file := filepath.Join(a.logDir, mocoagent.MySQLSlowLogName)
fileStat, err := os.Stat(file)
if err != nil {
a.logger.Error(err, "failed to get stat of log file")
return
}
if fileStat.Size() > rotationSize {
a.RotateLog()
}
}
86 changes: 67 additions & 19 deletions server/rotate_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package server

import (
"bytes"
"os"
"path/filepath"
"time"
@@ -12,7 +13,7 @@ import (
"github.com/prometheus/client_golang/prometheus/testutil"
)

var _ = Describe("log rotation", func() {
var _ = Describe("log rotation", Ordered, func() {
It("should rotate logs", func() {
By("starting MySQLd")
StartMySQLD(replicaHost, replicaPort, replicaServerID)
@@ -35,36 +36,83 @@ var _ = Describe("log rotation", func() {
Expect(err).ShouldNot(HaveOccurred())
defer agent.CloseDB()

By("preparing log files for testing")
slowFile := filepath.Join(tmpDir, mocoagent.MySQLSlowLogName)
errFile := filepath.Join(tmpDir, mocoagent.MySQLErrorLogName)
logFiles := []string{slowFile, errFile}
By("preparing log file for testing")
logFile := filepath.Join(tmpDir, mocoagent.MySQLSlowLogName)

for _, file := range logFiles {
_, err := os.Create(file)
Expect(err).ShouldNot(HaveOccurred())
}
_, err = os.Create(logFile)
Expect(err).ShouldNot(HaveOccurred())

agent.RotateLog()

for _, file := range logFiles {
_, err := os.Stat(file + ".0")
Expect(err).ShouldNot(HaveOccurred())
}
_, err = os.Stat(logFile + ".0")
Expect(err).ShouldNot(HaveOccurred())
Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 1))
Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 0))

By("creating the same name directory")
for _, file := range logFiles {
err := os.Rename(file+".0", file)
Expect(err).ShouldNot(HaveOccurred())
err = os.Mkdir(file+".0", 0777)
Expect(err).ShouldNot(HaveOccurred())
}
err = os.Rename(logFile+".0", logFile)
Expect(err).ShouldNot(HaveOccurred())
err = os.Mkdir(logFile+".0", 0777)
Expect(err).ShouldNot(HaveOccurred())

agent.RotateLog()

Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 2))
Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 1))
})

It("should rotate logs by RotateLogIfSizeExceeded if size exceeds", func() {
By("starting MySQLd")
StartMySQLD(replicaHost, replicaPort, replicaServerID)
defer StopAndRemoveMySQLD(replicaHost)

sockFile := filepath.Join(socketDir(replicaHost), "mysqld.sock")
tmpDir, err := os.MkdirTemp("", "moco-test-agent-")
Expect(err).NotTo(HaveOccurred())
defer os.RemoveAll(tmpDir)

conf := MySQLAccessorConfig{
Host: "localhost",
Port: replicaPort,
Password: agentUserPassword,
ConnMaxIdleTime: 30 * time.Minute,
ConnectionTimeout: 3 * time.Second,
ReadTimeout: 30 * time.Second,
}
agent, err := New(conf, testClusterName, sockFile, tmpDir, maxDelayThreshold, time.Second, testLogger)
Expect(err).ShouldNot(HaveOccurred())
defer agent.CloseDB()

By("preparing log file for testing")
logFile := filepath.Join(tmpDir, mocoagent.MySQLSlowLogName)

logDataSize := 512
data := bytes.Repeat([]byte("a"), logDataSize)
f, err := os.Create(logFile)
Expect(err).ShouldNot(HaveOccurred())
f.Write(data)

agent.RotateLogIfSizeExceeded(int64(logDataSize) + 1)

Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 2))
Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 1))

agent.RotateLogIfSizeExceeded(int64(logDataSize) - 1)

_, err = os.Stat(logFile + ".0")
Expect(err).ShouldNot(HaveOccurred())
Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 3))
Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 1))

By("creating the same name directory")
err = os.Rename(logFile+".0", logFile)
Expect(err).ShouldNot(HaveOccurred())
err = os.Mkdir(logFile+".0", 0777)
Expect(err).ShouldNot(HaveOccurred())

agent.RotateLogIfSizeExceeded(int64(logDataSize) - 1)

Expect(testutil.ToFloat64(metrics.LogRotationCount)).To(BeNumerically("==", 4))
Expect(testutil.ToFloat64(metrics.LogRotationFailureCount)).To(BeNumerically("==", 2))
})
})