diff --git a/config/config.go b/config/config.go index b6dcc3ed..74c62a98 100644 --- a/config/config.go +++ b/config/config.go @@ -47,6 +47,15 @@ type ServerProperties struct { // config file path CfPath string `cfg:"cf,omitempty"` + + //eviction + MaxmemoryPolicy string `cfg:"maxmemory-policy"` + LfuDecayTime int32 `cfg:"lfu-decay-time"` + LfuLogFactor int32 `cfg:"lfu-log-factor"` + LfuInitVal uint8 `cfg:"lfu-init-val"` + MaxmemorySamples int `cfg:"maxmemory-samples"` + // MB + Maxmemory uint64 `cfg:"maxmemory"` } type ServerInfo struct { diff --git a/database/database.go b/database/database.go index 859327a1..f51b7d5d 100644 --- a/database/database.go +++ b/database/database.go @@ -2,6 +2,11 @@ package database import ( + "github.com/hdt3213/godis/config" + "github.com/hdt3213/godis/database/eviction" + "github.com/hdt3213/godis/lib/mem" + "github.com/hdt3213/godis/lib/utils" + "runtime" "strings" "time" @@ -27,6 +32,9 @@ type DB struct { ttlMap *dict.ConcurrentDict // key -> version(uint32) versionMap *dict.ConcurrentDict + // key -> eviction(uint32) + evictionMap *dict.ConcurrentDict + evictionPolicy eviction.MaxmemoryPolicy // addaof is used to add command to aof addAof func(CmdLine) @@ -115,7 +123,12 @@ func (db *DB) execNormalCommand(cmdLine [][]byte) redis.Reply { write, read := prepare(cmdLine[1:]) db.addVersion(write...) db.RWLocks(write, read) - defer db.RWUnLocks(write, read) + db.initEvictionMark(write) + db.updateEvictionMark(read) + defer func() { + db.RWUnLocks(write, read) + db.Eviction() + }() fun := cmd.executor return fun(db, cmdLine[1:]) } @@ -294,3 +307,88 @@ func (db *DB) ForEach(cb func(key string, data *database.DataEntity, expiration return cb(key, entity, expiration) }) } + +//eviction +func (db *DB) Eviction() { + // is not out of max-memory,no need to lock + if db.evictionPolicy == nil || !mem.GetMaxMemoryState(nil) { + return + } + mem.Lock.Lock() + defer mem.Lock.Unlock() + var memFreed uint64 = 0 + var memToFree uint64 + mem.GetMaxMemoryState(memToFree) + for memFreed < memToFree { + var keys []string + if db.evictionPolicy.IsAllKeys() { + keys = db.data.RandomDistinctKeys(config.Properties.MaxmemorySamples) + } else { + keys = db.ttlMap.RandomDistinctKeys(config.Properties.MaxmemorySamples) + } + + marks := make([]eviction.KeyMark, config.Properties.MaxmemorySamples) + for i, key := range keys { + mark, _ := db.evictionMap.Get(key) + marks[i] = eviction.KeyMark{ + Key: key, + Mark: mark.(int32), + } + } + key := db.evictionPolicy.Eviction(marks) + delta := mem.UsedMemory() + db.Remove(key) + runtime.GC() + delta -= mem.UsedMemory() + memFreed += delta + db.addAof(utils.ToCmdLine2("DEL", key)) + } + +} + +//MakeEviction make a new mark about a key +func (db *DB) initEvictionMark(keys []string) { + if db.evictionPolicy == nil { + return + } + mark := db.evictionPolicy.MakeMark() + for _, key := range keys { + db.evictionMap.Put(key, mark) + } +} + +//UpdateMark update mark about eviction +func (db *DB) updateEvictionMark(keys []string) { + if db.evictionPolicy == nil { + return + } + for _, key := range keys { + mark, exists := db.evictionMap.Get(key) + if !exists { + continue + } + l := mark.(int32) + updateMark := db.evictionPolicy.UpdateMark(l) + db.evictionMap.Put(key, updateMark) + } +} + +func makeEvictionPolicy() eviction.MaxmemoryPolicy { + policy := config.Properties.MaxmemoryPolicy + if policy == "volatile-lru" { + return &eviction.LRUPolicy{ + AllKeys: false, + } + } else if policy == "volatile-lfu" { + return &eviction.LRUPolicy{} + } else if policy == "allkeys-lru" { + return &eviction.LRUPolicy{ + AllKeys: true, + } + } else if policy == "allkeys-lfu" { + return &eviction.LRUPolicy{ + AllKeys: true, + } + } + return nil +} diff --git a/database/eviction/interface.go b/database/eviction/interface.go new file mode 100644 index 00000000..9d6e9010 --- /dev/null +++ b/database/eviction/interface.go @@ -0,0 +1,13 @@ +package eviction + +type MaxmemoryPolicy interface { + MakeMark() int32 + UpdateMark(int32) int32 + Eviction([]KeyMark) string + IsAllKeys() bool +} + +type KeyMark struct { + Key string + Mark int32 +} diff --git a/database/eviction/lfu.go b/database/eviction/lfu.go new file mode 100644 index 00000000..b62f4f96 --- /dev/null +++ b/database/eviction/lfu.go @@ -0,0 +1,104 @@ +package eviction + +import ( + "github.com/hdt3213/godis/config" + "math/rand" + "time" +) + +type LFUPolicy struct { + AllKeys bool +} + +func (policy *LFUPolicy) IsAllKeys() bool { + return policy.AllKeys +} + +//MakeMark create a new mark +func (policy *LFUPolicy) MakeMark() (lfu int32) { + lfu = lfuGetTimeInMinutes()<<8 | int32(config.Properties.LfuInitVal) + return lfu +} + +//UpdateMark when read a key ,update the key's mark +func (policy *LFUPolicy) UpdateMark(lfu int32) int32 { + counter := GetLFUCounter(lfu) + decr := lfuDecrAndReturn(lfu) + incr := LFULogIncr(counter - uint8(decr)) + return lfuGetTimeInMinutes()<<8 | int32(incr) +} + +//Eviction choose a key for eviction +func (policy *LFUPolicy) Eviction(marks []KeyMark) string { + key := marks[0].Key + min := GetLFUCounter(marks[0].Mark) + for i := 1; i < len(marks); i++ { + counter := GetLFUCounter(marks[i].Mark) + if min > counter { + key = marks[i].Key + min = counter + } + } + return key +} + +func GetLFUCounter(lfu int32) uint8 { + return uint8(lfu & 0xff) +} + +// LFULogIncr counter increase +func LFULogIncr(counter uint8) uint8 { + if counter == 255 { + return 255 + } + r := rand.Float64() + baseval := float64(counter - config.Properties.LfuInitVal) + + if baseval < 0 { + baseval = 0 + } + + p := 1.0 / (baseval*float64(config.Properties.LfuLogFactor) + 1) + + if r < p { + counter++ + } + return counter +} + +//LFUDecrAndReturn counter decr +func lfuDecrAndReturn(lfu int32) int32 { + ldt := lfu >> 8 + + counter := lfu & 0xff + + var numPeriods int32 + if config.Properties.LfuDecayTime > 0 { + numPeriods = lfuTimeElapsed(ldt) / config.Properties.LfuDecayTime + } else { + numPeriods = 0 + } + + if numPeriods > 0 { + if numPeriods > counter { + counter = 0 + } else { + counter = counter - numPeriods + } + } + return counter +} + +// LFUTimeElapsed Obtaining the time difference from the last time +func lfuTimeElapsed(ldt int32) int32 { + now := lfuGetTimeInMinutes() + if now >= ldt { + return now - ldt + } + return 65535 - ldt + now +} + +// LFUGetTimeInMinutes Accurate to the minute +func lfuGetTimeInMinutes() int32 { + return int32(time.Now().Unix()/60) & 65535 +} diff --git a/database/eviction/lru.go b/database/eviction/lru.go new file mode 100644 index 00000000..8dca2149 --- /dev/null +++ b/database/eviction/lru.go @@ -0,0 +1,36 @@ +package eviction + +import ( + "time" +) + +type LRUPolicy struct { + AllKeys bool +} + +func (policy *LRUPolicy) IsAllKeys() bool { + return policy.AllKeys +} +func (policy *LRUPolicy) MakeMark() (lru int32) { + return LRUGetTimeInSecond() +} + +func (policy *LRUPolicy) UpdateMark(lru int32) int32 { + return LRUGetTimeInSecond() +} + +func (policy *LRUPolicy) Eviction(marks []KeyMark) string { + key := marks[0].Key + min := marks[0].Mark + for i := 1; i < len(marks); i++ { + if min > marks[i].Mark { + key = marks[i].Key + min = marks[i].Mark + } + } + return key +} + +func LRUGetTimeInSecond() int32 { + return int32(time.Now().Unix() & 0xffffffff) +} diff --git a/database/lfu_test.go b/database/lfu_test.go new file mode 100644 index 00000000..0c629246 --- /dev/null +++ b/database/lfu_test.go @@ -0,0 +1,53 @@ +package database + +import ( + "fmt" + "github.com/hdt3213/godis/config" + "github.com/hdt3213/godis/database/eviction" + "github.com/hdt3213/godis/lib/mem" + "github.com/hdt3213/godis/lib/utils" + "testing" +) + +func TestLFUEvictionKey(t *testing.T) { + setLFUConfig() + testDB.Flush() + marks := make([]eviction.KeyMark, 10) + for i := 0; i < 10; i++ { + + marks[i] = eviction.KeyMark{ + Mark: int32(i), + Key: fmt.Sprint(i), + } + } + s := testDB.evictionPolicy.Eviction(marks) + if s != "0" { + t.Errorf("eviction key is wrong") + } + config.Properties = &config.ServerProperties{} +} + +func TestLFU(t *testing.T) { + testDB.Flush() + setLFUConfig() + for i := 0; i < 10000; i++ { + key := utils.RandString(10) + value := utils.RandString(10) + testDB.Exec(nil, utils.ToCmdLine("SET", key, value)) + if mem.GetMaxMemoryState(nil) { + t.Errorf("memory out of config") + } + } + config.Properties = &config.ServerProperties{} +} + +func setLFUConfig() { + config.Properties = &config.ServerProperties{ + //go test in the window used 2800 MB + Maxmemory: 3000, + MaxmemoryPolicy: "allkeys-lfu", + LfuLogFactor: 5, + MaxmemorySamples: 5, + } + testDB.evictionPolicy = makeEvictionPolicy() +} diff --git a/database/lru_test.go b/database/lru_test.go new file mode 100644 index 00000000..a0d672f7 --- /dev/null +++ b/database/lru_test.go @@ -0,0 +1,53 @@ +package database + +import ( + "fmt" + "github.com/hdt3213/godis/config" + "github.com/hdt3213/godis/database/eviction" + "github.com/hdt3213/godis/lib/mem" + "github.com/hdt3213/godis/lib/utils" + "testing" +) + +func TestLRUEvictionKey(t *testing.T) { + testDB.Flush() + setLRUConfig() + marks := make([]eviction.KeyMark, 10) + for i := 0; i < 10; i++ { + + marks[i] = eviction.KeyMark{ + Mark: int32(i), + Key: fmt.Sprint(i), + } + } + s := testDB.evictionPolicy.Eviction(marks) + if s != "0" { + t.Errorf("eviction key is wrong") + } + config.Properties = &config.ServerProperties{} +} + +func TestLRU(t *testing.T) { + testDB.Flush() + setLRUConfig() + for i := 0; i < 1000; i++ { + key := utils.RandString(10) + value := utils.RandString(10) + testDB.Exec(nil, utils.ToCmdLine("SET", key, value)) + + if mem.GetMaxMemoryState(nil) { + t.Errorf("memory out of config") + } + } + config.Properties = &config.ServerProperties{} +} + +func setLRUConfig() { + config.Properties = &config.ServerProperties{ + Maxmemory: 3000, + MaxmemoryPolicy: "allkeys-lru", + LfuLogFactor: 5, + MaxmemorySamples: 5, + } + testDB.evictionPolicy = makeEvictionPolicy() +} diff --git a/database/util_test.go b/database/util_test.go index c992b3f8..2141c06d 100644 --- a/database/util_test.go +++ b/database/util_test.go @@ -6,9 +6,11 @@ import ( func makeTestDB() *DB { return &DB{ - data: dict.MakeConcurrent(dataDictSize), - versionMap: dict.MakeConcurrent(dataDictSize), - ttlMap: dict.MakeConcurrent(ttlDictSize), - addAof: func(line CmdLine) {}, + data: dict.MakeConcurrent(dataDictSize), + versionMap: dict.MakeConcurrent(dataDictSize), + ttlMap: dict.MakeConcurrent(ttlDictSize), + evictionMap: dict.MakeConcurrent(dataDictSize), + evictionPolicy: makeEvictionPolicy(), + addAof: func(line CmdLine) {}, } } diff --git a/lib/mem/mem.go b/lib/mem/mem.go new file mode 100644 index 00000000..e2885429 --- /dev/null +++ b/lib/mem/mem.go @@ -0,0 +1,29 @@ +package mem + +import ( + "github.com/hdt3213/godis/config" + "runtime" + "sync" +) + +var Lock sync.Mutex + +func UsedMemory() uint64 { + var m runtime.MemStats + runtime.ReadMemStats(&m) + return m.Alloc / 1024 / 1024 +} + +func GetMaxMemoryState(toFree interface{}) bool { + if config.Properties.Maxmemory == 0 { + return false + } + if UsedMemory() > config.Properties.Maxmemory { + toFree, ok := toFree.(*uint64) + if ok && toFree != nil { + *toFree = UsedMemory() - config.Properties.Maxmemory + } + return true + } + return false +}