support eviction

config/config.go

@@ -47,6 +47,15 @@ type ServerProperties struct {
 
 	// config file path
 	CfPath string `cfg:"cf,omitempty"`
+
+	//eviction
+	MaxmemoryPolicy  string `cfg:"maxmemory-policy"`
+	LfuDecayTime     int32  `cfg:"lfu-decay-time"`
+	LfuLogFactor     int32  `cfg:"lfu-log-factor"`
+	LfuInitVal       uint8  `cfg:"lfu-init-val"`
+	MaxmemorySamples int    `cfg:"maxmemory-samples"`
+	// MB
+	Maxmemory uint64 `cfg:"maxmemory"`
 }
 
 type ServerInfo struct {

database/database.go

@@ -2,6 +2,11 @@
 package database
 
 import (
+	"github.com/hdt3213/godis/config"
+	"github.com/hdt3213/godis/database/eviction"
+	"github.com/hdt3213/godis/lib/mem"
+	"github.com/hdt3213/godis/lib/utils"
+	"runtime"
 	"strings"
 	"time"
 
@@ -27,6 +32,9 @@ type DB struct {
 	ttlMap *dict.ConcurrentDict
 	// key -> version(uint32)
 	versionMap *dict.ConcurrentDict
+	// key -> eviction(uint32)
+	evictionMap    *dict.ConcurrentDict
+	evictionPolicy eviction.MaxmemoryPolicy
 
 	// addaof is used to add command to aof
 	addAof func(CmdLine)
@@ -115,7 +123,12 @@ func (db *DB) execNormalCommand(cmdLine [][]byte) redis.Reply {
 	write, read := prepare(cmdLine[1:])
 	db.addVersion(write...)
 	db.RWLocks(write, read)
-	defer db.RWUnLocks(write, read)
+	db.initEvictionMark(write)
+	db.updateEvictionMark(read)
+	defer func() {
+		db.RWUnLocks(write, read)
+		db.Eviction()
+	}()
 	fun := cmd.executor
 	return fun(db, cmdLine[1:])
 }
@@ -294,3 +307,88 @@ func (db *DB) ForEach(cb func(key string, data *database.DataEntity, expiration
 		return cb(key, entity, expiration)
 	})
 }
+
+//eviction
+func (db *DB) Eviction() {
+	// is not out of max-memory,no need to lock
+	if db.evictionPolicy == nil || !mem.GetMaxMemoryState(nil) {
+		return
+	}
+	mem.Lock.Lock()
+	defer mem.Lock.Unlock()
+	var memFreed uint64 = 0
+	var memToFree uint64
+	mem.GetMaxMemoryState(memToFree)
+	for memFreed < memToFree {
+		var keys []string
+		if db.evictionPolicy.IsAllKeys() {
+			keys = db.data.RandomDistinctKeys(config.Properties.MaxmemorySamples)
+		} else {
+			keys = db.ttlMap.RandomDistinctKeys(config.Properties.MaxmemorySamples)
+		}
+
+		marks := make([]eviction.KeyMark, config.Properties.MaxmemorySamples)
+		for i, key := range keys {
+			mark, _ := db.evictionMap.Get(key)
+			marks[i] = eviction.KeyMark{
+				Key:  key,
+				Mark: mark.(int32),
+			}
+		}
+		key := db.evictionPolicy.Eviction(marks)
+		delta := mem.UsedMemory()
+		db.Remove(key)
+		runtime.GC()
+		delta -= mem.UsedMemory()
+		memFreed += delta
+		db.addAof(utils.ToCmdLine2("DEL", key))
+	}
+
+}
+
+//MakeEviction make a new mark about a key
+func (db *DB) initEvictionMark(keys []string) {
+	if db.evictionPolicy == nil {
+		return
+	}
+	mark := db.evictionPolicy.MakeMark()
+	for _, key := range keys {
+		db.evictionMap.Put(key, mark)
+	}
+}
+
+//UpdateMark update mark about eviction
+func (db *DB) updateEvictionMark(keys []string) {
+	if db.evictionPolicy == nil {
+		return
+	}
+	for _, key := range keys {
+		mark, exists := db.evictionMap.Get(key)
+		if !exists {
+			continue
+		}
+		l := mark.(int32)
+		updateMark := db.evictionPolicy.UpdateMark(l)
+		db.evictionMap.Put(key, updateMark)
+	}
+}
+
+func makeEvictionPolicy() eviction.MaxmemoryPolicy {
+	policy := config.Properties.MaxmemoryPolicy
+	if policy == "volatile-lru" {
+		return &eviction.LRUPolicy{
+			AllKeys: false,
+		}
+	} else if policy == "volatile-lfu" {
+		return &eviction.LRUPolicy{}
+	} else if policy == "allkeys-lru" {
+		return &eviction.LRUPolicy{
+			AllKeys: true,
+		}
+	} else if policy == "allkeys-lfu" {
+		return &eviction.LRUPolicy{
+			AllKeys: true,
+		}
+	}
+	return nil
+}

database/eviction/interface.go

@@ -0,0 +1,13 @@
+package eviction
+
+type MaxmemoryPolicy interface {
+	MakeMark() int32
+	UpdateMark(int32) int32
+	Eviction([]KeyMark) string
+	IsAllKeys() bool
+}
+
+type KeyMark struct {
+	Key  string
+	Mark int32
+}

database/eviction/lfu.go

@@ -0,0 +1,104 @@
+package eviction
+
+import (
+	"github.com/hdt3213/godis/config"
+	"math/rand"
+	"time"
+)
+
+type LFUPolicy struct {
+	AllKeys bool
+}
+
+func (policy *LFUPolicy) IsAllKeys() bool {
+	return policy.AllKeys
+}
+
+//MakeMark create a new mark
+func (policy *LFUPolicy) MakeMark() (lfu int32) {
+	lfu = lfuGetTimeInMinutes()<<8 | int32(config.Properties.LfuInitVal)
+	return lfu
+}
+
+//UpdateMark when read a key ,update the key's mark
+func (policy *LFUPolicy) UpdateMark(lfu int32) int32 {
+	counter := GetLFUCounter(lfu)
+	decr := lfuDecrAndReturn(lfu)
+	incr := LFULogIncr(counter - uint8(decr))
+	return lfuGetTimeInMinutes()<<8 | int32(incr)
+}
+
+//Eviction choose a key for eviction
+func (policy *LFUPolicy) Eviction(marks []KeyMark) string {
+	key := marks[0].Key
+	min := GetLFUCounter(marks[0].Mark)
+	for i := 1; i < len(marks); i++ {
+		counter := GetLFUCounter(marks[i].Mark)
+		if min > counter {
+			key = marks[i].Key
+			min = counter
+		}
+	}
+	return key
+}
+
+func GetLFUCounter(lfu int32) uint8 {
+	return uint8(lfu & 0xff)
+}
+
+// LFULogIncr counter increase
+func LFULogIncr(counter uint8) uint8 {
+	if counter == 255 {
+		return 255
+	}
+	r := rand.Float64()
+	baseval := float64(counter - config.Properties.LfuInitVal)
+
+	if baseval < 0 {
+		baseval = 0
+	}
+
+	p := 1.0 / (baseval*float64(config.Properties.LfuLogFactor) + 1)
+
+	if r < p {
+		counter++
+	}
+	return counter
+}
+
+//LFUDecrAndReturn counter decr
+func lfuDecrAndReturn(lfu int32) int32 {
+	ldt := lfu >> 8
+
+	counter := lfu & 0xff
+
+	var numPeriods int32
+	if config.Properties.LfuDecayTime > 0 {
+		numPeriods = lfuTimeElapsed(ldt) / config.Properties.LfuDecayTime
+	} else {
+		numPeriods = 0
+	}
+
+	if numPeriods > 0 {
+		if numPeriods > counter {
+			counter = 0
+		} else {
+			counter = counter - numPeriods
+		}
+	}
+	return counter
+}
+
+// LFUTimeElapsed Obtaining the time difference from the last time
+func lfuTimeElapsed(ldt int32) int32 {
+	now := lfuGetTimeInMinutes()
+	if now >= ldt {
+		return now - ldt
+	}
+	return 65535 - ldt + now
+}
+
+// LFUGetTimeInMinutes Accurate to the minute
+func lfuGetTimeInMinutes() int32 {
+	return int32(time.Now().Unix()/60) & 65535
+}

database/eviction/lru.go

@@ -0,0 +1,36 @@
+package eviction
+
+import (
+	"time"
+)
+
+type LRUPolicy struct {
+	AllKeys bool
+}
+
+func (policy *LRUPolicy) IsAllKeys() bool {
+	return policy.AllKeys
+}
+func (policy *LRUPolicy) MakeMark() (lru int32) {
+	return LRUGetTimeInSecond()
+}
+
+func (policy *LRUPolicy) UpdateMark(lru int32) int32 {
+	return LRUGetTimeInSecond()
+}
+
+func (policy *LRUPolicy) Eviction(marks []KeyMark) string {
+	key := marks[0].Key
+	min := marks[0].Mark
+	for i := 1; i < len(marks); i++ {
+		if min > marks[i].Mark {
+			key = marks[i].Key
+			min = marks[i].Mark
+		}
+	}
+	return key
+}
+
+func LRUGetTimeInSecond() int32 {
+	return int32(time.Now().Unix() & 0xffffffff)
+}

database/lfu_test.go

@@ -0,0 +1,53 @@
+package database
+
+import (
+	"fmt"
+	"github.com/hdt3213/godis/config"
+	"github.com/hdt3213/godis/database/eviction"
+	"github.com/hdt3213/godis/lib/mem"
+	"github.com/hdt3213/godis/lib/utils"
+	"testing"
+)
+
+func TestLFUEvictionKey(t *testing.T) {
+	setLFUConfig()
+	testDB.Flush()
+	marks := make([]eviction.KeyMark, 10)
+	for i := 0; i < 10; i++ {
+
+		marks[i] = eviction.KeyMark{
+			Mark: int32(i),
+			Key:  fmt.Sprint(i),
+		}
+	}
+	s := testDB.evictionPolicy.Eviction(marks)
+	if s != "0" {
+		t.Errorf("eviction key is wrong")
+	}
+	config.Properties = &config.ServerProperties{}
+}
+
+func TestLFU(t *testing.T) {
+	testDB.Flush()
+	setLFUConfig()
+	for i := 0; i < 10000; i++ {
+		key := utils.RandString(10)
+		value := utils.RandString(10)
+		testDB.Exec(nil, utils.ToCmdLine("SET", key, value))
+		if mem.GetMaxMemoryState(nil) {
+			t.Errorf("memory out of config")
+		}
+	}
+	config.Properties = &config.ServerProperties{}
+}
+
+func setLFUConfig() {
+	config.Properties = &config.ServerProperties{
+		//go test in the window used 2800 MB
+		Maxmemory:        3000,
+		MaxmemoryPolicy:  "allkeys-lfu",
+		LfuLogFactor:     5,
+		MaxmemorySamples: 5,
+	}
+	testDB.evictionPolicy = makeEvictionPolicy()
+}