Merge pull request #247 from checkr/zz/add-eval-source

Extend DBDRIVER to load from json_file or json_http and add EvalOnlyMode

docs/api_docs/bundle.yaml

@@ -783,6 +783,23 @@ paths:
           description: generic error response
           schema:
             $ref: '#/definitions/error'
+  /export/eval_cache/json:
+    get:
+      tags:
+        - export
+      operationId: getExportEvalCacheJSON
+      description: Export JSON format of the eval cache dump
+      produces:
+        - application/json
+      responses:
+        '200':
+          description: OK
+          schema:
+            type: object
+        default:
+          description: generic error response
+          schema:
+            $ref: '#/definitions/error'
 definitions:
   flag:
     type: object

pkg/config/config.go

@@ -14,6 +14,12 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// EvalOnlyModeDBDrivers is a list of DBDrivers that we should only run in EvalOnlyMode.
+var EvalOnlyModeDBDrivers = map[string]struct{}{
+	"json_file": {},
+	"json_http": {},
+}
+
 // Global is the global dependency we can use, such as the new relic app instance
 var Global = struct {
 	NewrelicApp  newrelic.Application
@@ -24,13 +30,20 @@ var Global = struct {
 func init() {
 	env.Parse(&Config)
 
+	setupEvalOnlyMode()
 	setupSentry()
 	setupLogrus()
 	setupStatsd()
 	setupNewrelic()
 	setupPrometheus()
 }
 
+func setupEvalOnlyMode() {
+	if _, ok := EvalOnlyModeDBDrivers[Config.DBDriver]; ok {
+		Config.EvalOnlyMode = true
+	}
+}
+
 func setupLogrus() {
 	l, err := logrus.ParseLevel(Config.LogrusLevel)
 	if err != nil {

pkg/config/env.go

@@ -19,30 +19,44 @@ var Config = struct {
 	// MiddlewareGzipEnabled - to enable gzip middleware
 	MiddlewareGzipEnabled bool `env:"FLAGR_MIDDLEWARE_GZIP_ENABLED" envDefault:"true"`
 
-	// EvalLoggingEnabled - to enable the logging for eval results
-	EvalLoggingEnabled bool `env:"FLAGR_EVAL_LOGGING_ENABLED" envDefault:"true"`
-
 	// RateLimiterPerFlagPerSecondConsoleLogging - to rate limit the logging rate
 	// per flag per second
 	RateLimiterPerFlagPerSecondConsoleLogging int `env:"FLAGR_RATELIMITER_PERFLAG_PERSECOND_CONSOLE_LOGGING" envDefault:"100"`
 
-	// EvalCacheRefreshTimeout - timeout of getting the flags data from DB into the in-memory evaluation cache
-	EvalCacheRefreshTimeout time.Duration `env:"FLAGR_EVALCACHE_REFRESHTIMEOUT" envDefault:"59s"`
-	// EvalCacheRefreshInterval - time interval of getting the flags data from DB into the in-memory evaluation cache
-	EvalCacheRefreshInterval time.Duration `env:"FLAGR_EVALCACHE_REFRESHINTERVAL" envDefault:"3s"`
 	// EvalEnableDebug - controls if we want to return evaluation debugging information back to the api requests
 	// Note that this is a global switch:
 	//     if it's disabled, no evaluation debug info will be returned.
 	//     if it's enabled, it respects evaluation request's enableDebug field
 	EvalDebugEnabled bool `env:"FLAGR_EVAL_DEBUG_ENABLED" envDefault:"true"`
+	// EvalLoggingEnabled - to enable the logging for eval results
+	EvalLoggingEnabled bool `env:"FLAGR_EVAL_LOGGING_ENABLED" envDefault:"true"`
+	// EvalCacheRefreshTimeout - timeout of getting the flags data from DB into the in-memory evaluation cache
+	EvalCacheRefreshTimeout time.Duration `env:"FLAGR_EVALCACHE_REFRESHTIMEOUT" envDefault:"59s"`
+	// EvalCacheRefreshInterval - time interval of getting the flags data from DB into the in-memory evaluation cache
+	EvalCacheRefreshInterval time.Duration `env:"FLAGR_EVALCACHE_REFRESHINTERVAL" envDefault:"3s"`
+	// EvalOnlyMode - will only expose the evaluation related endpoints.
+	// This field will be derived from DBDriver
+	EvalOnlyMode bool `env:"FLAGR_EVAL_ONLY_MODE" envDefault:"false"`
+
+	/**
+	DBDriver and DBConnectionStr define how we can write and read flags data.
+	For databases, flagr supports sqlite3, mysql and postgres.
+	For read-only evaluation, flagr supports file and http.
 
-	// DBDriver - Flagr supports sqlite3, mysql, postgres
-	DBDriver string `env:"FLAGR_DB_DBDRIVER" envDefault:"sqlite3"`
-	// DBConnectionStr - examples
-	// sqlite3:  "/tmp/file.db"
-	// sqlite3:  ":memory:"
-	// mysql:    "root:@tcp(127.0.0.1:18100)/flagr?parseTime=true"
-	// postgres: "host=myhost user=root dbname=flagr password=mypassword"
+	Examples:
+
+	FLAGR_DB_DBDRIVER     FLAGR_DB_DBCONNECTIONSTR
+	=================     ===============================================================
+	"sqlite3"             "/tmp/file.db"
+	"sqlite3"             ":memory:"
+	"mysql"               "root:@tcp(127.0.0.1:18100)/flagr?parseTime=true"
+	"postgres"            "host=myhost user=root dbname=flagr password=mypassword"
+
+	"json_file"           "/tmp/flags.json"                    # (it automatically sets EvalOnlyMode=true)
+	"json_http"           "https://example.com/flags.json"     # (it automatically sets EvalOnlyMode=true)
+
+	*/
+	DBDriver        string `env:"FLAGR_DB_DBDRIVER" envDefault:"sqlite3"`
 	DBConnectionStr string `env:"FLAGR_DB_DBCONNECTIONSTR" envDefault:"flagr.sqlite"`
 	// DBConnectionDebug controls whether to show the database connection debugging logs
 	// warning: it may log the credentials to the stdout

pkg/entity/flag.go

@@ -18,7 +18,7 @@ type Flag struct {
 	Enabled     bool
 	Segments    []Segment
 	Variants    []Variant
-	SnapshotID  uint   `json:"-"`
+	SnapshotID  uint
 	Notes       string `sql:"type:text"`
 
 	DataRecordsEnabled bool

pkg/handler/eval_cache.go

@@ -7,18 +7,23 @@ import (
 	"github.com/checkr/flagr/pkg/config"
 	"github.com/checkr/flagr/pkg/entity"
 	"github.com/checkr/flagr/pkg/util"
+
 	"github.com/sirupsen/logrus"
+	"github.com/zhouzhuojie/withtimeout"
 )
 
 var (
 	singletonEvalCache     *EvalCache
 	singletonEvalCacheOnce sync.Once
 )
 
+type mapCache map[string]*entity.Flag
+
 // EvalCache is the in-memory cache just for evaluation
 type EvalCache struct {
-	mapCache     map[string]*entity.Flag
 	mapCacheLock sync.RWMutex
+	idCache      mapCache
+	keyCache     mapCache
 
 	refreshTimeout  time.Duration
 	refreshInterval time.Duration
@@ -28,7 +33,8 @@ type EvalCache struct {
 var GetEvalCache = func() *EvalCache {
 	singletonEvalCacheOnce.Do(func() {
 		ec := &EvalCache{
-			mapCache:        make(map[string]*entity.Flag),
+			idCache:         make(map[string]*entity.Flag),
+			keyCache:        make(map[string]*entity.Flag),
 			refreshTimeout:  config.Config.EvalCacheRefreshTimeout,
 			refreshInterval: config.Config.EvalCacheRefreshInterval,
 		}
@@ -56,48 +62,34 @@ func (ec *EvalCache) Start() {
 // GetByFlagKeyOrID gets the flag by Key or ID
 func (ec *EvalCache) GetByFlagKeyOrID(keyOrID interface{}) *entity.Flag {
 	ec.mapCacheLock.RLock()
-	f := ec.mapCache[util.SafeString(keyOrID)]
-	ec.mapCacheLock.RUnlock()
-	return f
-}
+	defer ec.mapCacheLock.RUnlock()
 
-var fetchAllFlags = func() ([]entity.Flag, error) {
-	// Use eager loading to avoid N+1 problem
-	// doc: http://jinzhu.me/gorm/crud.html#preloading-eager-loading
-	fs := []entity.Flag{}
-	err := entity.PreloadSegmentsVariants(getDB()).Find(&fs).Error
-	return fs, err
+	s := util.SafeString(keyOrID)
+	f, ok := ec.idCache[s]
+	if !ok {
+		f = ec.keyCache[s]
+	}
+	return f
 }
 
 func (ec *EvalCache) reloadMapCache() error {
 	if config.Config.NewRelicEnabled {
 		defer config.Global.NewrelicApp.StartTransaction("eval_cache_reload", nil, nil).End()
 	}
 
-	fs, err := fetchAllFlags()
-	if err != nil {
-		return err
-	}
-	m := make(map[string]*entity.Flag)
-	for i := range fs {
-		ptr := &fs[i]
-		if ptr.ID != 0 {
-			m[util.SafeString(ptr.ID)] = ptr
-		}
-		if ptr.Key != "" {
-			m[ptr.Key] = ptr
-		}
-	}
-
-	for _, f := range m {
-		err := f.PrepareEvaluation()
+	_, _, err := withtimeout.Do(ec.refreshTimeout, func() (interface{}, error) {
+		idCache, keyCache, err := ec.fetchAllFlags()
 		if err != nil {
-			return err
+			return nil, err
 		}
-	}
 
-	ec.mapCacheLock.Lock()
-	ec.mapCache = m
-	ec.mapCacheLock.Unlock()
-	return nil
+		ec.mapCacheLock.Lock()
+		defer ec.mapCacheLock.Unlock()
+
+		ec.idCache = idCache
+		ec.keyCache = keyCache
+		return nil, err
+	})
+
+	return err
 }