Merge pull request #73 from github/throttle-critical

Throttling & critical load

Author: Shlomi Noach

build.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 #
-RELEASE_VERSION="0.9.4"
+RELEASE_VERSION="0.9.5"
 
 buildpath=/tmp/gh-ost
 target=gh-ost

go/base/context.go

@@ -7,7 +7,6 @@ package base
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -65,9 +64,10 @@ type MigrationContext struct {
 	ThrottleControlReplicaKeys          *mysql.InstanceKeyMap
 	ThrottleFlagFile                    string
 	ThrottleAdditionalFlagFile          string
+	ThrottleQuery                       string
 	ThrottleCommandedByUser             int64
-	maxLoad                             map[string]int64
-	maxLoadMutex                        *sync.Mutex
+	maxLoad                             LoadMap
+	criticalLoad                        LoadMap
 	PostponeCutOverFlagFile             string
 	SwapTablesTimeoutSeconds            int64
 	PanicFlagFile                       string
@@ -148,8 +148,8 @@ func newMigrationContext() *MigrationContext {
 		ApplierConnectionConfig:             mysql.NewConnectionConfig(),
 		MaxLagMillisecondsThrottleThreshold: 1000,
 		SwapTablesTimeoutSeconds:            3,
-		maxLoad:                             make(map[string]int64),
-		maxLoadMutex:                        &sync.Mutex{},
+		maxLoad:                             NewLoadMap(),
+		criticalLoad:                        NewLoadMap(),
 		throttleMutex:                       &sync.Mutex{},
 		ThrottleControlReplicaKeys:          mysql.NewInstanceKeyMap(),
 		configMutex:                         &sync.Mutex{},
@@ -278,46 +278,64 @@ func (this *MigrationContext) IsThrottled() (bool, string) {
 	return this.isThrottled, this.throttleReason
 }
 
-func (this *MigrationContext) GetMaxLoad() map[string]int64 {
-	this.maxLoadMutex.Lock()
-	defer this.maxLoadMutex.Unlock()
+func (this *MigrationContext) GetThrottleQuery() string {
+	var query string
 
-	tmpMaxLoadMap := make(map[string]int64)
-	for k, v := range this.maxLoad {
-		tmpMaxLoadMap[k] = v
-	}
-	return tmpMaxLoadMap
+	this.throttleMutex.Lock()
+	defer this.throttleMutex.Unlock()
+
+	query = this.ThrottleQuery
+	return query
+}
+
+func (this *MigrationContext) SetThrottleQuery(newQuery string) {
+	this.throttleMutex.Lock()
+	defer this.throttleMutex.Unlock()
+
+	this.ThrottleQuery = newQuery
+}
+
+func (this *MigrationContext) GetMaxLoad() LoadMap {
+	this.throttleMutex.Lock()
+	defer this.throttleMutex.Unlock()
+
+	return this.maxLoad.Duplicate()
+}
+
+func (this *MigrationContext) GetCriticalLoad() LoadMap {
+	this.throttleMutex.Lock()
+	defer this.throttleMutex.Unlock()
+
+	return this.criticalLoad.Duplicate()
 }
 
 // ReadMaxLoad parses the `--max-load` flag, which is in multiple key-value format,
 // such as: 'Threads_running=100,Threads_connected=500'
 // It only applies changes in case there's no parsing error.
 func (this *MigrationContext) ReadMaxLoad(maxLoadList string) error {
-	if maxLoadList == "" {
-		return nil
+	loadMap, err := ParseLoadMap(maxLoadList)
+	if err != nil {
+		return err
 	}
-	this.maxLoadMutex.Lock()
-	defer this.maxLoadMutex.Unlock()
+	this.throttleMutex.Lock()
+	defer this.throttleMutex.Unlock()
 
-	tmpMaxLoadMap := make(map[string]int64)
+	this.maxLoad = loadMap
+	return nil
+}
 
-	maxLoadConditions := strings.Split(maxLoadList, ",")
-	for _, maxLoadCondition := range maxLoadConditions {
-		maxLoadTokens := strings.Split(maxLoadCondition, "=")
-		if len(maxLoadTokens) != 2 {
-			return fmt.Errorf("Error parsing max-load condition: %s", maxLoadCondition)
-		}
-		if maxLoadTokens[0] == "" {
-			return fmt.Errorf("Error parsing status variable in max-load condition: %s", maxLoadCondition)
-		}
-		if n, err := strconv.ParseInt(maxLoadTokens[1], 10, 0); err != nil {
-			return fmt.Errorf("Error parsing numeric value in max-load condition: %s", maxLoadCondition)
-		} else {
-			tmpMaxLoadMap[maxLoadTokens[0]] = n
-		}
+// ReadMaxLoad parses the `--max-load` flag, which is in multiple key-value format,
+// such as: 'Threads_running=100,Threads_connected=500'
+// It only applies changes in case there's no parsing error.
+func (this *MigrationContext) ReadCriticalLoad(criticalLoadList string) error {
+	loadMap, err := ParseLoadMap(criticalLoadList)
+	if err != nil {
+		return err
 	}
+	this.throttleMutex.Lock()
+	defer this.throttleMutex.Unlock()
 
-	this.maxLoad = tmpMaxLoadMap
+	this.criticalLoad = loadMap
 	return nil
 }
 

go/base/load_map.go

@@ -0,0 +1,70 @@
+/*
+   Copyright 2016 GitHub Inc.
+	 See https://github.com/github/gh-ost/blob/master/LICENSE
+*/
+
+package base
+
+import (
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// LoadMap is a mapping of status variable & threshold
+// e.g. [Threads_connected: 100, Threads_running: 50]
+type LoadMap map[string]int64
+
+func NewLoadMap() LoadMap {
+	result := make(map[string]int64)
+	return result
+}
+
+// NewLoadMap parses a `--*-load` flag (e.g. `--max-load`), which is in multiple
+// key-value format, such as:
+//   'Threads_running=100,Threads_connected=500'
+func ParseLoadMap(loadList string) (LoadMap, error) {
+	result := NewLoadMap()
+	if loadList == "" {
+		return result, nil
+	}
+
+	loadConditions := strings.Split(loadList, ",")
+	for _, loadCondition := range loadConditions {
+		loadTokens := strings.Split(loadCondition, "=")
+		if len(loadTokens) != 2 {
+			return result, fmt.Errorf("Error parsing load condition: %s", loadCondition)
+		}
+		if loadTokens[0] == "" {
+			return result, fmt.Errorf("Error parsing status variable in load condition: %s", loadCondition)
+		}
+		if n, err := strconv.ParseInt(loadTokens[1], 10, 0); err != nil {
+			return result, fmt.Errorf("Error parsing numeric value in load condition: %s", loadCondition)
+		} else {
+			result[loadTokens[0]] = n
+		}
+	}
+
+	return result, nil
+}
+
+// Duplicate creates a clone of this map
+func (this *LoadMap) Duplicate() LoadMap {
+	dup := make(map[string]int64)
+	for k, v := range *this {
+		dup[k] = v
+	}
+	return dup
+}
+
+// String() returns a string representation of this map
+func (this *LoadMap) String() string {
+	tokens := []string{}
+	for key, val := range *this {
+		token := fmt.Sprintf("%s=%d", key, val)
+		tokens = append(tokens, token)
+	}
+	sort.Strings(tokens)
+	return strings.Join(tokens, ",")
+}

go/base/load_map_test.go

@@ -0,0 +1,58 @@
+/*
+   Copyright 2016 GitHub Inc.
+	 See https://github.com/github/gh-ost/blob/master/LICENSE
+*/
+
+package base
+
+import (
+	"testing"
+
+	"github.com/outbrain/golib/log"
+	test "github.com/outbrain/golib/tests"
+)
+
+func init() {
+	log.SetLevel(log.ERROR)
+}
+
+func TestParseLoadMap(t *testing.T) {
+	{
+		loadList := ""
+		m, err := ParseLoadMap(loadList)
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(len(m), 0)
+	}
+	{
+		loadList := "threads_running=20,threads_connected=10"
+		m, err := ParseLoadMap(loadList)
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(len(m), 2)
+		test.S(t).ExpectEquals(m["threads_running"], int64(20))
+		test.S(t).ExpectEquals(m["threads_connected"], int64(10))
+	}
+	{
+		loadList := "threads_running=20=30,threads_connected=10"
+		_, err := ParseLoadMap(loadList)
+		test.S(t).ExpectNotNil(err)
+	}
+	{
+		loadList := "threads_running=20,threads_connected"
+		_, err := ParseLoadMap(loadList)
+		test.S(t).ExpectNotNil(err)
+	}
+}
+
+func TestString(t *testing.T) {
+	{
+		m, _ := ParseLoadMap("")
+		s := m.String()
+		test.S(t).ExpectEquals(s, "")
+	}
+	{
+		loadList := "threads_running=20,threads_connected=10"
+		m, _ := ParseLoadMap(loadList)
+		s := m.String()
+		test.S(t).ExpectEquals(s, "threads_connected=10,threads_running=20")
+	}
+}

go/base/utils.go

@@ -33,10 +33,12 @@ func FileExists(fileName string) bool {
 	return false
 }
 
+// StringContainsAll returns true if `s` contains all non empty given `substrings`
+// The function returns `false` if no non-empty arguments are given.
 func StringContainsAll(s string, substrings ...string) bool {
 	nonEmptyStringsFound := false
 	for _, substring := range substrings {
-		if s == "" {
+		if substring == "" {
 			continue
 		}
 		if strings.Contains(s, substring) {

go/base/utils_test.go

@@ -0,0 +1,29 @@
+/*
+   Copyright 2016 GitHub Inc.
+	 See https://github.com/github/gh-ost/blob/master/LICENSE
+*/
+
+package base
+
+import (
+	"testing"
+
+	"github.com/outbrain/golib/log"
+	test "github.com/outbrain/golib/tests"
+)
+
+func init() {
+	log.SetLevel(log.ERROR)
+}
+
+func TestStringContainsAll(t *testing.T) {
+	s := `insert,delete,update`
+
+	test.S(t).ExpectFalse(StringContainsAll(s))
+	test.S(t).ExpectFalse(StringContainsAll(s, ""))
+	test.S(t).ExpectFalse(StringContainsAll(s, "drop"))
+	test.S(t).ExpectTrue(StringContainsAll(s, "insert"))
+	test.S(t).ExpectFalse(StringContainsAll(s, "insert", "drop"))
+	test.S(t).ExpectTrue(StringContainsAll(s, "insert", ""))
+	test.S(t).ExpectTrue(StringContainsAll(s, "insert", "update", "delete"))
+}