Add backup and restore for bootstrapping

.gitignore

@@ -10,3 +10,6 @@ snapshotserver/snapshotserver
 .vscode
 test-reports/
 docker-test-reports/
+bootstrap/cloud_functions/config.json
+bootstrap/cloud_functions/node_modules
+bootstrap/cloud_functions/test.out

bootstrap/README.md

@@ -0,0 +1,24 @@
+# Bootstrap
+
+The Bootstrap and Bootstrap Google Cloud Functions allow a cluster of Transicator Change Servers to backup and restore 
+from a Google Cloud Store. This helps avoid a potential issue when scaling up the Change Server cluster where existing 
+clients may attach to the new Change Server, receive a "snapshot too old" message because the new Change Server
+doesn't have all necessary context, and force the clients to request new snapshots.
+
+The algorithm at this point is simple: If the backup file age > minBackupAge configured on the bootstrap, the backup 
+offered by the Change Server is accepted and stored. A future enhancement may be to consider a delta of changes made 
+between backups or similar. 
+
+## Using
+
+1. Install the Google Cloud Functions as detailed in [cloud_functions/README.md]()
+2. Configure Change Server to point to the correct URLs.
+
+## Testing
+
+1. As above, install the Google Cloud Functions.
+2. If you haven't set the correct REGION and PROJECT_ID env vars in your terminal already, do so now.  
+3. Run `go test`
+
+Note: The Tests may not always succeed 100% do to the asynchronous and distributed nature of Google Cloud Functions and 
+Google Cloud Storage.

bootstrap/bootstrap.go

@@ -0,0 +1,135 @@
+/*
+Copyright 2016 The Transicator Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package bootstrap
+
+import (
+	"fmt"
+	log "github.com/Sirupsen/logrus"
+	"github.com/apigee-labs/transicator/storage"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"time"
+)
+
+const (
+	headerID     = "x-bootstrap-id"
+	headerSecret = "x-bootstrap-secret"
+	contentType  = "application/octet-stream"
+)
+
+type Error struct {
+	Status  int
+	Message string
+}
+
+func (e Error) Error() string {
+	return fmt.Sprintf("backup error, status: %d, message: %s", e.Status, e.Message)
+}
+
+var client = http.Client{Timeout: 3 * time.Minute}
+
+// note: current behavior is to always make the backup and then just allow the send to fail
+// if the server doesn't want it.
+// note: err returned is nil and uploaded is false if backup was rejected as being too soon
+func Backup(db storage.DB, uri, id, secret string) (err error, uploaded bool) {
+
+	// make backup
+	tempDirURL, err := ioutil.TempDir("", "transicator_sqlite_backup")
+	if err != nil {
+		return err, false
+	}
+	defer os.RemoveAll(tempDirURL)
+	backupDirName := fmt.Sprintf("%s/backup", tempDirURL)
+
+	bc := db.Backup(backupDirName)
+	for {
+		br := <-bc
+		log.Debugf("Backup %d remaining\n", br.PagesRemaining)
+		if err = br.Error; err != nil {
+			log.Error(err)
+			return err, false
+		}
+		if br.Done {
+			break
+		}
+	}
+
+	backupFileName := fmt.Sprintf("%s/transicator", backupDirName)
+	fileReader, err := os.Open(backupFileName)
+	if err != nil {
+		log.Error(err)
+		return err, false
+	}
+	defer fileReader.Close()
+
+	// send backup
+	req, err := http.NewRequest("POST", uri, fileReader)
+	if err != nil {
+		return err, false
+	}
+	req.Header.Set(headerID, id)
+	req.Header.Set(headerSecret, secret)
+	req.Header.Set("Content-Type", contentType)
+
+	res, err := client.Do(req)
+	if err != nil {
+		return err, false
+	}
+	defer res.Body.Close()
+
+	switch res.StatusCode {
+	case 200:
+		log.Debug("backup uploaded successfully")
+		return nil, true
+	case 429:
+		log.Debug("backup upload not needed")
+		return nil, false
+	default:
+		msg, _ := ioutil.ReadAll(res.Body)
+		return Error{
+			res.StatusCode,
+			string(msg),
+		}, false
+	}
+}
+
+func Restore(dbDir, uri, id, secret string) error {
+
+	req, err := http.NewRequest("GET", uri, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set(headerID, id)
+	req.Header.Set(headerSecret, secret)
+	req.Header.Set("Accept", contentType)
+
+	res, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != 200 {
+		msg, _ := ioutil.ReadAll(res.Body)
+		return Error{
+			res.StatusCode,
+			string(msg),
+		}
+	}
+
+	return storage.RestoreBackup(res.Body, dbDir)
+}

bootstrap/bootstrap_suite_test.go

@@ -0,0 +1,36 @@
+package bootstrap_test
+
+import (
+	"fmt"
+	log "github.com/Sirupsen/logrus"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/spf13/viper"
+	"testing"
+)
+
+func TestBootstrap(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Bootstrap Suite")
+}
+
+var backupURI, restoreURI string
+
+var _ = BeforeSuite(func() {
+	log.SetLevel(log.DebugLevel)
+
+	err := viper.BindEnv("region", "REGION")
+	Expect(err).NotTo(HaveOccurred())
+	err = viper.BindEnv("projectID", "PROJECT_ID")
+	Expect(err).NotTo(HaveOccurred())
+
+	region := viper.GetString("region")
+	projectID := viper.GetString("projectID")
+
+	if region == "" || projectID == "" {
+		Fail("Set env vars: REGION and PROJECT_ID before running test")
+	}
+
+	backupURI = fmt.Sprintf("https://%s-%s.cloudfunctions.net/bootstrapBackup", region, projectID)
+	restoreURI = fmt.Sprintf("https://%s-%s.cloudfunctions.net/bootstrapRestore", region, projectID)
+})

bootstrap/bootstrap_test.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2016 The Transicator Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package bootstrap_test
+
+import (
+	"github.com/apigee-labs/transicator/bootstrap"
+	"github.com/apigee-labs/transicator/storage"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"io/ioutil"
+	"os"
+	"time"
+)
+
+const (
+	secret = "testing"
+)
+
+var _ = Describe("Bootstrap tests", func() {
+
+	var (
+		testDB  storage.DB
+		testDir string
+	)
+
+	BeforeEach(func() {
+		tmpDir, err := ioutil.TempDir("", "bootstrap_test")
+		Expect(err).NotTo(HaveOccurred())
+
+		testDB, err = storage.Open(tmpDir)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		testDB.Close()
+		err := testDB.Delete()
+		Expect(err).Should(Succeed())
+		os.RemoveAll(testDir)
+	})
+
+	It("can backup and restore a database", func() {
+		id := "test1"
+
+		err := testDB.Put("foo", 1, 1, []byte("Hello!"))
+		Expect(err).NotTo(HaveOccurred())
+
+		err, uploaded := bootstrap.Backup(testDB, backupURI, id, secret)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(uploaded).To(BeTrue())
+
+		restoreDir, err := ioutil.TempDir(testDir, "bootstrap_test")
+		Expect(err).NotTo(HaveOccurred())
+
+		// this is to help (although it can't guarantee) that GCS synchronizes between backup calls
+		time.Sleep(5 * time.Second)
+
+		err = bootstrap.Restore(restoreDir, restoreURI, id, secret)
+		Expect(err).NotTo(HaveOccurred())
+
+		db, err := storage.Open(restoreDir)
+		Expect(err).Should(Succeed())
+
+		ret, err := db.Get("foo", 1, 1)
+		Expect(err).Should(Succeed())
+		Expect(ret).Should(Equal([]byte("Hello!")))
+		db.Close()
+	}, 6)
+
+	Context("backup", func() {
+
+		It("should not replace a recent backup", func() {
+			id := "test2"
+
+			err, uploaded := bootstrap.Backup(testDB, backupURI, id, secret)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(uploaded).To(BeTrue())
+
+			// this is to help (although it can't guarantee) that GCS synchronizes between backup calls
+			time.Sleep(5 * time.Second)
+
+			err, uploaded = bootstrap.Backup(testDB, backupURI, id, secret)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(uploaded).To(BeFalse())
+		}, 6)
+
+		It("should fail without id", func() {
+			err, uploaded := bootstrap.Backup(testDB, backupURI, "", secret)
+			Expect(err).To(HaveOccurred())
+			Expect(uploaded).To(BeFalse())
+		})
+
+		It("should fail without secret", func() {
+			err, uploaded := bootstrap.Backup(testDB, backupURI, "xxx", "")
+			Expect(err).To(HaveOccurred())
+			Expect(uploaded).To(BeFalse())
+		})
+	})
+
+	Context("restore", func() {
+
+		It("should fail without id", func() {
+			err := bootstrap.Restore(testDir, backupURI, "", secret)
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("should fail without secret", func() {
+			err := bootstrap.Restore(testDir, backupURI, "xxx", "")
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("should fail with 404 if backup is not present", func() {
+			err := bootstrap.Restore(testDir, restoreURI, "not a real backup", secret)
+			Expect(err).To(HaveOccurred())
+			e, ok := err.(bootstrap.Error)
+			Expect(ok).To(BeTrue())
+			Expect(e.Status).To(Equal(404))
+		})
+
+		It("should fail with 403 if wrong secret", func() {
+			id := "test3"
+
+			err, uploaded := bootstrap.Backup(testDB, backupURI, id, secret)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(uploaded).To(BeTrue())
+
+			// this is to help (although it can't guarantee) that GCS synchronizes between backup calls
+			time.Sleep(5 * time.Second)
+
+			err = bootstrap.Restore(testDir, restoreURI, id, "not the real secret")
+			Expect(err).To(HaveOccurred())
+			e, ok := err.(bootstrap.Error)
+			Expect(ok).To(BeTrue())
+			Expect(e.Status).To(Equal(403))
+		}, 6)
+
+	})
+})