[ci skip] 2019.02.19-2175

.circleci/config.yml

@@ -1,4 +1,56 @@
-version: 2
+version: 2.1
+commands:
+  dctest-command:
+    description: "datacenter test"
+    parameters:
+      script:
+        type: string
+      suite:
+        type: string
+      datacenter:
+        type: string
+        default: ""
+    steps:
+      - run:
+          name: Store Service Account
+          command: |
+            if [ -f .skip ]; then exit 0; fi
+            echo $GCLOUD_SERVICE_ACCOUNT > account.json
+            gcloud auth activate-service-account --key-file=account.json
+      - run:
+          name: Store secrets
+          command: |
+            if [ -f .skip ]; then exit 0; fi
+            echo "$QUAY_PASSWORD" > secrets
+      - run:
+          name: Store github-token
+          command: |
+            if [ -f .skip ]; then exit 0; fi
+            echo "$NECO_GITHUB_TOKEN" > github-token
+      - run:
+          name: Watch console on boot-0
+          command: |
+            if [ -f .skip ]; then exit 0; fi
+            ./bin/watch_boot0
+          background: true
+      - run:
+          name: Watch neco-updater logs
+          command: |
+            if [ -f .skip ]; then exit 0; fi
+            ./bin/watch_service neco-updater
+          background: true
+      - run:
+          name: Watch neco-worker logs
+          command: |
+            if [ -f .skip ]; then exit 0; fi
+            ./bin/watch_service neco-worker
+          background: true
+      - run:
+          command: |
+            if [ -f .skip ]; then exit 0; fi
+            ./bin/<<parameters.script>> <<parameters.suite>> <<parameters.datacenter>>
+          no_output_timeout: 20m
+
 jobs:
   build:
     docker:
@@ -43,38 +95,9 @@ jobs:
     resource_class: large
     steps:
       - checkout
-      - run:
-          name: Store Service Account
-          command: |
-            echo $GCLOUD_SERVICE_ACCOUNT > account.json
-            gcloud auth activate-service-account --key-file=account.json
-      - run:
-          name: Store secrets
-          command: |
-            echo "$QUAY_PASSWORD" > secrets
-      - run:
-          name: Store github-token
-          command: |
-            echo "$NECO_GITHUB_TOKEN" > github-token
-      - run:
-          name: Watch console on boot-0
-          command: |
-            ./bin/watch_boot0
-          background: true
-      - run:
-          name: Watch neco-updater logs
-          command: |
-            ./bin/watch_service neco-updater
-          background: true
-      - run:
-          name: Watch neco-worker logs
-          command: |
-            ./bin/watch_service neco-worker
-          background: true
-      - run:
-          command: |
-            ./bin/run-dctest.sh
-          no_output_timeout: 20m
+      - dctest-command:
+          script: run-dctest.sh
+          suite: ./functions
       - run:
           name: Clean up SSH keys generated by `gcloud compute`
           command: |
@@ -90,81 +113,106 @@ jobs:
           root: .
           paths:
             - "artifacts_release.go"
-  update-release:
+  dctest-release-functions:
       docker:
       - image: google/cloud-sdk
       steps:
       - checkout
-      - attach_workspace:
-          at: /tmp/workspace
+      # - attach_workspace:
+      #     at: /tmp/workspace
       - run:
           name: Configure GIT
           command: |
             git config --global user.email "[email protected]"
             git config --global user.name "cybozu-neco"
       - run: git checkout release
       - run: git merge --no-commit origin/master
-      - run: cp /tmp/workspace/artifacts_release.go .
-      - run: 
+      # - run: cp /tmp/workspace/artifacts_release.go .
+      - run:
           name: Check diff
           command: |
             diffs=$(git status -s)
-            if [ "$diffs" = "" ]; then exit 0; fi
-            touch .diff
+            if [ "$diffs" = "" ]; then touch .skip; exit 0; fi
             printf "%s\n" "$diffs"
+      - dctest-command:
+          script: run-dctest-release.sh
+          suite: ./functions
+  dctest-release-upgrade-staging:
+      docker:
+      - image: google/cloud-sdk
+      steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/workspace
       - run:
-          name: Store Service Account
-          command: |
-            if [ ! -f .diff ]; then exit 0; fi
-            echo $GCLOUD_SERVICE_ACCOUNT > account.json
-            gcloud auth activate-service-account --key-file=account.json
-      - run:
-          name: Store secrets
-          command: |
-            if [ ! -f .diff ]; then exit 0; fi
-            echo "$QUAY_PASSWORD" > secrets
-      - run:
-          name: Store github-token
+          name: Configure GIT
           command: |
-            echo "$NECO_GITHUB_TOKEN" > github-token
+            git config --global user.email "[email protected]"
+            git config --global user.name "cybozu-neco"
+      - run: git checkout release
+      - run: git merge --no-commit origin/master
+      - run: cp /tmp/workspace/artifacts_release.go .
       - run:
-          name: Watch console on boot-0
+          name: Check diff
           command: |
-            if [ ! -f .diff ]; then exit 0; fi
-            ./bin/watch_boot0
-          background: true
+            diffs=$(git status -s)
+            if [ "$diffs" = "" ]; then touch .skip; exit 0; fi
+            printf "%s\n" "$diffs"
+      - dctest-command:
+          script: run-dctest-release.sh
+          suite: ./bootstrap
+          datacenter: staging
       - run:
-          name: Watch neco-updater logs
           command: |
-            if [ ! -f .diff ]; then exit 0; fi
-            ./bin/watch_service neco-updater
-          background: true
+            if [ -f .skip ]; then exit 0; fi
+            ./bin/run-dctest-upgrade.sh
+          no_output_timeout: 20m
+  update-release:
+      docker:
+      - image: quay.io/cybozu/golang:1.11-bionic
+      steps:
+      - checkout
+#      - attach_workspace:
+#          at: /tmp/workspace
       - run:
-          name: Watch neco-worker logs
+          name: Configure GIT
           command: |
-            if [ ! -f .diff ]; then exit 0; fi
-            ./bin/watch_service neco-worker
-          background: true
+            git config --global user.email "[email protected]"
+            git config --global user.name "cybozu-neco"
+      - run: git checkout release
+      - run: git merge --no-commit origin/master
+#      - run: cp /tmp/workspace/artifacts_release.go .
       - run:
+          name: Check diff
           command: |
-            if [ ! -f .diff ]; then exit 0; fi
-            ./bin/run-dctest-release.sh
-          no_output_timeout: 20m
+            diffs=$(git status -s)
+            if [ "$diffs" = "" ]; then touch .skip; exit 0; fi
+            printf "%s\n" "$diffs"
       - add_ssh_keys:
           fingerprints:
             - "d7:2a:c8:5d:c4:32:7e:20:6f:bc:20:7c:ab:c4:24:88"
+      - run:
+          name: Store github-token
+          command: |
+            if [ -f .skip ]; then exit 0; fi
+            echo "$NECO_GITHUB_TOKEN" > github-token
       - run:
           name: Commit and push to GitHub
           command: |
-            if [ ! -f .diff ]; then exit 0; fi
+            if [ -f .skip ]; then exit 0; fi
             ./bin/commit-push.sh
 
 workflows:
   version: 2
   main:
     jobs:
       - build
-      - dctest
+      - dctest:
+          filters:
+            branches:
+              ignore:
+                - release
+                - master
   release:
     jobs:
       - build_release:
@@ -191,19 +239,41 @@ workflows:
                 - release
     jobs:
       - generate-artifacts
-      - update-release:
+      - dctest-release-functions:
+          requires:
+            - generate-artifacts
+      - dctest-release-upgrade-staging:
           requires:
             - generate-artifacts
+      - update-release:
+          requires:
+            - dctest-release-functions
+            - dctest-release-upgrade-staging
   master:
     jobs:
-      - generate-artifacts:
+      # - generate-artifacts:
+      #     filters:
+      #       branches:
+      #         only:
+      #           - master
+      - dctest-release-functions:
+          # requires:
+          #   - generate-artifacts
           filters:
             branches:
               only:
                 - master
+      # - dctest-release-upgrade-staging:
+      #     requires:
+      #       - generate-artifacts
+      #     filters:
+      #       branches:
+      #         only:
+      #           - master
       - update-release:
           requires:
-            - generate-artifacts
+            - dctest-release-functions
+            # - dctest-release-upgrade-staging
           filters:
             branches:
               only:

.gitignore

@@ -13,10 +13,15 @@
 .#*
 \#*#
 .vscode
+.DS_Store
 
 # Debian package
 /work
 /neco_*_amd64.deb
 
 # ignore go.sum
 /go.sum
+
+# ignore gcp files
+/.necogcp.yml
+pkg/necogcp-app/cron.yaml

Makefile

@@ -21,10 +21,11 @@ SHAREDIR := $(WORKDIR)/usr/share/neco
 VERSION = 0.0.1-master
 DEST = .
 DEB = neco_$(VERSION)_amd64.deb
-BIN_PKGS = ./pkg/neco
+BIN_PKGS = ./pkg/neco ./pkg/necogcp
 SBIN_PKGS = ./pkg/neco-updater ./pkg/neco-worker ./pkg/sabakan-serf-handler
 NODE_EXPORTER_VERSION = 0.17.0
 NODE_EXPORTER_URL = https://github.com/prometheus/node_exporter/archive/v$(NODE_EXPORTER_VERSION).tar.gz
+STATIK = gcp/statik/statik.go
 
 all:
 	@echo "Specify one of these targets:"
@@ -42,7 +43,11 @@ start-etcd:
 stop-etcd:
 	systemctl --user stop neco-etcd.service
 
-test:
+$(STATIK):
+	mkdir -p $(dir $(STATIK))
+	go generate ./pkg/necogcp/...
+
+test: $(STATIK)
 	test -z "$$(gofmt -s -l . | grep -v '^vendor' | tee /dev/stderr)"
 	test -z "$$(golint $$(go list -tags='$(GOTAGS)' ./... | grep -v /vendor/) | grep -v '/dctest/.*: should not use dot imports' | tee /dev/stderr)"
 	go build -tags='$(GOTAGS)' ./...
@@ -58,7 +63,7 @@ mod:
 
 deb: $(DEB)
 
-$(DEB):
+$(DEB): $(STATIK)
 	rm -rf $(WORKDIR)
 	cp -r debian $(WORKDIR)
 	mkdir -p $(WORKDIR)/src $(BINDIR) $(SBINDIR) $(SHAREDIR) $(DOCDIR)/neco $(DOCDIR)/node_exporter
@@ -77,12 +82,15 @@ $(DEB):
 	$(FAKEROOT) dpkg-deb --build $(WORKDIR) $(DEST)
 	rm -rf $(WORKDIR)
 
+necogcp: $(STATIK)
+	go install ./pkg/necogcp
+
 setup:
-	GO111MODULE=off go get -u golang.org/x/lint/golint
+	GO111MODULE=off go get -u golang.org/x/lint/golint github.com/rakyll/statik
 	$(SUDO) apt-get update
 	$(SUDO) apt-get -y install --no-install-recommends $(PACKAGES)
 
 clean:
 	rm -rf $(ETCD_DIR) $(WORKDIR) $(DEB)
 
-.PHONY:	all start-etcd stop-etcd test mod deb setup clean
+.PHONY:	all start-etcd stop-etcd test mod deb necogcp setup clean