update station application to use go-redis/resis/v8 (#108)

This PR updates the application to go-redis/redis/v8 and the applies the proper changes to the way we use the API. This should be compatible with the redis default server used in typical station installs. In support of this change the Docker build process no longer needs to checkout a custom version of the go-redis/redis repo and now. This lead to some modification to the docker workflow that fixes a bug when building using local modifications to source code. Also this PR revamps the CI/CD pipeline to validate tests, linting, and formatting for both rust and golang code.

.github/workflows/build.yml

@@ -6,86 +6,80 @@ name: CI-build
 # events but only for the master branch
 on:
   push:
-    branches: [ master, staging ]
+    branches: [ master ]
   pull_request:
-    branches: [ master, staging ]
+    branches: [ master ]
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
+
   # This workflow contains a single job called "build"
   build:
+    name: Build Station pieces
     # The type of runner that the job will run on
-    runs-on: ubuntu-18.04
-
+    runs-on: ubuntu-latest
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
         with:
           path: go/src/github.com/refraction-networking/conjure
           submodules: recursive
-        
+
       # Build the conjure station
       - name: Install Conjure build dependencies and Build Station Elements
-        env: 
-          GO_VERSION: 1.15.4
-          # RUSTVERSION: 1.47.0
         run: |
-          # Apt deps
           sudo apt-get update
           sudo apt-get install protobuf-compiler gcc curl git wget software-properties-common -y -q
           sudo apt-get install libzmq3-dev libssl-dev pkg-config libgmp3-dev -y -q
-          sudo add-apt-repository universe 
-          wget https://packages.ntop.org/apt-stable/18.04/all/apt-ntop-stable.deb 
-          sudo apt-get install ./apt-ntop-stable.deb 
+          sudo add-apt-repository universe
+          wget https://packages.ntop.org/apt-stable/18.04/all/apt-ntop-stable.deb
+          sudo apt-get install ./apt-ntop-stable.deb
           sudo apt-get update
           sudo apt-get install pfring
           echo "Apt dependencies installed"
-          echo "--------------------------------------"
-          
-          echo "HOME=$HOME"
-          export GOPATH=`pwd`/go
-          echo "GOPATH=$GOPATH"
-          cd go/src/github.com/refraction-networking/conjure
-
-          # Install rust
-          curl https://sh.rustup.rs -sSf -o install_rust.sh; sh install_rust.sh -y;          
-          cargo install protobuf-codegen
-          export PATH=$PATH:$HOME/.cargo/bin
-          source $HOME/.cargo/env
-          echo "Rust successfully installed"
-          echo "--------------------------------------"
 
-          # Install Golang
-          sudo rm -rf /usr/local/go /usr/bin/go
-          wget -q https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz
-          sudo tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz
-          export PATH=$PATH:/usr/local/go/bin
-          export GOROOT="/usr/local/go"
-          go version
-          # temp fix before transition to redis v8
-          go get -u github.com/go-redis/redis || true && cd $GOPATH/src/github.com/go-redis/redis && git checkout tags/v7.4.0 -b v7-master && cd -
-          go get -u github.com/BurntSushi/toml || true
-          go get -u github.com/gorilla/mux || true
-          go get -d -u -t github.com/refraction-networking/gotapdance/... || true
-          go get -u github.com/refraction-networking/conjure/application/... || true 
-          go get -u github.com/refraction-networking/conjure/registration-api/... || true 
-          echo "Golang and go dependencies successfully installed"
-          echo "--------------------------------------"
+      - name: Build Rust Library
+        run: |
+          cd $GITHUB_WORKSPACE/go/src/github.com/refraction-networking/conjure
+          make rust
 
-          # Build it
-          cd $GOPATH/src/github.com/refraction-networking/conjure
-          make
+      - name: Build Conjure Detector
+        run: |
+          cd $GITHUB_WORKSPACE/go/src/github.com/refraction-networking/conjure
+          make libtd
+          make conjure
           echo "Station successfully built"
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15.4
+
+      - name: Build app
+        run: |
+          cd $GITHUB_WORKSPACE/go/src/github.com/refraction-networking/conjure
+          make app
+
+      - name: Build registration-api
+        run: |
+          cd $GITHUB_WORKSPACE/go/src/github.com/refraction-networking/conjure
+          make registration-api
+
+      - name: Store build artifacts
+        run: |
           mkdir -p  $GITHUB_WORKSPACE/bin
+          cd $GITHUB_WORKSPACE/go/src/github.com/refraction-networking/conjure
           cp conjure application/application registration-api/registration-api $GITHUB_WORKSPACE/bin
           cd $GITHUB_WORKSPACE && tar -czf conjure-station.tar.gz bin
-          
-      
-      - name: Save Build artifacts
+
+      - name: Save build artifacts
         uses: actions/upload-artifact@v2
         with:
           name: conjure-station.tar.gz
           path: |
             conjure-station.tar.gz
-            
+

.github/workflows/golang.yml

@@ -9,20 +9,52 @@ on:
 jobs:
 
   build:
+    name: Build and Test using Golang ${{ matrix.go-version }}
+    strategy:
+      fail-fast: false
+      matrix:
+        go-version: [1.15.x, 1.16.x, 1.17.x]
+
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
+        with:
+          path: go/src/github.com/refraction-networking/conjure
+
+
+      - name: Install deps
+        run: |
+          sudo apt-get update
+          sudo apt-get install protobuf-compiler software-properties-common -y -q
+          sudo apt-get install libzmq3-dev libssl-dev pkg-config libgmp3-dev -y -q
 
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.15.4
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
 
-    - name: Build app
-      run: go build -v ./application
+      - name: Test
+        run: |
+          cd $GITHUB_WORKSPACE/go/src/github.com/refraction-networking/conjure
+          export GOPATH="$GITHUB_WORKSPACE/go"
+          go test -v ./...
 
-    - name: Build registration-api
-      run: go build -v ./registration-api
+      - name: Build app
+        run: |
+          cd $GITHUB_WORKSPACE/go/src/github.com/refraction-networking/conjure
+          make app
 
-    - name: Test
-      run: go test -v ./...
+      - name: Build registration-api
+        run: |
+          cd $GITHUB_WORKSPACE/go/src/github.com/refraction-networking/conjure
+          make registration-api
+
+  golangci-lint:
+    name: Format and Lint with golangci-lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: golangci-lint
+        uses: golangci/[email protected]
+        with:
+          version: latest

.github/workflows/rust.yml

@@ -0,0 +1,70 @@
+name: rust
+
+on:
+  push:
+    branches: [ master]
+  pull_request:
+    branches: [ master ]
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+
+jobs:
+  test:
+    name: Test Rust ${{ matrix.rust }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # ubuntu-latest is another name for ubuntu-20.04
+          # - { rust: stable,           os: ubuntu-latest }
+          - { rust: stable,           os: ubuntu-18.04 }
+          - { rust: stable,           os: ubuntu-20.04 }
+          - { rust: nightly,          os: ubuntu-latest }
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: install deps
+        run: |
+          sudo apt-get update
+          sudo apt-get install protobuf-compiler software-properties-common -y -q
+          sudo apt-get install libzmq3-dev libssl-dev pkg-config libgmp3-dev -y -q
+
+      - uses: hecrj/setup-rust-action@v1
+        with:
+          rust-version: ${{ matrix.rust }}
+
+      - run: cargo test --verbose --workspace
+      # - run: cargo test --verbose --workspace --all-features
+      # - run: cargo test --verbose --workspace --no-default-features
+
+  clippy:
+    name: Lint with clippy
+    runs-on: ubuntu-latest
+    env:
+      RUSTFLAGS: -Dwarnings
+    steps:
+      - uses: actions/checkout@v2
+      - name: install deps
+        run: |
+          sudo apt-get update
+          sudo apt-get install protobuf-compiler software-properties-common -y -q
+          sudo apt-get install libzmq3-dev libssl-dev pkg-config libgmp3-dev -y -q
+      - uses: hecrj/setup-rust-action@v1
+        with:
+          components: clippy
+      - run: cargo clippy --workspace --all-targets --verbose
+      # - run: cargo clippy --workspace --all-targets --verbose --no-default-features
+      # - run: cargo clippy --workspace --all-targets --verbose --all-features
+
+  rustfmt:
+    name: Verify code formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: hecrj/setup-rust-action@v1
+        with:
+          components: rustfmt
+      - run: cargo fmt --all -- --check

Makefile

@@ -16,7 +16,7 @@ rust: ./src/*.rs
 	cargo build --${DEBUG_OR_RELEASE}
 
 test:
-	cargo test --${DEBUG_OR_RELEASE} 
+	cargo test --${DEBUG_OR_RELEASE}
 
 app:
 	cd ./application/ && make
@@ -31,6 +31,12 @@ conjure: detect.c loadkey.c rust_util.c rust libtapdance
 registration-api:
 	cd ./registration-api/ && make
 
+# Note this copies in the whole current directory as context and results in
+# overly large context. should not be used to build release/production images.
+custom-build:
+	docker build --build-arg CUSTOM_BUILD=1 -f docker/Dockerfile .
+
+
 clean:
 	cargo clean
 	rm -f ${TARGETS} *.o *~

README.md

@@ -37,10 +37,6 @@ Building the station requires both go and rust:
 ```sh
 sudo apt install  wget git make gcc bison flex protobuf-compiler curl libssl-dev pkg-config libgmp3-dev libzmq3-dev
 go get -d -u -t github.com/refraction-networking/gotapdance/...
-
-# Due to compatibility issue with redis v8 -- checkout the stable 7.4.0 go library version.
-go get -d github.com/go-redis/redis
-cd ${GOPATH}/src/github.com/go-redis/redis && git checkout tags/v7.4.0 -b v7-master 
 ```
 
 **Install PF_RING** 

application/Makefile

@@ -1,2 +1,2 @@
 all:
-	/usr/local/go/bin/go build -race -a .
+	go build -race -a .

application/lib/detector_channel.go

@@ -1,11 +1,12 @@
 package lib
 
 import (
+	"context"
 	"log"
 	"os"
 	"sync"
 
-	"github.com/go-redis/redis"
+	"github.com/go-redis/redis/v8"
 )
 
 var client *redis.Client
@@ -27,8 +28,9 @@ func initRedisClient() {
 		PoolSize: 100,
 	})
 
+	ctx := context.Background()
 	// Ping to test redis connection
-	_, err := client.Ping().Result()
+	_, err := client.Ping(ctx).Result()
 	if err != nil {
 		logger := log.New(os.Stderr, "[REDIS] ", log.Ldate|log.Lmicroseconds)
 		logger.Printf("redis connection ping failed.")

application/lib/registration.go

@@ -694,5 +694,7 @@ func registerForDetector(reg *DecoyRegistration) {
 		// throw(fit)
 		return
 	}
-	client.Publish(DETECTOR_REG_CHANNEL, string(s2d))
+
+	ctx := context.Background()
+	client.Publish(ctx, DETECTOR_REG_CHANNEL, string(s2d))
 }