Layout Update (#179)

* add prelim changes for inf updates

* README mistakes

* build and test fixes

* env var export in start_zbalance

.github/workflows/build.yml

@@ -55,9 +55,9 @@ jobs:
           echo "Station successfully built"
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.18.x
+          go-version: 'stable'
 
       - name: Build app
         run: |
@@ -71,9 +71,9 @@ jobs:
 
       - name: Store build artifacts
         run: |
-          mkdir -p  $GITHUB_WORKSPACE/bin
+          mkdir -p  $GITHUB_WORKSPACE/
           cd $GITHUB_WORKSPACE/go/src/github.com/refraction-networking/conjure
-          cp conjure application/application cmd/registration-server/registration-server $GITHUB_WORKSPACE/bin
+          cp -r ./bin/ $GITHUB_WORKSPACE/
           cd $GITHUB_WORKSPACE && tar -czf conjure-station.tar.gz bin
 
       - name: Save build artifacts

.gitignore

@@ -4,4 +4,5 @@ conjure
 application/application
 libtapdance/genkey
 cmd/registration-server/registration-server
-target
+target
+bin/

Makefile

@@ -8,7 +8,7 @@ TD_LIB=./libtapdance/libtapdance.a
 LIBS=${RUST_LIB} ${TD_LIB} -L/usr/local/lib -lpcap -lpfring -lzmq -lcrypto -lpthread -lrt -lgmp -ldl -lm
 CFLAGS = -Wall -DENABLE_BPF -DHAVE_PF_RING -DHAVE_PF_RING_ZC -DTAPDANCE_USE_PF_RING_ZERO_COPY -O2 # -g
 PROTO_RS_PATH=src/signalling.rs
-
+EXE_DIR=./bin
 
 all: rust libtd conjure app registration-server ${PROTO_RS_PATH}
 
@@ -21,20 +21,24 @@ test:
 	cargo test --${DEBUG_OR_RELEASE}
 
 app:
-	cd ./application/ && make
+	[ -d $(EXE_DIR) ] || mkdir -p $(EXE_DIR)
+	go build -o ${EXE_DIR}/application ./application
 
 libtd:
 	cd ./libtapdance/ && make libtapdance.a
 
 conjure: detect.c loadkey.c rust_util.c rust libtapdance
-	${CC} ${CFLAGS} -o $@ detect.c loadkey.c rust_util.c ${LIBS}
-# gcc -Wall -DENABLE_BPF -DHAVE_PF_RING -DHAVE_PF_RING_ZC -DTAPDANCE_USE_PF_RING_ZERO_COPY -O2 -o conjure detect.c loadkey.c rust_util.c ./target/release/librust_dark_decoy.a ./libtapdance/libtapdance.a -lpfring -lpcap -L/usr/local/lib -lzmq -lcrypto -lpthread -lrt -lgmp -ldl -lm
+	[ -d $(EXE_DIR) ] || mkdir -p $(EXE_DIR)
+	${CC} ${CFLAGS} -o ${EXE_DIR}/$@ detect.c loadkey.c rust_util.c ${LIBS}
+
 
 conjure-sim: detect.c loadkey.c rust_util.c rust libtapdance
-	${CC} -Wall -O2 -o conjure detect.c loadkey.c rust_util.c ${LIBS}
+	[ -d $(EXE_DIR) ] || mkdir -p $(EXE_DIR)
+	${CC} -Wall -O2 -o ${EXE_DIR}/conjure detect.c loadkey.c rust_util.c ${LIBS}
 
 registration-server:
-	cd ./cmd/registration-server/ && make
+	[ -d $(EXE_DIR) ] || mkdir -p $(EXE_DIR)
+	go build -o ${EXE_DIR}/registration-server ./cmd/registration-server
 
 PARAMS := det app reg zbalance sim
 target := unk
@@ -81,7 +85,7 @@ endif
 
 clean:
 	cargo clean
-	rm -f ${TARGETS} *.o *~
+	rm -f ${TARGETS} *.o *~ ${EXE_DIR}
 
 ${PROTO_RS_PATH}:
 	cd ./proto/ && make

README.md

@@ -9,7 +9,7 @@
 <a href="https://goreportcard.com/report/github.com/refraction-networking/conjure"><img src="https://goreportcard.com/badge/github.com/refraction-networking/conjure"></a>
 </p>
 
-### See also 
+### See also
 
 [Refraction Client Library](https://github.com/refraction-networking/gotapdance) -
 pure golang client library for connecting to refraction systems. Implements BOTH
@@ -23,14 +23,13 @@ station code implementing the previous iteration of refraction networking develo
 This is an abridged install and configuration process, for expanded install instructions,
 configuration options, or multi-station deployments see [the wiki](https://github.com/refraction-networking/conjure/wiki).
 
-
 ### Requirements
 
 Building the station requires both go and rust:
 
 - [Install Golang](https://golang.org/doc/install)
 
-- [Install Rust](https://www.rust-lang.org/tools/install) 
+- [Install Rust](https://www.rust-lang.org/tools/install)
 
 **Install packages and go libraries**
 
@@ -39,12 +38,12 @@ sudo apt install  wget git make gcc bison flex protobuf-compiler curl libssl-dev
 go get -d -u -t github.com/refraction-networking/gotapdance/...
 ```
 
-**Install PF_RING** 
+**Install PF_RING**
 
 1. [Install from Package](https://github.com/refraction-networking/conjure/wiki/PF_RING#from-packages)
 
 2. [Install From Git / Source](https://github.com/refraction-networking/conjure/wiki/PF_RING#from-source)
-    * if installing from git / source make the zbalance_ipc executable, and ensure that it is available through your `$PATH`.
+    - if installing from git / source make the zbalance_ipc executable, and ensure that it is available through your `$PATH`.
 
 ### Build the station
 
@@ -57,6 +56,38 @@ make
 
 ### Configure
 
+The layout of configuration expected by the default layout of a production server is:
+
+```sh
+## Station specific configuration and files go in /var/lib/conjure
+$ tree /var/lib/conjure/
+/var/lib/conjure/
+├── app_config.toml
+├── ClientConf      # if running the registration server locally
+├── conjure.conf
+├── phantom_subnets.toml
+├── privkey
+├── pubkey
+└── reg_config.toml # if running the registration server locally
+
+## Scripts, executables, and the default environment script (conjure.conf) go in /opt/conjure
+$ tree /opt/conjure/
+/opt/conjure/
+├── bin
+│   ├── application
+│   ├── conjure
+│   └── registration_server  # if running the registration server locally
+├── on-reboot.sh
+├── scripts
+│   ├── install_pfring.sh
+│   ├── start_application.sh
+│   ├── start_detector.sh
+│   ├── start_registrar.sh
+│   └── start_zbalance_ipc.sh
+└── sysconfig
+    └── conjure.conf    # Expected by systemd services, applies overrides from /var/lib/conjure/conjure.conf
+```
+
 To run a station configuration modifications are required. This section outlines
 some minimal changes, for more configuration options see the [wiki configuration page](https://github.com/refraction-networking/conjure/wiki/Configuration).
 
@@ -67,24 +98,25 @@ some minimal changes, for more configuration options see the [wiki configuration
     # The interface(s) which PF_RING Zero Copy will tap.
     CJ_IFACE="zc:enp179s0f0,zc:enp179s0f1"
 
-    Public addresses that of non-tap interface - used for kernel DNAT 
+    Public addresses that of non-tap interface - used for kernel DNAT
     IP4_ADDR="<PUBLIC_STATION_V4_ADDRESS>"
     IP6_ADDR="<PUBLIC_STATION_V6_ADDRESS>"
 
     ```
 
     Note: ipv6 in disabled by default. To enable IPv6 modify
     `application/config.toml`
+
     ```diff
     # Allow the station to opt out of either version of internet protocol to limit a
     # station to handling one or the other. For example, v6 on small station deployment
-    # with only v6 phantom subnet,  v4 only on station with no puvlic v6 address. 
+    # with only v6 phantom subnet,  v4 only on station with no puvlic v6 address.
     enable_v4 = true
     -enable_v6 = false
     +enable_v6 = true
     ```
 
-2. Define application parameters in `application/congfig.toml`
+2. Define application parameters in `application/app_config.toml`
 
     ```toml
     # ============[ REQUIRED ]============
@@ -105,16 +137,16 @@ updated going forward with new generations) in `sysconfig/phantom_subnets.toml`
             Generation = 1
             [[Networks.1.WeightedSubnets]]
                 Weight = 9
-                Subnets = ["192.122.190.0/24", "2001:0123:4567:89ab::/64"] 
+                Subnets = ["192.122.190.0/24", "2001:0123:4567:89ab::/64"]
 
         [Networks.2]
             Generation = 2
             [[Networks.2.WeightedSubnets]]
                 Weight = 9
-                Subnets = ["192.0.0.0/24", "2001:0123:4567:89ab::/64"] 
+                Subnets = ["192.0.0.0/24", "2001:0123:4567:89ab::/64"]
             [[Networks.2.WeightedSubnets]]
                 Weight = 1
-                Subnets = ["2001:0123:4567:89ab::/96"] 
+                Subnets = ["2001:0123:4567:89ab::/96"]
     ```
 
 ### Setup
@@ -155,7 +187,7 @@ sudo systemctl enable conjure-registration-server
 Start the station.
 
 ```sh
-# zbalance has to be first or the detector will throw an error 
+# zbalance has to be first or the detector will throw an error
 systemctl start zbalance
 
 # Next start the detector and station application processes
@@ -166,4 +198,4 @@ systemctl start conjure-app
 systemctl start conjure-registration-server
 ```
 
-## [FAQ](https://github.com/refraction-networking/conjure/wiki/FAQ) | [WIKI](https://github.com/refraction-networking/conjure/wiki) 
+## [FAQ](https://github.com/refraction-networking/conjure/wiki/FAQ) | [WIKI](https://github.com/refraction-networking/conjure/wiki)

application/lib/config_test.go

@@ -11,7 +11,7 @@ import (
 // TestConfigParse double checks to ensure that the identity struct reflection
 // trick works and that the fields are accessible.
 func TestConfigParse(t *testing.T) {
-	os.Setenv("CJ_STATION_CONFIG", "../config.toml")
+	os.Setenv("CJ_STATION_CONFIG", "../app_config.toml")
 
 	var c Config
 	_, err := toml.DecodeFile(os.Getenv("CJ_STATION_CONFIG"), &c)

cmd/registration-server/config.toml → cmd/registration-server/reg_config.toml

@@ -5,7 +5,7 @@ dns_listen_addr = "[::]:53"
 domain = "r.refraction.network"
 
 # Path to Conjure private key file
-dns_private_key_path = "/opt/conjure/sysconfig/privkey"
+dns_private_key_path = "/var/lib/conjure/privkey"
 
 # Log level, one of the following: panic, fatal, error, warn, info, debug, trace
 log_level = "info"
@@ -21,7 +21,7 @@ zmq_port = 5591
 zmq_bind_addr = "*"
 
 # The path on disk to the private key used for the ZMQ socket
-zmq_privkey_path = "/opt/conjure/sysconfig/privkey"
+zmq_privkey_path = "/var/lib/conjure/privkey"
 
 # The type of authentication to use on the ZMQ socket.
 # Can be set to "NULL" which allows all connections, or "CURVE"
@@ -44,4 +44,4 @@ station_pubkeys = [
 bidirectional_api_generation = 957
 
 # Path on disk to the latest ClientConfig file that the station should use
-clientconf_path = "/opt/conjure/sysconfig/ClientConf"
+clientconf_path = "/var/lib/conjure/ClientConf"

scripts/start_zbalance_ipc.sh

@@ -34,12 +34,15 @@ check_ZC_driver() {
 	echo ''
 	sleep 10
 	exit 1;
-	fi	
+	fi
 }
 
 # Run zbalance. Needed for zero-copy mode Conjure
 # See README.md
+# load config. will access config in /var/lib/conjure for overrides
+set -a
 source /opt/conjure/sysconfig/conjure.conf
+set +a
 
 # CJ_IFACE could be a CSV list of interfaces.
 # Pull them apart to ensure each gets zc: prefix
@@ -53,7 +56,7 @@ do
         ifcelem=${ifc}
     fi
 
-    check_ZC_driver ${ifcelem} 
+    check_ZC_driver ${ifcelem}
 
     if [ $didfirst -ne 0 ]; then
         ifcarg="$ifcarg,$ifcelem"

src/process_packet.rs

@@ -382,7 +382,7 @@ mod tests {
 
     #[test]
     fn test_filter_station_traffic() {
-        env::set_var("CJ_STATION_CONFIG", "./application/config.toml");
+        env::set_var("CJ_STATION_CONFIG", "./application/app_config.toml");
 
         // --
         let conf_path = env::var("CJ_STATION_CONFIG").unwrap();

sysconfig/conjure-app.service

@@ -10,13 +10,13 @@ Type=simple
 WorkingDirectory=/opt/conjure/
 
 SyslogIdentifier=conjure
-EnvironmentFile=/opt/conjure/sysconfig/conjure.conf
+# EnvironmentFile=/opt/conjure/sysconfig/conjure.conf
 
 # # makes if binary doesn't exist
 # ExecStartPre=/bin/bash /opt/conjure/build.sh
 # Ensure zbalance finished starting (else pfring_zc_ipc_attach can give Invalid argument)
 ExecStartPre=/bin/sleep 10
-ExecStart=/opt/conjure/application/application
+ExecStart=/bin/bash /opt/conjure/scripts/start_application.sh
 
 # send SIGHUP to the station process
 ExecReload=/bin/kill -HUP $MAINPID

sysconfig/conjure-det.service

@@ -10,13 +10,13 @@ Type=simple
 WorkingDirectory=/opt/conjure/
 
 SyslogIdentifier=conjure
-EnvironmentFile=/opt/conjure/sysconfig/conjure.conf
+# EnvironmentFile=/opt/conjure/sysconfig/conjure.conf
 
 # # makes if binary doesn't exist
 # ExecStartPre=/bin/bash /opt/conjure/build.sh
 # Ensure zbalance finished starting (else pfring_zc_ipc_attach can give Invalid argument)
 ExecStartPre=/bin/sleep 10
-ExecStart=/opt/conjure/conjure -c ${CJ_CLUSTER_ID} -o ${CJ_COREBASE} -n ${CJ_CORECOUNT} -l ${CJ_LOG_INTERVAL} -K ${CJ_PRIVKEY} -s ${CJ_SKIP_CORE} -z ${CJ_QUEUE_OFFSET}
+ExecStart=/bin/bash /opt/conjure/scripts/start_detector.sh
 
 # on stop processes will get SIGTERM, and after 10 secs - SIGKILL (default 90)
 TimeoutStopSec=10

sysconfig/conjure-reg.service

@@ -7,9 +7,9 @@ Type=simple
 WorkingDirectory=/opt/conjure/cmd/registration-server/
 
 SyslogIdentifier=conjure
-EnvironmentFile=/opt/conjure/sysconfig/conjure.conf
+# EnvironmentFile=/opt/conjure/sysconfig/conjure.conf
 
-ExecStart=/opt/conjure/cmd/registration-server/registration-server --config /opt/conjure/cmd/registration-server/config.toml
+ExecStart=/bin/bash /opt/conjure/scripts/start_registrar.sh
 
 # send SIGHUP to the registration server process
 ExecReload=/bin/kill -HUP $MAINPID

sysconfig/conjure.conf

@@ -32,7 +32,7 @@ CJ_SKIP_CORE=
 CJ_LOG_INTERVAL=5
 
 # private key used by the detectors
-CJ_PRIVKEY=/opt/conjure/sysconfig/privkey
+CJ_PRIVKEY=/var/lib/conjure/privkey
 
 # if running on a host where we need to connect to zbalance queues starting at a non-zero index
 #    specify that index here. queue attachment will happen incrementally from this index up to
@@ -41,15 +41,15 @@ CJ_QUEUE_OFFSET=0
 
 # Path to the configuration file for the application and proxying portions of
 # the station. Used by the conjure-app service
-CJ_STATION_CONFIG=/opt/conjure/application/config.toml
+CJ_STATION_CONFIG=/var/lib/conjure/app_config.toml
 
 # Path to the configuration file for the registration api. Used by the conjure-reg service
-CJ_REGISTRAR_CONFIG=/opt/conjure/cmd/registration-server/config.toml
+CJ_REGISTRAR_CONFIG=/var/lib/conjure/reg_config.toml
 
 # Path to the configuration file containing phantom subnets. used primarily by
 # the conjure-app service to figure out what subnets to choose from for new
 # registrations.
-PHANTOM_SUBNET_LOCATION=/opt/conjure/sysconfig/phantom_subnets.toml
+PHANTOM_SUBNET_LOCATION=/var/lib/conjure/phantom_subnets.toml
 
 # Allow the station to log client IPs (default disabled)
 LOG_CLIENT_IP=false
@@ -172,4 +172,4 @@ if [ -r /var/lib/conjure/conjure.conf ] && [ -z "${CJ_STOP_RECURSE+x}" ]; then
     . /var/lib/conjure/conjure.conf
 fi
 
-unset CJ_STOP_RECURSE
+unset CJ_STOP_RECURSE

sysconfig/zbalance.service

@@ -9,7 +9,7 @@ Type=simple
 WorkingDirectory=/opt/conjure/
 
 SyslogIdentifier=zbalance
-EnvironmentFile=/opt/conjure/sysconfig/conjure.conf
+# EnvironmentFile=/opt/conjure/sysconfig/conjure.conf
 
 # makes if binary doesn't exist
 #ExecStartPre=/usr/bin/make zbalance