diff --git a/.dockerignore b/.dockerignore index fcb07f0b..f587fb3e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,5 @@ -docker -!docker/* -docker/Dockerfile -docker/docker-compose* -docker/.dockerignore -docker/*.swp -.dockerignore +.git *.swp +/docker/** +!docker/*entrypoint.sh +.dockerignore diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 67cc16b4..9e8c46fc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,8 @@ name: CI-build on: push: branches: [ master, staging ] + pull_request: + branches: [ master, staging ] # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: @@ -29,24 +31,22 @@ jobs: GO_VERSION: 1.15.4 # RUSTVERSION: 1.47.0 run: | - echo "HOME=$HOME" - export GOPATH=`pwd`/go - echo "GOPATH=$GOPATH" - cd go/src/github.com/refraction-networking/conjure - # Apt deps sudo apt-get update - sudo apt-get install protobuf-compiler gcc curl git wget -y -q + sudo apt-get install protobuf-compiler gcc curl git wget software-properties-common -y -q sudo apt-get install libzmq3-dev libssl-dev pkg-config libgmp3-dev -y -q + sudo add-apt-repository universe + wget https://packages.ntop.org/apt-stable/18.04/all/apt-ntop-stable.deb + sudo apt-get install ./apt-ntop-stable.deb + sudo apt-get update + sudo apt-get install pfring echo "Apt dependencies installed" echo "--------------------------------------" - # Build PF_Ring libraries - cd $GOPATH/src/github.com/refraction-networking/conjure/PF_RING/userland/lib && ./configure && make - cd $GOPATH/src/github.com/refraction-networking/conjure/PF_RING/userland/libpcap && ./configure && make - cd $GOPATH/src/github.com/refraction-networking/conjure - echo "PF_Ring libraries successfully built" - echo "--------------------------------------" + echo "HOME=$HOME" + export GOPATH=`pwd`/go + echo "GOPATH=$GOPATH" + cd go/src/github.com/refraction-networking/conjure # Install rust curl https://sh.rustup.rs -sSf -o install_rust.sh; sh install_rust.sh -y; @@ -66,6 +66,7 @@ jobs: # temp fix before transition to redis v8 go get -u github.com/go-redis/redis || true && cd $GOPATH/src/github.com/go-redis/redis && git checkout tags/v7.4.0 -b v7-master && cd - go get -u github.com/BurntSushi/toml || true + go get -u github.com/gorilla/mux || true go get -d -u -t github.com/refraction-networking/gotapdance/... || true go get -u github.com/refraction-networking/conjure/application/... || true go get -u github.com/refraction-networking/conjure/registration-api/... || true @@ -77,7 +78,7 @@ jobs: make echo "Station successfully built" mkdir -p $GITHUB_WORKSPACE/bin - cp dark-decoy application/application registration-api/registration-api $GITHUB_WORKSPACE/bin + cp conjure application/application registration-api/registration-api $GITHUB_WORKSPACE/bin cd $GITHUB_WORKSPACE && tar -czf conjure-station.tar.gz bin diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index d72c7b12..00000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "PF_RING"] - branch = 7.4.0-stable - path = PF_RING - url = https://github.com/ntop/PF_RING.git diff --git a/Makefile b/Makefile index eed737ff..d7badb38 100644 --- a/Makefile +++ b/Makefile @@ -3,16 +3,14 @@ CC = ${CROSS_COMPILE}gcc #--platform=native DEBUG_OR_RELEASE = release -PFRINGDIR=./PF_RING/ -PFRING_LIBS=${PFRINGDIR}/userland/lib/libpfring.a ${PFRINGDIR}/userland/libpcap/libpcap.a RUST_LIB=./target/release/librust_dark_decoy.a TD_LIB=./libtapdance/libtapdance.a -LIBS=${PFRING_LIBS} ${RUST_LIB} ${TD_LIB} -L/usr/local/lib -lzmq -lcrypto -lpthread -lrt -lgmp -ldl -lm -CFLAGS = -Wall -DENABLE_BPF -DHAVE_PF_RING -DHAVE_PF_RING_ZC -DTAPDANCE_USE_PF_RING_ZERO_COPY -I${PFRINGDIR}/userland/lib/ -I${PFRINGDIR}/kernel -O2 # -g +LIBS=${RUST_LIB} ${TD_LIB} -L/usr/local/lib -lpcap -lpfring -lzmq -lcrypto -lpthread -lrt -lgmp -ldl -lm +CFLAGS = -Wall -DENABLE_BPF -DHAVE_PF_RING -DHAVE_PF_RING_ZC -DTAPDANCE_USE_PF_RING_ZERO_COPY -O2 # -g PROTO_RS_PATH=src/signalling.rs -all: rust libtd dark-decoy app registration-api ${PROTO_RS_PATH} +all: rust libtd conjure app registration-api ${PROTO_RS_PATH} rust: ./src/*.rs cargo build --${DEBUG_OR_RELEASE} @@ -26,8 +24,9 @@ app: libtd: cd ./libtapdance/ && make libtapdance.a -dark-decoy: detect.c loadkey.c rust_util.c rust libtapdance +conjure: detect.c loadkey.c rust_util.c rust libtapdance ${CC} ${CFLAGS} -o $@ detect.c loadkey.c rust_util.c ${LIBS} +# gcc -Wall -DENABLE_BPF -DHAVE_PF_RING -DHAVE_PF_RING_ZC -DTAPDANCE_USE_PF_RING_ZERO_COPY -O2 -o conjure detect.c loadkey.c rust_util.c ./target/release/librust_dark_decoy.a ./libtapdance/libtapdance.a -lpfring -lpcap -L/usr/local/lib -lzmq -lcrypto -lpthread -lrt -lgmp -ldl -lm registration-api: cd ./registration-api/ && make diff --git a/PF_RING b/PF_RING deleted file mode 160000 index c5146860..00000000 --- a/PF_RING +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c5146860f7ecc00d765af9a7108602bf1b8b5c54 diff --git a/README.md b/README.md index 4b5d5a21..9fbb5ff2 100644 --- a/README.md +++ b/README.md @@ -1,70 +1,172 @@ -# dark-decoys +

+refract +

Conjure Refraction Station

+ +

Refraction Networking is a free-to-use anti-censorship technology, that places proxies at Internet Service Providers, so they are harder to block. This repository implements the conjure stations system including various registration channels, transport protocols, and configuration options.

+

+ + + +

+ +### See also + +[Refraction Client Library](https://github.com/refraction-networking/gotapdance) - +pure golang client library for connecting to refraction systems. Implements BOTH +conjure and tapdance. + +[Tapdance Station](https://github.com/refraction-networking/tapdance) - Tapdance +station code implementing the previous iteration of refraction networking development. ## Install -### Install dependencies +This is an abridged install and configuration process, for expanded install instructions, +configuration options, or multi-station deployments see [the wiki](https://github.com/refraction-networking/conjure/wiki). + + +### Requirements + +Building the station requires both go and rust: + +- [Install Golang](https://golang.org/doc/install) + +- [Install Rust](https://www.rust-lang.org/tools/install) + +**Install packages and go libraries** ```sh -sudo apt install libzmq3-dev redis-server +sudo apt install wget git make gcc bison flex protobuf-compiler curl libssl-dev pkg-config libgmp3-dev libzmq3-dev go get -d -u -t github.com/refraction-networking/gotapdance/... -go get -d github.com/go-redis/redis +# Due to compatibility issue with redis v8 -- checkout the stable 7.4.0 go library version. +go get -d github.com/go-redis/redis +cd ${GOPATH}/src/github.com/go-redis/redis && git checkout tags/v7.4.0 -b v7-master ``` -### Install PF_RING +**Install PF_RING** -1. [Install PF_RING kernel module](https://www.ntop.org/guides/pf_ring/get_started/git_installation.html#kernel-module-installation) +1. [Install from Package](https://github.com/refraction-networking/conjure/wiki/PF_RING#from-packages) -2. [Install PF_RING Libpfring and Libpcap](https://www.ntop.org/guides/pf_ring/get_started/git_installation.html#libpfring-and-libpcap-installation) +2. [Install From Git / Source](https://github.com/refraction-networking/conjure/wiki/PF_RING#from-source) ### Build the station ```sh make -#future +## future # sudo make install ``` -## Setup +### Configure -```sh -# Add prerouting rules for ipv4 and ipv6 to Destination NAT (DNAT) to change destination IP addr -sudo iptables -t nat -I PREROUTING 1 -p tcp -i tun0 -j DNAT --to 192.122.200.231:41245 -sudo ip6tables -t nat -I PREROUTING 1 -p tcp -i tun0 -j DNAT --to [2001:48a8:687f:2::2]:41245 +To run a station configuration modifications are required. This section outlines +some minimal changes, for more configuration options see the [wiki configuration page](https://github.com/refraction-networking/conjure/wiki/Configuration). + +1. Define global paths, core usage, and pf_ring parameters in `sysconfig/conjure.conf` + + ```conf + # ============[ REQUIRED ]============ + # The interface(s) which PF_RING Zero Copy will tap. + CJ_IFACE="zc:enp179s0f0,zc:enp179s0f1" + + Public addresses that of non-tap interface - used for kernel DNAT + IP4_ADDR="" + IP6_ADDR="" + + ``` -# Disable Reverse_Path Filtering so that we may deal with spoofed addresses. -sudo sysctl -w net.ipv4.conf.tun0.rp_filter=0 + Note: ipv6 in disabled by default. To enable IPv6 modify + `application/config.toml` + ```diff + # Allow the station to opt out of either version of internet protocol to limit a + # station to handling one or the other. For example, v6 on small station deployment + # with only v6 phantom subnet, v4 only on station with no puvlic v6 address. + enable_v4 = true + -enable_v6 = false + +enable_v6 = true + ``` -# Make sure INPUT iptables has an accept for `tun0` (isn't blocking packets after they're DNAT'd) -sudo iptables -I INPUT 1 -i tun0 -j ACCEPT -sudo ip6tables -I INPUT 1 -i tun0 -j ACCEPT +2. Define application parameters in `application/congfig.toml` + ```toml + # ============[ REQUIRED ]============ + ## Detector + [[connect_sockets]] -# Repeat for `tun1`/`tun2`/`tun3`/etc... if you have multiple cores -# ... + address = "ipc://@detector" + type = "NULL" + + ``` + +3. Define the phantom subnet generations that will be supported (this can be +updated going forward with new generations) in `sysconfig/phantom_subnets.toml` + + ```toml + [Networks] + [Networks.1] + Generation = 1 + [[Networks.1.WeightedSubnets]] + Weight = 9 + Subnets = ["192.122.190.0/24", "2001:0123:4567:89ab::/64"] + + [Networks.2] + Generation = 2 + [[Networks.2.WeightedSubnets]] + Weight = 9 + Subnets = ["192.0.0.0/24", "2001:0123:4567:89ab::/64"] + [[Networks.2.WeightedSubnets]] + Weight = 1 + Subnets = ["2001:0123:4567:89ab::/96"] + ``` + +### Setup + +Conjure relies on the kernel to handle provide DNAT to establish these rules we +need to configure and run the environment configuration script. + +After defining `IP4_ADDR`, `IP6_ADDR`, and core usage parameters in the +`conjure.conf` file run the `on-reboot.sh` script to initialize all required +interfaces and rules. + +```sh +./on-reboot.sh ``` -## Run +Generate station keys using the libtapdance tools + +```ssh +cd libtapdance && make genkey +./libtapdance/genkey +mv libtapdance/{priv,pub}key sysconfig/ +``` + +### Run + +Copy (or link) the systemd service configurations to the appropriate location ```sh -> dark_decoy -c -i zc: [opts] +sudo cp sysconfig/*.service /etc/systemd/system/ +sudo systemctl enable zbalance +sudo systemctl enable conjure-app +sudo systemctl enable conjure-det -# Options +# if enabling and supporting registration api or multi-station registration sharing +sudo systemctl enable conjure-registration-api +``` + +Start the station. -# REQUIRED -# -c - The cluster id specified to PF_RING when starting `zbalance_ipc` -# -i - The interface on which PF_RING Zero Copy is running. +```sh +# zbalance has to be first or the detector will throw an error +systemctl start zbalance -# OPTIONAL -# -n - Number of cores to be used (default -1 = all cores) -# -s - specify core_id to be skipped when allocating threads. -# -K - Specify custom private key to be used by station -# -a - Custom Address of ZMQ server +# Next start the detector and station application processes +systemctl start conjure-det +systemctl start conjure-app -# DEBUG -# -l - In seconds, interval between logging of bandwidth, tag checks/s, etc. -# -o - Start processes on $core_affinity_offset+$cpu_procs. -# This allows us to run debug/production pf_rings on different cores -# entirely (which rust likes), and with different cluster_ids. +# if enabling and supporting registration api or multi-station registration sharing +systemctl start conjure-registration-api ``` + +## [FAQ](https://github.com/refraction-networking/conjure/wiki/FAQ) | [WIKI](https://github.com/refraction-networking/conjure/wiki) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8b51e2a4..df9938a1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,57 +1,69 @@ -FROM ubuntu:20.04 as build_base +#FROM ubuntu:20.04 as build_base +FROM ubuntu:20.04 as build_base_go # PATH="/opt/PF_RING/userland/examples_zc:$PATH" - -#wget make gcc bison flex -RUN apt-get update && \ - apt-get -y -q install wget git make gcc bison flex && \ - git clone --recurse-submodule https://github.com/refraction-networking/conjure.git && \ - cd /conjure/PF_RING/userland/lib && ./configure && make && \ - cd /conjure/PF_RING/userland/libpcap && ./configure && make && \ - cd /conjure/PF_RING/userland && ./configure && make && \ - cp -r /conjure/PF_RING /opt/PF_RING && \ - apt-get clean all - -FROM build_base as build_base_go ARG GO_VERSION=1.15.3 ARG CUSTOM_BUILD +ARG BRANCH=master ENV PATH="/usr/local/go/bin:/root/.cargo/bin:${PATH}" \ GOPATH="/root/go" \ GOROOT="/usr/local/go" -COPY . /tmp/conjure - -# Install rust and go -RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get -y -q install protobuf-compiler curl libssl-dev pkg-config libgmp3-dev libzmq3-dev && \ +# Install dependencies: including rust and go +RUN apt-get update && \ + DEBIAN_FRONTEND="noninteractive" apt-get -y -q install wget git make gcc bison flex protobuf-compiler curl libssl-dev pkg-config libgmp3-dev libzmq3-dev && \ + apt-get clean all && \ wget -q https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz && \ tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz && \ curl https://sh.rustup.rs -sSf -o install_rust.sh; sh install_rust.sh -y && \ cargo install protobuf-codegen +# Install PFRING to get libraries +RUN apt-get install -y -q software-properties-common wget && \ + add-apt-repository universe && \ + wget https://packages.ntop.org/apt-stable/20.04/all/apt-ntop-stable.deb && \ + apt install ./apt-ntop-stable.deb && \ + apt-get clean all && \ + apt-get update && \ + apt-get install -y -q pfring && \ + apt-get clean all + +# Get go-reddis and make sure it's version is 7.4.0. RUN go get -u -d github.com/go-redis/redis || true && cd ${GOPATH}/src/github.com/go-redis/redis && git checkout tags/v7.4.0 -b v7-master -RUN go get -u -d github.com/BurntSushi/toml + +# Copy docker context dir. This is used as a source if CUSTOM_BUILD is enabled +COPY . /tmp/conjure +# Get Conjure or copy a directory Dockerfile is in. Switched by CUSTOM_BUILD var RUN bash -c 'if [[ -z "$CUSTOM_BUILD" ]] ; then \ go get -d github.com/refraction-networking/conjure/... ; \ else mkdir -p ${GOPATH}/src/github.com/refraction-networking; cp -r /tmp/conjure ${GOPATH}/src/github.com/refraction-networking/conjure ; \ fi' -RUN rm -rf ${GOPATH}/src/github.com/refracion-networking/conjure/PF_RING && cp -r /conjure/PF_RING ${GOPATH}/src/github.com/refraction-networking/conjure + +# Checkout needed branch and compile RUN cd /root/go/src/github.com/refraction-networking/conjure && \ + git checkout ${BRANCH} && \ go get ./... || true && \ make RUN cp -r /root/go/src/github.com/refraction-networking/conjure /opt/conjure - FROM ubuntu:20.04 as zbalance ENV CJ_IFACE=lo \ CJ_CLUSTER_ID=98 \ CJ_CORECOUNT=1 \ CJ_COREBASE=0 \ ZBALANCE_HASH_MODE=1 -COPY --from=build_base /opt/PF_RING /opt/PF_RING +#COPY --from=build_base /opt/PF_RING /opt/PF_RING + +RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get -y -q install libelf1 + +COPY --from=build_base_go /usr/bin/zbalance_ipc /usr/bin/zbalance_ipc COPY ./docker/zbalance-entrypoint.sh /entrypoint.sh ENTRYPOINT ["bash", "/entrypoint.sh"] + + + FROM ubuntu:20.04 as detector ENV CJ_CLUSTER_ID=98 \ CJ_CORECOUNT=1 \ @@ -63,20 +75,23 @@ ENV CJ_CLUSTER_ID=98 \ CJ_STATION_CONFIG=/opt/conjure/application/config.toml \ CJ_IP4_ADDR=127.0.0.1 \ CJ_IP6_ADDR=[::1] -COPY --from=build_base_go /opt/conjure/dark-decoy /opt/conjure/dark-decoy +#COPY --from=build_base_go /opt/conjure/dark-decoy /opt/conjure/dark-decoy +COPY --from=build_base_go /opt/conjure/conjure /opt/conjure/conjure COPY --from=build_base_go /opt/conjure/application/config.toml /opt/conjure/application/config.toml COPY ./docker/detector-entrypoint.sh /entrypoint.sh -# this list will be removed in a near future -RUN touch /var/lib/dark-decoy.prefixes +COPY --from=build_base_go /usr/local/lib/libpcap.so /usr/local/lib/libpcap.so RUN apt-get update && apt-get -y -q install libzmq3-dev iproute2 iptables && apt-get clean all ENTRYPOINT [ "/entrypoint.sh"] + + + FROM ubuntu:20.04 as application ENV CJ_STATION_CONFIG=/opt/conjure/application/config.toml \ PHANTOM_SUBNET_LOCATION=/opt/conjure/sysconfig/phantom_subnets.toml COPY --from=build_base_go /opt/conjure/application/application /opt/conjure/application/application -RUN apt-get update && apt-get -y -q install libzmq3-dev +RUN apt-get update && apt-get -y -q install libzmq3-dev && apt-get clean all COPY --from=build_base_go /opt/conjure/application/config.toml ${CJ_STATION_CONFIG} COPY --from=build_base_go /opt/conjure/application/lib/test/phantom_subnets.toml ${PHANTOM_SUBNET_LOCATION} #COPY ./docker/application-entrypoint.sh /entrypoint.sh diff --git a/docker/detector-entrypoint.sh b/docker/detector-entrypoint.sh index 4e68aea0..06924cee 100755 --- a/docker/detector-entrypoint.sh +++ b/docker/detector-entrypoint.sh @@ -5,7 +5,7 @@ OFFSET=${CJ_QUEUE_OFFSET:-2} cleanup() { echo $(ps aux) - start-stop-daemon --stop --oknodo --retry 15 -n dark-decoy + start-stop-daemon --stop --oknodo --retry 15 -n conjure #pkill dark-decoy echo $(ps aux) for CORE in `seq $OFFSET $((OFFSET + CORE_COUNT -1 ))` @@ -65,7 +65,7 @@ do fi done echo "Prerequisite configuration complete." -/opt/conjure/dark-decoy -c ${CJ_CLUSTER_ID} -o ${CJ_COREBASE} -n ${CJ_CORECOUNT} -l ${CJ_LOG_INTERVAL} -K ${CJ_PRIVKEY} -s ${CJ_SKIP_CORE} -z ${CJ_QUEUE_OFFSET} & +/opt/conjure/conjure -c ${CJ_CLUSTER_ID} -o ${CJ_COREBASE} -n ${CJ_CORECOUNT} -l ${CJ_LOG_INTERVAL} -K ${CJ_PRIVKEY} -s ${CJ_SKIP_CORE} -z ${CJ_QUEUE_OFFSET} & wait $! cleanup diff --git a/docker/install_pfring_package.sh b/docker/install_pfring_package.sh new file mode 100755 index 00000000..f1bc3f12 --- /dev/null +++ b/docker/install_pfring_package.sh @@ -0,0 +1,71 @@ +#!/bin/bash +OS=$(lsb_release -si) +RELEASE=$(lsb_release -sr) +CODENAME=$(lsb_release -sc) + +function no_support { + echo "Script does not support selected parameters or OS ${OS} ${CODENAME} ${RELEASE}." + echo "Check https://packages.ntop.org/apt-stable/ for more information." + exit 1 +} + +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" + exit 1 +fi + +case "$OS" in + Ubuntu) + echo "OS: Ubuntu" + case $RELEASE in + 20.04 | 18.04) + apt-get install software-properties-common wget + add-apt-repository universe + wget https://packages.ntop.org/apt-stable/${RELEASE}/all/apt-ntop-stable.deb + apt install ./apt-ntop-stable.deb + ;; + 16.04) + wget https://packages.ntop.org/apt-stable/${RELEASE}/all/apt-ntop-stable.deb + apt install ./apt-ntop-stable.deb + ;; + *) + no_support + ;; + esac + ;; + Debian) + echo "OS: Debian" + case $CODENAME in + buster | stretch) + echo ${CODENAME} requires 'contrib' apt sources. Do you want to enable? + read -p "[N/y]" enable_contrib + if [ ${enable_contrib:-N} = 'y' ] + then + echo "Enabling 'contrib' sources in /etc/apt/sources.list" + sed -i.bak -e '/contrib/ ! s/^deb.*debian\.org.*$/\0 contrib/' /etc/apt/sources.list + wget https://packages.ntop.org/apt-stable/${CODENAME}/all/apt-ntop-stable.deb + apt install ./apt-ntop-stable.deb + else + no_support + fi + ;; + jessie) + wget https://packages.ntop.org/apt-stable/jessie/all/apt-ntop-stable.deb + dpkg -i apt-ntop-stable.deb + echo "deb http://archive.debian.org/debian jessie-backports main" >> /etc/apt/sources.list + echo 'Acquire::Check-Valid-Until no;' > /etc/apt/apt.conf.d/99no-check-valid-until + apt-get update && apt-get install libjson-c2 + ;; + *) + no_support + ;; + esac + ;; + *) + no_support + ;; +esac +apt-get clean all +apt-get update +apt-get install pfring-dkms nprobe ntopng n2disk cento +apt-get install pfring-drivers-zc-dkms diff --git a/docker/phantom_subnets.toml b/docker/phantom_subnets.toml index 077af9f3..fd2a5ead 100644 --- a/docker/phantom_subnets.toml +++ b/docker/phantom_subnets.toml @@ -4,20 +4,20 @@ [Networks.1] Generation = 1 [[Networks.1.WeightedSubnets]] - Weight = 9.0 + Weight = 9 Subnets = ["192.122.190.0/24", "2001:48a8:687f:1::/64"] [Networks.2] Generation = 2 [[Networks.2.WeightedSubnets]] - Weight = 1.0 + Weight = 1 Subnets = ["192.122.190.0/28", "2001:48a8:687f:1::/96"] [Networks.957] Generation = 957 [[Networks.957.WeightedSubnets]] - Weight = 9.0 + Weight = 9 Subnets = ["192.122.190.0/24", "2001:48a8:687f:1::/64"] [[Networks.957.WeightedSubnets]] - Weight = 1.0 + Weight = 1 Subnets = ["141.219.0.0/16", "35.8.0.0/16"] diff --git a/docker/zbalance-entrypoint.sh b/docker/zbalance-entrypoint.sh index b8d62619..00f16656 100755 --- a/docker/zbalance-entrypoint.sh +++ b/docker/zbalance-entrypoint.sh @@ -1,6 +1,39 @@ #!/bin/bash set -e + + +while [ $(sysctl -b vm.nr_hugepages) -lt 512 ] +do + echo 'Please set number of hugepages to at least 512.' + echo '' + echo 'To check current value run:' + echo ' sysctl vm.nr_hugepages' + echo 'OR' + echo ' cat /proc/sys/vm/nr_hugepages' + echo '' + echo 'To set number of hugepages run:' + echo ' sysctl -w vm.nr_hugepages=512' + echo '' + echo 'To make this setting persistent run:' + echo ' echo "vm.nr_hugepages=512" >> /etc/sysctl.conf' + echo '' + echo '' + echo 'Sleeping for 10 seconds' + sleep 10 +done + +while [ ! $(cat "/proc/net/pf_ring/dev/${CJ_IFACE}/info" | grep ZC) ] +do + echo 'Is ZC network drivers loaded? For instructions visit https://www.ntop.org/guides/pf_ring/get_started/packages_installation.html' + echo '' + echo 'To check for ZC driver run:' + echo ' cat /proc/net/pf_ring/dev/'${CJ_IFACE}'/info' + echo 'You should see "Polling Mode: ZC/NAPI"' + echo '' + sleep 10; +done + # TD_IFACE could be a CSV list of interfaces. # Pull them apart to ensure each gets zc: prefix ifcarg="" @@ -20,4 +53,4 @@ do fi done echo "Setting up with params: -i $ifcarg -c ${CJ_CLUSTER_ID} -n ${CJ_CORECOUNT} -m ${ZBALANCE_HASH_MODE} -g ${CJ_COREBASE}" -/opt/PF_RING/userland/examples_zc/zbalance_ipc -i $ifcarg -c ${CJ_CLUSTER_ID} -n ${CJ_CORECOUNT} -m ${ZBALANCE_HASH_MODE} -g ${CJ_COREBASE} +zbalance_ipc -i $ifcarg -c ${CJ_CLUSTER_ID} -n ${CJ_CORECOUNT} -m ${ZBALANCE_HASH_MODE} -g ${CJ_COREBASE} diff --git a/on-reboot.sh b/on-reboot.sh index 432e3604..4751dfc3 100755 --- a/on-reboot.sh +++ b/on-reboot.sh @@ -3,12 +3,9 @@ +source $CJ_PATH/sysconfig/conjure.conf -# SET THESE FOR EACH DIFFERENT STATION -IP4_ADDR="192.122.200.253" -IP6_ADDR="[fe80::225:90ff:fef9:6fb4]" -CORE_COUNT=6 -OFFSET=6 +CORE_COUNT=$CJ_CORECOUNT do_or_die() { $1 || exit_msg "command \"$1\" failed" diff --git a/sysconfig/conjure-det.service b/sysconfig/conjure-det.service index 2e230239..367c53c4 100644 --- a/sysconfig/conjure-det.service +++ b/sysconfig/conjure-det.service @@ -16,7 +16,7 @@ EnvironmentFile=/opt/conjure/sysconfig/conjure.conf # ExecStartPre=/bin/bash /opt/conjure/build.sh # Ensure zbalance finished starting (else pfring_zc_ipc_attach can give Invalid argument) ExecStartPre=/bin/sleep 10 -ExecStart=/opt/conjure/dark-decoy -c ${CJ_CLUSTER_ID} -o ${CJ_COREBASE} -n ${CJ_CORECOUNT} -l ${CJ_LOG_INTERVAL} -K ${CJ_PRIVKEY} -s ${CJ_SKIP_CORE} -z ${CJ_QUEUE_OFFSET} +ExecStart=/opt/conjure/conjure -c ${CJ_CLUSTER_ID} -o ${CJ_COREBASE} -n ${CJ_CORECOUNT} -l ${CJ_LOG_INTERVAL} -K ${CJ_PRIVKEY} -s ${CJ_SKIP_CORE} -z ${CJ_QUEUE_OFFSET} # on stop processes will get SIGTERM, and after 10 secs - SIGKILL (default 90) TimeoutStopSec=10 diff --git a/sysconfig/conjure.conf b/sysconfig/conjure.conf index 4fdb058c..a2e6eb80 100644 --- a/sysconfig/conjure.conf +++ b/sysconfig/conjure.conf @@ -55,3 +55,19 @@ ZBALANCE_CORE=1 # set this to the number of bytes the Ether+IP+GRE headers are for # that instance (e.g. ERSPAN on rockypika should be 42) PARSE_GRE_OFFSET=0 + + +CJ_PATH=/opt/conjure/ + +# ========================================================== # +# Local Network settings SET THESE FOR EACH DIFFERENT STATION + +# IPv4 address of non-tap local interface +IP4_ADDR="127.0.0.1" + +# IPv6 address of non-tap local interface. +IP6_ADDR="[::1]" + +# tun interface offset (used when running multiple station on the same machine) +OFFSET=0 + diff --git a/sysconfig/start_zbalance_ipc.sh b/sysconfig/start_zbalance_ipc.sh index 3b1ddb7b..ce454414 100755 --- a/sysconfig/start_zbalance_ipc.sh +++ b/sysconfig/start_zbalance_ipc.sh @@ -24,4 +24,16 @@ do didfirst=1 fi done -sudo ./PF_RING/userland/examples_zc/zbalance_ipc -i $ifcarg -c ${CJ_CLUSTER_ID} -n ${CJ_CORECOUNT} -m ${ZBALANCE_HASH_MODE} -g ${ZBALANCE_CORE} + +# PF_Ring should be installed and zbalance_ipc should be available in your path +if ! command -v zbalance_ipc &> /dev/null +then + echo "zbalance_ipc could not be found," + echo "please ensure pf_ring is instaled and zbalance_ipc is in you PATH." + echo "Installing from package automtically installs executable scripts" + echo "see https://www.ntop.org/guides/pf_ring/get_started/packages_installation.html" + exit +fi + + +sudo zbalance_ipc -i $ifcarg -c ${CJ_CLUSTER_ID} -n ${CJ_CORECOUNT} -m ${ZBALANCE_HASH_MODE} -g ${ZBALANCE_CORE}