Standalone Egress NAT

Signed-off-by: Patryk Strusiewicz-Surmacki <[email protected]>

Co-authored-by: Tomoya Terashima <[email protected]>

.github/workflows/ci.yaml

@@ -13,12 +13,16 @@ env:
 jobs:
   test:
     name: Small test
+    strategy:
+      matrix:
+        test-ipam: ["true", "false"]
     runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-go@v5
       with:
         go-version: ${{ env.go-version }}
+        cache-dependency-path: "**/go.sum"
     - name: Cache tools
       id: cache-tools
       uses: actions/cache@v4
@@ -29,7 +33,7 @@ jobs:
         key: cache-${{ env.cache-version }}-go-${{ env.go-version }}-${{ hashFiles('v2/Makefile') }}
     - run: make setup
       if: steps.cache-tools.outputs.cache-hit != 'true'
-    - run: make test
+    - run: make test TEST_IPAM=${{ matrix.test-ipam }} TEST_EGRESS=true
     - run: make test-nodenet
       timeout-minutes: 10
     - run: make test-founat
@@ -40,35 +44,48 @@ jobs:
     strategy:
       matrix:
         kindest-node: ["1.28.15", "1.29.12", "1.30.8"]
-        ip-version: ["ipv4", "ipv6"]
+        with-ipam: ["false", "true"]
+        ipv6: ["false", "true"]
     runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-go@v5
       with:
         go-version: ${{ env.go-version }}
+        cache-dependency-path: "**/go.sum"
     - run: make image
     - run: make certs
     - name: Enable docker IPv6 mode
-      if: matrix.ip-version == 'ipv6'
+      if: matrix.ipv6 == 'true'
       working-directory: v2/e2e
       run: |
         sudo mkdir -p /etc/docker
         sudo cp daemon.json /etc/docker/daemon.json
         sudo systemctl restart docker.service
         sleep 10
         echo TEST_IPV6=true >> $GITHUB_ENV
-    - run: make start KUBERNETES_VERSION=${{ matrix.kindest-node }}
+    - run: make start KUBERNETES_VERSION=${{ matrix.kindest-node }} WITH_KINDNET=false TEST_IPV6=${{ matrix.ipv6 }}
+      if: matrix.with-ipam == 'true'
+      working-directory: v2/e2e
+    - run: make start KUBERNETES_VERSION=${{ matrix.kindest-node }} WITH_KINDNET=true TEST_IPV6=${{ matrix.ipv6 }}
+      if: matrix.with-ipam == 'false'
       working-directory: v2/e2e
     - run: make install-coil
+      if: matrix.with-ipam == 'true'
+      working-directory: v2/e2e
+    - run: make install-coil-egress-v4
+      if: matrix.with-ipam == 'false' && matrix.ipv6 == 'false'
+      working-directory: v2/e2e
+    - run: make install-coil-egress-v6
+      if: matrix.with-ipam == 'false' && matrix.ipv6 == 'true'
       working-directory: v2/e2e
-    - run: make test
+    - run: make test TEST_IPAM=${{ matrix.with-ipam }} TEST_EGRESS=true TEST_IPV6=${{ matrix.ipv6 }}
       working-directory: v2/e2e
     - run: make logs
       working-directory: v2/e2e
       if: always()
     - uses: actions/upload-artifact@v4
       if: always()
       with:
-        name: logs-${{ matrix.ip-version }}-${{ matrix.kindest-node }}.tar.gz
+        name: logs-ipv6-${{ matrix.ipv6 }}-with-ipam-${{ matrix.with-ipam }}-${{ matrix.kindest-node }}.tar.gz
         path: v2/e2e/logs.tar.gz

docs/cmd-coil-egress-controller.md

@@ -0,0 +1,27 @@
+coil-egress-controller
+===============
+
+`coil-egress-controller` is a Kubernetes controller for Coil custom resources related to on-demand NAT egress.
+It is intended to be run as a Pod in `kube-system` namespace.
+
+
+## Egress
+
+`coil-egress-controller` creates **Deployment** and **Service** for each Egress.
+
+It also creates `coil-egress` **ServiceAccount** in the namespace of Egress,
+and binds it to the **ClusterRoles** for `coil-egress`.
+
+## Command-line flags
+
+```
+Flags:
+      --cert-dir string        directory to locate TLS certs for webhook (default "/certs")
+      --egress-port int32      UDP port number used by coil-egress (default 5555)
+      --gc-interval duration   garbage collection interval (default 1h0m0s)
+      --health-addr string     bind address of health/readiness probes (default ":9387")
+  -h, --help                   help for coil-egress-controller
+      --metrics-addr string    bind address of metrics endpoint (default ":9386")
+  -v, --version                version for coil-egress-controller
+      --webhook-addr string    bind address of admission webhook (default ":9443")
+```

docs/cmd-coil-controller.md → docs/cmd-coil-ipam-controller.md

@@ -1,41 +1,33 @@
-coil-controller
+coil-ipam-controller
 ===============
 
-`coil-controller` is a Kubernetes controller for Coil custom resources.
+`coil-ipam-controller` is a Kubernetes controller for Coil IPAM related custom resources.
 It is intended to be run as a Pod in `kube-system` namespace.
 
 ## AddressPool and AddressBlock
 
-`coil-controller` has an in-memory database of address pools and
+`coil-ipam-controller` has an in-memory database of address pools and
 address blocks to allocate address blocks quickly.
 
 ## BlockRequest
 
-`coil-controller` watches newly created block requests and carve out
+`coil-ipam-controller` watches newly created block requests and carve out
 address blocks from the requested pool.
 
-## Egress
-
-`coil-controller` creates **Deployment** and **Service** for each Egress.
-
-It also creates `coil-egress` **ServiceAccount** in the namespace of Egress,
-and binds it to the **ClusterRoles** for `coil-egress`.
-
 ## Garbage collection
 
-`coil-controller` periodically checks orphaned address blocks and deletes them.
+`coil-ipam-controller` periodically checks orphaned address blocks and deletes them.
 
 ## Command-line flags
 
 ```
 Flags:
       --cert-dir string        directory to locate TLS certs for webhook (default "/certs")
-      --egress-port int32      UDP port number used by coil-egress (default 5555)
       --gc-interval duration   garbage collection interval (default 1h0m0s)
       --health-addr string     bind address of health/readiness probes (default ":9387")
-  -h, --help                   help for coil-controller
+  -h, --help                   help for coil-ipam-controller
       --metrics-addr string    bind address of metrics endpoint (default ":9386")
-  -v, --version                version for coil-controller
+  -v, --version                version for coil-ipam-controller
       --webhook-addr string    bind address of admission webhook (default ":9443")
 ```
 

docs/cni-grpc.md

@@ -7,6 +7,7 @@
     - [AddResponse](#pkg-cnirpc-AddResponse)
     - [CNIArgs](#pkg-cnirpc-CNIArgs)
     - [CNIArgs.ArgsEntry](#pkg-cnirpc-CNIArgs-ArgsEntry)
+    - [CNIArgs.InterfacesEntry](#pkg-cnirpc-CNIArgs-InterfacesEntry)
     - [CNIError](#pkg-cnirpc-CNIError)
 
     - [ErrorCode](#pkg-cnirpc-ErrorCode)
@@ -57,6 +58,8 @@ https://pkg.go.dev/github.com/containernetworking/[email protected]/pkg/skel?tab=doc#Cm
 | args | [CNIArgs.ArgsEntry](#pkg-cnirpc-CNIArgs-ArgsEntry) | repeated | Key-Value pairs parsed from CNI_ARGS |
 | path | [string](#string) |  |  |
 | stdin_data | [bytes](#bytes) |  |  |
+| ips | [string](#string) | repeated |  |
+| interfaces | [CNIArgs.InterfacesEntry](#pkg-cnirpc-CNIArgs-InterfacesEntry) | repeated |  |
 
 
 
@@ -79,6 +82,22 @@ https://pkg.go.dev/github.com/containernetworking/[email protected]/pkg/skel?tab=doc#Cm
 
 
 
+<a name="pkg-cnirpc-CNIArgs-InterfacesEntry"></a>
+
+### CNIArgs.InterfacesEntry
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| key | [string](#string) |  |  |
+| value | [bool](#bool) |  |  |
+
+
+
+
+
+
 <a name="pkg-cnirpc-CNIError"></a>
 
 ### CNIError

docs/design.md

@@ -73,7 +73,8 @@ Now that we have learned how to do these things, and want to add rich features s
 
 Coil v2 will consist of the following programs:
 
-- `coil-controller`: Kubernetes controller managing custom resources.
+- `coil-ipam-controller`: Kubernetes controller managing IPAM related custom resources.
+- `coil-egress-controller`: Kubernetes controller managing on-demand NAT egress related custom resources.
 - `coild`: Daemon program running on nodes.
 - `coil`: CNI interface that delegates requests from `kubelet` to `coild`.
 - `coil-egress`: Administration program running in Egress pods.
@@ -102,11 +103,11 @@ To make things simple, the default pool is the pool whose name is `default`.
 To reduce the number of advertised routes, addresses in an address pool are divided into fixed-size blocks.
 These blocks are called _address blocks_, and assigned to nodes.  Since all IP addresses in an address block are routed to the same node, only one route per address block need to be advertised.
 
-For example, if an address pool defines that the size of an address block is 2<sup>5</sup>, `coil-controller` will carve an address block for IPv4 with `/27` subnet mask out of the pool, and assigns it to a node.
+For example, if an address pool defines that the size of an address block is 2<sup>5</sup>, `coil-ipam-controller` will carve an address block for IPv4 with `/27` subnet mask out of the pool, and assigns it to a node.
 
 
 In general, avoiding immediate reuse of IP addresses is better not to confuse other software or components.
-To avoid such immediate reuse, `coil-controller` remembers the last used address, and it assigns the address from the next address.
+To avoid such immediate reuse, `coil-ipam-controller` remembers the last used address, and it assigns the address from the next address.
 
 The same problem may occur when we use address blocks of the size `/32`.
 In this case, there is a high chance of reusing the same address immediately.
@@ -333,7 +334,7 @@ Therefore, when the deletion of the owning `AddressPool` is directed, all `Addre
 That said, an `AddressBlock` should not be deleted until there are no more Pods with an address in the block.
 For this purpose, Coil adds a finalizer to each `AddressBlock`.  **`coild` checks the usage of addresses in the block**, and once there are no more Pods using the addresses, it removes the finalizer to delete the `AddressBlock`.
 
-`AddressBlock` should also be deleted when `Node` that acquired the block is deleted.  Since `coild` running as a DaemonSet pod cannot do this, **`coil-controller` watches Node deletions and removes `AddressBlocks`**.  `coil-controller` periodically checks dangling `AddressBlocks` and removes them.
+`AddressBlock` should also be deleted when `Node` that acquired the block is deleted.  Since `coild` running as a DaemonSet pod cannot do this, **`coil-ipam-controller` watches Node deletions and removes `AddressBlocks`**.  `coil-ipam-controller` periodically checks dangling `AddressBlocks` and removes them.
 
 `coild` also deletes `AddressBlock` when it frees the last IP address used in the block.  At startup, `coild` also checks each `AddressBlock` for the Node, and if no Pod is using the addresses in the block, it deletes the `AddressBlock`.
 
@@ -342,7 +343,7 @@ Note that Coil does not include `Node` in the list of owner references of an `Ad
 ### AddressPool
 
 Similar to an `AddressBlock` and its addresses, an `AddressPool` should not be deleted until there are no more `AddressBlock`s derived from the pool.
-For this purpose, Coil adds a finalizer to each `AddressPool`.  `coil-controller` checks the usage of blocks in the pool.
+For this purpose, Coil adds a finalizer to each `AddressPool`.  `coil-ipam-controller` checks the usage of blocks in the pool.
 
 Note that `blockOwnerDeletion: true` in `AddressBlock`'s `ownerReferences` does not always block the deletion of the owning `AddressPool`.
 This directive has effect only when foreground cascading deletion is adopted.
@@ -397,7 +398,7 @@ skinparam rectangle {
 together {
 actor User
 package Deployment {
-    [coil-controller] as controller
+    [coil-ipam-controller] as controller
 }
 database kube-apiserver as apiserver #lightblue {
     [AddressPool] as pool1

docs/setup.md

@@ -17,6 +17,8 @@ You can tweak optional parameters by editing [`kustomization.yaml`](../v2/kustom
   - [IPv4/v6 dual stack pool](#ipv4v6-dual-stack-pool)
 - [(Option) Configure BIRD](#option-configure-bird)
 - [Note on CRI runtime compatibility](#note-on-cri-runtime-compatibility)
+- [Standalone egress](#standalone-egress)
+  - [Configuration](#configuration)
 
 ## Install `kustomize`
 
@@ -37,7 +39,8 @@ This should generate the following PEM files:
 
 ```console
 $ ls config/default/*.pem
-config/default/cert.pem  config/default/key.pem
+config/default/cert.pem         config/default/egress-key.pem  config/default/ipam-key.pem
+config/default/egress-cert.pem  config/default/ipam-cert.pem   config/default/key.pem
 ```
 
 ## Edit `kustomization.yaml`
@@ -57,7 +60,7 @@ $ vi kustomization.yaml
 (actually, the example is a network configuration list).
 
 You may edit the file to, say, add Cilium for network policies or to tune MTU.
-Note that `coil` must be the first in the plugin list.
+Note that `coil` must be the first in the plugin list if IPAM is enabled.
 
 ```console
 vi netconf.json
@@ -76,7 +79,7 @@ The following example adds `tuning` and `bandwidth` plugins.
   "plugins": [
     {
       "type": "coil",
-      "socket": "/run/coild.sock"
+      "socket": "/run/coild.sock",
     },
     {
       "type": "tuning",
@@ -231,3 +234,86 @@ host directory.
 
 [netconf]: https://github.com/containernetworking/cni/blob/spec-v0.4.0/SPEC.md#network-configuration
 [BIRD]: https://bird.network.cz/
+
+## Standalone egress
+
+Coil can be run as standalone egress NAT controller, using CNI chaining with another CNI providing base connectivity. This chapter will guide you on how to achieve this.
+
+### Configuration
+
+To deploy Coil with only egress feature enabled the following changes are required in the configuration files:
+
+1. Comment all IPAM related pieces in the following `kustomization.yaml` files:
+    - `v2/config/crd/kustomization.yaml`
+    - `v2/config/default/kustomization.yaml`
+    - `v2/config/pod/kustomization.yaml`
+    - `v2/config/rbac/kustomization.yaml`
+
+1. Comment unnecessary resources in `config/crd/patches/remove_status.yaml`.
+1. Add following arguments to the `coild` contianer executable in `config/pod/coild.yaml`
+    ```yaml
+    containers:
+      - name: coild
+        image: coil:dev
+        command: ["coild"]
+        args:
+          - --zap-stacktrace-level=panic
+          - --enable-ipam=false
+          - --enable-egress=true
+          - --pod-table-id=0 # 255 if IPv6 is being used
+          - --protocol-id=2
+    ```
+1. Set CNI config filename using environment variable for init contianer `coil-installer` in `config/pod/coild.yaml`:
+    ```yaml
+    env:
+    - name: CNI_CONF_NAME
+      value: "01-coil.conflist"
+    ```
+1. Add configuration of your chosen CNI to `v2/netconf.json` before `coil` related configuration.
+1. Deploy `coil` to existing cluster as described in [Compile and apply the manifest](#compile-and-apply-the-manifest).
+
+### Testing standalone egress
+
+#### Testing with Kindnet using IPv4
+1. Generate certificates using `v2/Makefile`.
+    ```bash
+    cd v2 && make certs
+    ```
+1. Go to `v2/e2e`
+    ```bash
+    cd e2e
+    ```
+1. Create IPv4 based Kind cluster with Kindnet CNI deployed:
+    ```bash
+    WITH_KINDNET=true TEST_IPV6=false make start
+    ```
+1. Install Coil on the cluster:
+    ```bash
+    make install-coil-egress-v4
+    ```
+1. Run egress-only IPv4 tests:
+    ```bash
+    TEST_IPAM=false TEST_EGRESS=true TEST_IPV6=false make test
+    ```
+
+#### Testing with Kindnet using IPv6
+1. Generate certificates using `v2/Makefile`.
+    ```bash
+    cd v2 && make certs
+    ```
+1. Go to `v2/e2e`
+    ```bash
+    cd e2e
+    ```
+1. Create IPv6 based Kind cluster with Kindnet CNI deployed:
+    ```bash
+    WITH_KINDNET=true TEST_IPV6=true make start
+    ```
+1. Install Coil on the cluster:
+    ```bash
+    make install-coil-egress-v6
+    ```
+1. Run egress-only IPv6 tests:
+    ```bash
+    TEST_IPAM=false TEST_EGRESS=true TEST_IPV6=true make test
+    ```

v2/.gitignore

@@ -1,6 +1,9 @@
 /bin
-/config/default/webhook_manifests_patch.yaml
+/config/default/ipam/webhook_manifests_patch.yaml
+/config/default/egress/webhook_manifests_patch.yaml
 /include
 /testbin
 /tmp
 /work
+/e2e/tmp
+/e2e/kindnet-conf

v2/Makefile

@@ -15,7 +15,8 @@ CONTROLLER_GEN := $(shell pwd)/bin/controller-gen
 SETUP_ENVTEST := $(shell pwd)/bin/setup-envtest
 YQ := $(shell pwd)/bin/yq
 CRD_OPTIONS = "crd:crdVersions=v1"
-ROLES = config/rbac/coil-controller_role.yaml \
+ROLES = config/rbac/coil-ipam-controller_role.yaml \
+	config/rbac/coil-egress-controller_role.yaml \
 	config/rbac/coild_role.yaml \
 	config/rbac/coil-router_role.yaml \
 	config/rbac/coil-egress_role.yaml
@@ -39,7 +40,7 @@ SHELL = /bin/bash
 .PHONY: test
 test: simple-test setup-envtest
 	source <($(SETUP_ENVTEST) use -p env); \
-		go test -race -v -count 1 ./...
+		TEST_IPAM=$(TEST_IPAM) TEST_EGRESS=$(TEST_EGRESS) go test -race -v -count 1 ./...
 
 .PHONY: simple-test
 simple-test: test-tools
@@ -78,36 +79,60 @@ check-generate:
 
 # Generate manifests e.g. CRD, RBAC etc.
 .PHONY: manifests
-manifests: $(CONTROLLER_GEN) $(ROLES) $(YQ)
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+manifests: manifests-ipam manifests-egress
+
+
+.PHONY: manifests-egress
+manifests-egress: $(CONTROLLER_GEN) $(ROLES) $(YQ)
+	mkdir -p tmp/egress
+	cp api/v2/egress_webhook.go tmp/egress
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) webhook paths="./tmp/egress/..." output:stdout output:crd:artifacts:config=config/crd/bases > config/webhook/egress/manifests.yaml
+	sed -i 's/webhook-/egress-webhook-/g' config/webhook/egress/manifests.yaml
 	# Reduce the size of Egress CRD by deleting `description` fields below the pod's template because it exceeds the limit of `metadata.annotations` length when it's applied with client-side mode.
 	$(YQ) -i 'del(.spec.versions.[].schema.openAPIV3Schema.properties.spec.properties.template | ..  |select(key == "description"))' config/crd/bases/coil.cybozu.com_egresses.yaml
+	rm -rf tmp 2> /dev/null
+
+.PHONY: manifests-ipam
+manifests-ipam: $(CONTROLLER_GEN) $(ROLES) $(YQ)
+	mkdir -p tmp/ipam
+	cp api/v2/addresspool_webhook.go tmp/ipam
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) webhook paths="./tmp/ipam/..." output:stdout output:crd:artifacts:config=config/crd/bases > config/webhook/ipam/manifests.yaml
+	sed -i 's/webhook-/ipam-webhook-/g' config/webhook/ipam/manifests.yaml
+	rm -rf tmp 2> /dev/null
+
 
 $(YQ):
 	$(WGET) -O yq.tar.gz https://github.com/mikefarah/yq/releases/download/v$(YQ_VERSION)/yq_linux_amd64.tar.gz
 	tar -C $(shell pwd)/bin/ -zxf yq.tar.gz ./yq_linux_amd64 -O > $@
 	rm -f yq.tar.gz
 	chmod +x $@
 
-COIL_CONTROLLER_ROLE_DEPENDS = controllers/addresspool_controller.go \
+COIL_IPAM_CONTROLLER_ROLE_DEPENDS = controllers/addresspool_controller.go \
 	controllers/blockrequest_controller.go \
-	controllers/egress_controller.go \
-	controllers/clusterrolebinding_controller.go \
 	pkg/ipam/pool.go \
 	runners/garbage_collector.go
 
-config/rbac/coil-controller_role.yaml: $(COIL_CONTROLLER_ROLE_DEPENDS)
+config/rbac/coil-ipam-controller_role.yaml: $(COIL_IPAM_CONTROLLER_ROLE_DEPENDS)
 	-rm -rf work
 	mkdir work
 	sed '0,/^package/s/.*/package work/' controllers/addresspool_controller.go > work/addresspool_controller.go
 	sed '0,/^package/s/.*/package work/' controllers/blockrequest_controller.go > work/blockrequest_controller.go
-	sed '0,/^package/s/.*/package work/' controllers/egress_controller.go > work/egress_controller.go
-	sed '0,/^package/s/.*/package work/' controllers/clusterrolebinding_controller.go > work/clusterrolebinding_controller.go
 	sed '0,/^package/s/.*/package work/' pkg/ipam/pool.go > work/pool.go
 	sed '0,/^package/s/.*/package work/' runners/garbage_collector.go > work/garbage_collector.go
-	$(CONTROLLER_GEN) rbac:roleName=coil-controller paths=./work output:stdout > $@
+	$(CONTROLLER_GEN) rbac:roleName=coil-ipam-controller paths=./work output:stdout > $@
 	rm -rf work
 
+COIL_EGRESS_CONTROLLER_ROLE_DEPENDS = controllers/egress_controller.go \
+	controllers/clusterrolebinding_controller.go
+
+config/rbac/coil-egress-controller_role.yaml: $(COIL_EGRESS_CONTROLLER_ROLE_DEPENDS)
+	-rm -rf work
+	mkdir work
+	sed '0,/^package/s/.*/package work/' controllers/egress_controller.go > work/egress_controller.go
+	sed '0,/^package/s/.*/package work/' controllers/clusterrolebinding_controller.go > work/clusterrolebinding_controller.go
+	$(CONTROLLER_GEN) rbac:roleName=coil-egress-controller paths=./work output:stdout > $@
+	# rm -rf work
+
 COILD_DEPENDS = controllers/blockrequest_watcher.go \
 	pkg/ipam/node.go \
 	runners/coild_server.go
@@ -142,12 +167,21 @@ config/rbac/coil-egress_role.yaml: controllers/pod_watcher.go
 
 # TLS certificates for webhook
 .PHONY: certs
-certs: config/default/cert.pem config/default/key.pem config/default/webhook_manifests_patch.yaml
+certs: config/default/cert.pem config/default/key.pem config/default/ipam-cert.pem config/default/ipam-key.pem config/default/ipam/webhook_manifests_patch.yaml config/default/egress-cert.pem config/default/egress-key.pem config/default/egress/webhook_manifests_patch.yaml
 
 config/default/cert.pem config/default/key.pem:
-	go run ./cmd/gencert -outdir=$(PWD)/config/default
+	go run ./cmd/gencert -outdir=$(PWD)/config/default -cn=coilv2-webhook-service -host=coilv2-webhook-service.kube-system.svc -certname=cert.pem -keyname=key.pem
+
+config/default/ipam-cert.pem config/default/ipam-key.pem:
+	go run ./cmd/gencert -outdir=$(PWD)/config/default -cn=coilv2-ipam-webhook-service -host=coilv2-ipam-webhook-service.kube-system.svc -certname=ipam-cert.pem -keyname=ipam-key.pem -ca=cert.pem -cakey=key.pem
+
+config/default/egress-cert.pem config/default/egress-key.pem:
+	go run ./cmd/gencert -outdir=$(PWD)/config/default -cn=coilv2-egress-webhook-service -host=coilv2-egress-webhook-service.kube-system.svc -certname=egress-cert.pem -keyname=egress-key.pem -ca=cert.pem -cakey=key.pem
+
+config/default/ipam/webhook_manifests_patch.yaml: config/default/ipam-cert.pem config/default/ipam/webhook_manifests_patch.yaml.tmpl
+	sed "s/%CACERT%/$$(base64 -w0 < $<)/g" $@.tmpl > $@
 
-config/default/webhook_manifests_patch.yaml: config/default/cert.pem config/default/webhook_manifests_patch.yaml.tmpl
+config/default/egress/webhook_manifests_patch.yaml: config/default/egress-cert.pem config/default/egress/webhook_manifests_patch.yaml.tmpl
 	sed "s/%CACERT%/$$(base64 -w0 < $<)/g" $@.tmpl > $@
 
 # Generate code
@@ -171,7 +205,8 @@ pkg/cnirpc/cni_grpc.pb.go: pkg/cnirpc/cni.proto
 .PHONY: build
 build:
 	GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil -ldflags="-s -w" cmd/coil/*.go
-	GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil-controller -ldflags="-s -w" cmd/coil-controller/*.go
+	GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil-ipam-controller -ldflags="-s -w" cmd/coil-ipam-controller/*.go
+	GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil-egress-controller -ldflags="-s -w" cmd/coil-egress-controller/*.go
 	GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil-egress -ldflags="-s -w" cmd/coil-egress/*.go
 	GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil-installer -ldflags="-s -w" cmd/coil-installer/*.go
 	GOARCH=$(GOARCH) CGO_ENABLED=0 go build -o work/coil-router -ldflags="-s -w" cmd/coil-router/*.go

v2/api/v2/suite_test.go

@@ -50,7 +50,10 @@ var _ = BeforeSuite(func() {
 	testEnv = &envtest.Environment{
 		CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")},
 		WebhookInstallOptions: envtest.WebhookInstallOptions{
-			Paths: []string{filepath.Join("..", "..", "config", "webhook")},
+			Paths: []string{
+				filepath.Join("..", "..", "config", "webhook", "ipam"),
+				filepath.Join("..", "..", "config", "webhook", "egress"),
+			},
 		},
 		ErrorIfCRDPathMissing: true,
 	}

v2/cmd/coil-egress-controller/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/cybozu-go/coil/v2/cmd/coil-egress-controller/sub"
+
+func main() {
+	sub.Execute()
+}

v2/cmd/coil-egress-controller/sub/root.go

@@ -0,0 +1,56 @@
+package sub
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	v2 "github.com/cybozu-go/coil/v2"
+	"github.com/spf13/cobra"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+)
+
+var config struct {
+	metricsAddr string
+	healthAddr  string
+	webhookAddr string
+	certDir     string
+	egressPort  int32
+	zapOpts     zap.Options
+}
+
+var rootCmd = &cobra.Command{
+	Use:     "coil-egress-controller",
+	Short:   "controller for coil egress related custom resources",
+	Long:    `coil-egress-controller is a Kubernetes controller for coil egress related custom resources.`,
+	Version: v2.Version(),
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		cmd.SilenceUsage = true
+		return subMain()
+	},
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+	pf := rootCmd.PersistentFlags()
+	pf.StringVar(&config.metricsAddr, "metrics-addr", ":9396", "bind address of metrics endpoint")
+	pf.StringVar(&config.healthAddr, "health-addr", ":9397", "bind address of health/readiness probes")
+	pf.StringVar(&config.webhookAddr, "webhook-addr", ":9444", "bind address of admission webhook")
+	pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook")
+	pf.Int32Var(&config.egressPort, "egress-port", 5555, "UDP port number used by coil-egress")
+
+	goflags := flag.NewFlagSet("klog", flag.ExitOnError)
+	klog.InitFlags(goflags)
+	config.zapOpts.BindFlags(goflags)
+
+	pf.AddGoFlagSet(goflags)
+}

v2/cmd/coil-egress-controller/sub/run.go

@@ -0,0 +1,120 @@
+package sub
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"time"
+
+	v2 "github.com/cybozu-go/coil/v2"
+	coilv2 "github.com/cybozu-go/coil/v2/api/v2"
+	"github.com/cybozu-go/coil/v2/controllers"
+	"github.com/cybozu-go/coil/v2/pkg/constants"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+)
+
+const (
+	gracefulTimeout = 20 * time.Second
+)
+
+var (
+	scheme   = runtime.NewScheme()
+	setupLog = ctrl.Log.WithName("setup")
+)
+
+func init() {
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(coilv2.AddToScheme(scheme))
+
+	// +kubebuilder:scaffold:scheme
+}
+
+func subMain() error {
+	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&config.zapOpts)))
+
+	host, portStr, err := net.SplitHostPort(config.webhookAddr)
+	if err != nil {
+		return fmt.Errorf("invalid webhook address: %w", err)
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		return fmt.Errorf("invalid webhook address: %w", err)
+	}
+
+	timeout := gracefulTimeout
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		Scheme:                  scheme,
+		LeaderElection:          true,
+		LeaderElectionID:        "coil-egress-leader",
+		LeaderElectionNamespace: "kube-system", // coil should run in kube-system
+		Metrics: metricsserver.Options{
+			BindAddress: config.metricsAddr,
+		},
+		GracefulShutdownTimeout: &timeout,
+		HealthProbeBindAddress:  config.healthAddr,
+		WebhookServer: webhook.NewServer(webhook.Options{
+			Host:    host,
+			Port:    port,
+			CertDir: config.certDir,
+		}),
+	})
+	if err != nil {
+		return err
+	}
+
+	if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil {
+		return err
+	}
+	if err := mgr.AddReadyzCheck("ping", healthz.Ping); err != nil {
+		return err
+	}
+
+	// register controllers
+
+	ctx := ctrl.SetupSignalHandler()
+
+	podNS := os.Getenv(constants.EnvPodNamespace)
+	podName := os.Getenv(constants.EnvPodName)
+	img, err := controllers.GetImage(mgr.GetAPIReader(), client.ObjectKey{Namespace: podNS, Name: podName})
+	if err != nil {
+		return err
+	}
+	egressctrl := controllers.EgressReconciler{
+		Client: mgr.GetClient(),
+		Scheme: scheme,
+		Image:  img,
+		Port:   config.egressPort,
+	}
+	if err := egressctrl.SetupWithManager(mgr); err != nil {
+		return err
+	}
+
+	if err := controllers.SetupCRBReconciler(mgr); err != nil {
+		return err
+	}
+
+	// register webhooks
+
+	if err := (&coilv2.Egress{}).SetupWebhookWithManager(mgr); err != nil {
+		return err
+	}
+
+	// start manager
+
+	setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
+	if err := mgr.Start(ctx); err != nil {
+		setupLog.Error(err, "problem running manager")
+		return err
+	}
+
+	return nil
+}

v2/cmd/coil-installer/sub/install.go

@@ -4,6 +4,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 func installCniConf(cniConfName, cniEtcDir, cniNetConf, cniNetConfFile string) error {
@@ -29,9 +30,11 @@ func installCniConf(cniConfName, cniEtcDir, cniNetConf, cniNetConfFile string) e
 		if fi.IsDir() {
 			continue
 		}
-		err := os.Remove(filepath.Join(cniEtcDir, fi.Name()))
-		if err != nil {
-			return err
+		if strings.Contains(fi.Name(), "conflist") {
+			err := os.Remove(filepath.Join(cniEtcDir, fi.Name()))
+			if err != nil {
+				return err
+			}
 		}
 	}
 

v2/cmd/coil-ipam-controller/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/cybozu-go/coil/v2/cmd/coil-ipam-controller/sub"
+
+func main() {
+	sub.Execute()
+}

v2/cmd/coil-controller/sub/root.go → v2/cmd/coil-ipam-controller/sub/root.go

@@ -18,7 +18,6 @@ var config struct {
 	webhookAddr string
 	certDir     string
 	gcInterval  time.Duration
-	egressPort  int32
 	zapOpts     zap.Options
 }
 
@@ -49,7 +48,6 @@ func init() {
 	pf.StringVar(&config.webhookAddr, "webhook-addr", ":9443", "bind address of admission webhook")
 	pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook")
 	pf.DurationVar(&config.gcInterval, "gc-interval", 1*time.Hour, "garbage collection interval")
-	pf.Int32Var(&config.egressPort, "egress-port", 5555, "UDP port number used by coil-egress")
 
 	goflags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(goflags)

v2/cmd/coil-controller/sub/run.go → v2/cmd/coil-ipam-controller/sub/run.go

@@ -3,22 +3,19 @@ package sub
 import (
 	"fmt"
 	"net"
-	"os"
 	"strconv"
 	"time"
 
 	v2 "github.com/cybozu-go/coil/v2"
 	coilv2 "github.com/cybozu-go/coil/v2/api/v2"
 	"github.com/cybozu-go/coil/v2/controllers"
-	"github.com/cybozu-go/coil/v2/pkg/constants"
 	"github.com/cybozu-go/coil/v2/pkg/indexing"
 	"github.com/cybozu-go/coil/v2/pkg/ipam"
 	"github.com/cybozu-go/coil/v2/runners"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -57,7 +54,7 @@ func subMain() error {
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                  scheme,
 		LeaderElection:          true,
-		LeaderElectionID:        "coil-leader",
+		LeaderElectionID:        "coil-ipam-leader",
 		LeaderElectionNamespace: "kube-system", // coil should run in kube-system
 		Metrics: metricsserver.Options{
 			BindAddress: config.metricsAddr,
@@ -107,34 +104,11 @@ func subMain() error {
 		return err
 	}
 
-	podNS := os.Getenv(constants.EnvPodNamespace)
-	podName := os.Getenv(constants.EnvPodName)
-	img, err := controllers.GetImage(mgr.GetAPIReader(), client.ObjectKey{Namespace: podNS, Name: podName})
-	if err != nil {
-		return err
-	}
-	egressctrl := controllers.EgressReconciler{
-		Client: mgr.GetClient(),
-		Scheme: scheme,
-		Image:  img,
-		Port:   config.egressPort,
-	}
-	if err := egressctrl.SetupWithManager(mgr); err != nil {
-		return err
-	}
-
-	if err := controllers.SetupCRBReconciler(mgr); err != nil {
-		return err
-	}
-
 	// register webhooks
 
 	if err := (&coilv2.AddressPool{}).SetupWebhookWithManager(mgr); err != nil {
 		return err
 	}
-	if err := (&coilv2.Egress{}).SetupWebhookWithManager(mgr); err != nil {
-		return err
-	}
 
 	// other runners
 

v2/cmd/coil/main.go

@@ -13,19 +13,17 @@ import (
 	"github.com/cybozu-go/coil/v2/pkg/cnirpc"
 )
 
-const rpcTimeout = 1 * time.Minute
+const (
+	rpcTimeout = 1 * time.Minute
+)
 
 func cmdAdd(args *skel.CmdArgs) error {
 	conf, err := parseConfig(args.StdinData)
 	if err != nil {
 		return err
 	}
 
-	if conf.PrevResult != nil {
-		return types.NewError(types.ErrInvalidNetworkConfig, "coil must be called as the first plugin", "")
-	}
-
-	cniArgs, err := makeCNIArgs(args)
+	cniArgs, err := makeCNIArgs(args, conf)
 	if err != nil {
 		return err
 	}
@@ -45,7 +43,13 @@ func cmdAdd(args *skel.CmdArgs) error {
 		return convertError(err)
 	}
 
-	result, err := current.NewResult(resp.Result)
+	var result types.Result
+	if conf.PrevResult != nil {
+		result, err = current.NewResultFromResult(conf.PrevResult)
+	} else {
+		result, err = current.NewResult(resp.Result)
+	}
+
 	if err != nil {
 		return types.NewError(types.ErrDecodingFailure, "failed to unmarshal result", err.Error())
 	}
@@ -59,7 +63,7 @@ func cmdDel(args *skel.CmdArgs) error {
 		return err
 	}
 
-	cniArgs, err := makeCNIArgs(args)
+	cniArgs, err := makeCNIArgs(args, conf)
 	if err != nil {
 		return err
 	}
@@ -87,7 +91,7 @@ func cmdCheck(args *skel.CmdArgs) error {
 		return err
 	}
 
-	cniArgs, err := makeCNIArgs(args)
+	cniArgs, err := makeCNIArgs(args, conf)
 	if err != nil {
 		return err
 	}

v2/cmd/coil/rpc.go

@@ -2,32 +2,57 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"net"
+	"strconv"
 
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
+	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/cybozu-go/coil/v2/pkg/cnirpc"
+	"github.com/cybozu-go/coil/v2/pkg/constants"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/resolver"
 	"google.golang.org/grpc/status"
 )
 
 // makeCNIArgs creates *CNIArgs.
-func makeCNIArgs(args *skel.CmdArgs) (*cnirpc.CNIArgs, error) {
+func makeCNIArgs(args *skel.CmdArgs, conf *PluginConf) (*cnirpc.CNIArgs, error) {
 	env := &PluginEnvArgs{}
 	if err := types.LoadArgs(args.Args, env); err != nil {
 		return nil, types.NewError(types.ErrInvalidEnvironmentVariables, "failed to load CNI_ARGS", err.Error())
 	}
 
+	argsData := env.Map()
+	argsData[constants.IsChained] = strconv.FormatBool(conf.PrevResult != nil)
+
+	ips := []string{}
+	interfaces := map[string]bool{}
+	if conf.PrevResult != nil {
+		prevResult, err := current.GetResult(conf.PrevResult)
+		if err != nil {
+			return nil, fmt.Errorf("error getting previous CNI result: %w", err)
+		}
+		for _, ip := range prevResult.IPs {
+			ips = append(ips, ip.Address.IP.String())
+		}
+		for _, intf := range prevResult.Interfaces {
+			interfaces[intf.Name] = intf.Sandbox != ""
+		}
+	}
+
 	cniArgs := &cnirpc.CNIArgs{
 		ContainerId: args.ContainerID,
 		Netns:       args.Netns,
 		Ifname:      args.IfName,
-		Args:        env.Map(),
+		Args:        argsData,
 		Path:        args.Path,
 		StdinData:   args.StdinData,
+		Ips:         ips,
+		Interfaces:  interfaces,
 	}
+
 	return cniArgs, nil
 }
 

v2/cmd/coild/sub/root.go

@@ -1,31 +1,14 @@
 package sub
 
 import (
-	"flag"
 	"fmt"
 	"os"
 
 	v2 "github.com/cybozu-go/coil/v2"
-	"github.com/cybozu-go/coil/v2/pkg/constants"
+	"github.com/cybozu-go/coil/v2/pkg/config"
 	"github.com/spf13/cobra"
-	"k8s.io/klog/v2"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
-var config struct {
-	metricsAddr      string
-	healthAddr       string
-	podTableId       int
-	podRulePrio      int
-	exportTableId    int
-	protocolId       int
-	socketPath       string
-	compatCalico     bool
-	egressPort       int
-	registerFromMain bool
-	zapOpts          zap.Options
-}
-
 var rootCmd = &cobra.Command{
 	Use:   "coild",
 	Short: "gRPC server running on each node",
@@ -40,31 +23,14 @@ coil CNI plugin.`,
 	},
 }
 
+var cfg *config.Config
+
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
+	cfg = config.Parse(rootCmd)
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println(err)
 		os.Exit(1)
 	}
 }
-
-func init() {
-	pf := rootCmd.PersistentFlags()
-	pf.StringVar(&config.metricsAddr, "metrics-addr", ":9384", "bind address of metrics endpoint")
-	pf.StringVar(&config.healthAddr, "health-addr", ":9385", "bind address of health/readiness probes")
-	pf.IntVar(&config.podTableId, "pod-table-id", 116, "routing table ID to which coild registers routes for Pods")
-	pf.IntVar(&config.podRulePrio, "pod-rule-prio", 2000, "priority with which the rule for Pod table is inserted")
-	pf.IntVar(&config.exportTableId, "export-table-id", 119, "routing table ID to which coild exports routes")
-	pf.IntVar(&config.protocolId, "protocol-id", 30, "route author ID")
-	pf.StringVar(&config.socketPath, "socket", constants.DefaultSocketPath, "UNIX domain socket path")
-	pf.BoolVar(&config.compatCalico, "compat-calico", false, "make veth name compatible with Calico")
-	pf.IntVar(&config.egressPort, "egress-port", 5555, "UDP port number for egress NAT")
-	pf.BoolVar(&config.registerFromMain, "register-from-main", false, "help migration from Coil 2.0.1")
-
-	goflags := flag.NewFlagSet("klog", flag.ExitOnError)
-	klog.InitFlags(goflags)
-	config.zapOpts.BindFlags(goflags)
-
-	pf.AddGoFlagSet(goflags)
-}

v2/cmd/coild/sub/run.go

@@ -38,13 +38,12 @@ var (
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(coilv2.AddToScheme(scheme))
-
 	// +kubebuilder:scaffold:scheme
 }
 
 func subMain() error {
 	// coild needs a raw zap logger for grpc_zip.
-	zapLogger := zap.NewRaw(zap.UseFlagOptions(&config.zapOpts))
+	zapLogger := zap.NewRaw(zap.UseFlagOptions(&cfg.ZapOpts))
 	defer zapLogger.Sync()
 
 	grpcLogger := zapLogger.Named("grpc")
@@ -55,15 +54,19 @@ func subMain() error {
 		return errors.New(constants.EnvNode + " environment variable should be set")
 	}
 
+	if !cfg.EnableIPAM && !cfg.EnableEgress {
+		return errors.New("configuration error: both IPAM and egress are disabled")
+	}
+
 	timeout := gracefulTimeout
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:         scheme,
 		LeaderElection: false,
 		Metrics: metricsserver.Options{
-			BindAddress: config.metricsAddr,
+			BindAddress: cfg.MetricsAddr,
 		},
 		GracefulShutdownTimeout: &timeout,
-		HealthProbeBindAddress:  config.healthAddr,
+		HealthProbeBindAddress:  cfg.HealthAddr,
 	})
 	if err != nil {
 		return err
@@ -76,15 +79,17 @@ func subMain() error {
 		return err
 	}
 
-	exporter := nodenet.NewRouteExporter(config.exportTableId, config.protocolId, ctrl.Log.WithName("route-exporter"))
+	exporter := nodenet.NewRouteExporter(cfg.ExportTableId, cfg.ProtocolId, ctrl.Log.WithName("route-exporter"))
 	nodeIPAM := ipam.NewNodeIPAM(nodeName, ctrl.Log.WithName("node-ipam"), mgr, exporter)
-	watcher := &controllers.BlockRequestWatcher{
-		Client:   mgr.GetClient(),
-		NodeIPAM: nodeIPAM,
-		NodeName: nodeName,
-	}
-	if err := watcher.SetupWithManager(mgr); err != nil {
-		return err
+	if cfg.EnableIPAM {
+		watcher := &controllers.BlockRequestWatcher{
+			Client:   mgr.GetClient(),
+			NodeIPAM: nodeIPAM,
+			NodeName: nodeName,
+		}
+		if err := watcher.SetupWithManager(mgr); err != nil {
+			return err
+		}
 	}
 
 	ctx := context.Background()
@@ -94,50 +99,57 @@ func subMain() error {
 	}
 
 	podNet := nodenet.NewPodNetwork(
-		config.podTableId,
-		config.podRulePrio,
-		config.protocolId,
+		cfg.PodTableId,
+		cfg.PodRulePrio,
+		cfg.ProtocolId,
 		ipv4,
 		ipv6,
-		config.compatCalico,
-		config.registerFromMain,
-		ctrl.Log.WithName("pod-network"))
+		cfg.CompatCalico,
+		cfg.RegisterFromMain,
+		ctrl.Log.WithName("pod-network"),
+		cfg.EnableIPAM)
 	if err := podNet.Init(); err != nil {
 		return err
 	}
-	podConfigs, err := podNet.List()
-	if err != nil {
-		return err
-	}
 
-	for _, c := range podConfigs {
-		if err := nodeIPAM.Register(ctx, c.PoolName, c.ContainerId, c.IFace, c.IPv4, c.IPv6); err != nil {
+	if cfg.EnableIPAM {
+		podConfigs, err := podNet.List()
+		if err != nil {
+			return err
+		}
+
+		for _, c := range podConfigs {
+			if err := nodeIPAM.Register(ctx, c.PoolName, c.ContainerId, c.IFace, c.IPv4, c.IPv6); err != nil {
+				return err
+			}
+		}
+		if err := nodeIPAM.GC(ctx); err != nil {
 			return err
 		}
-	}
-	if err := nodeIPAM.GC(ctx); err != nil {
-		return err
 	}
 
-	os.Remove(config.socketPath)
-	l, err := net.Listen("unix", config.socketPath)
+	os.Remove(cfg.SocketPath)
+	l, err := net.Listen("unix", cfg.SocketPath)
 	if err != nil {
 		return err
 	}
-	server := runners.NewCoildServer(l, mgr, nodeIPAM, podNet, runners.NewNATSetup(config.egressPort), grpcLogger)
+	server := runners.NewCoildServer(l, mgr, nodeIPAM, podNet, runners.NewNATSetup(cfg.EgressPort), cfg, grpcLogger, runners.ProcessLinkAlias)
 	if err := mgr.Add(server); err != nil {
 		return err
 	}
 
-	egressWatcher := &controllers.EgressWatcher{
-		Client:     mgr.GetClient(),
-		NodeName:   nodeName,
-		PodNet:     podNet,
-		EgressPort: config.egressPort,
-	}
-	if err := egressWatcher.SetupWithManager(mgr); err != nil {
-		return err
+	if cfg.EnableEgress {
+		egressWatcher := &controllers.EgressWatcher{
+			Client:     mgr.GetClient(),
+			NodeName:   nodeName,
+			PodNet:     podNet,
+			EgressPort: cfg.EgressPort,
+		}
+		if err := egressWatcher.SetupWithManager(mgr); err != nil {
+			return err
+		}
 	}
+
 	ctx2 := ctrl.SetupSignalHandler()
 	if err := indexing.SetupIndexForPodByNodeName(ctx2, mgr); err != nil {
 		return err

v2/cmd/gencert/main.go

@@ -7,6 +7,7 @@ import (
 	"crypto/x509/pkix"
 	"encoding/pem"
 	"flag"
+	"fmt"
 	"log"
 	"math/big"
 	"os"
@@ -16,37 +17,65 @@ import (
 )
 
 var (
-	host     = flag.String("host", "coilv2-webhook-service.kube-system.svc", "TLS hostname")
-	validFor = flag.Duration("duration", 36500*24*time.Hour, "Duration that certificate is valid for")
-	outDir   = flag.String("outdir", ".", "Directory where the certificate files are created")
+	host         = flag.String("host", "coilv2-webhook-service.kube-system.svc", "TLS hostname")
+	validFor     = flag.Duration("duration", 36500*24*time.Hour, "Duration that certificate is valid for")
+	outDir       = flag.String("outdir", ".", "Directory where the certificate files are created")
+	commonName   = flag.String("cn", "coilv2-webhook-service", "Certificate common name")
+	outCert      = flag.String("certname", "cert.pem", "Certificate filename")
+	outKey       = flag.String("keyname", "key.pem", "Key filename")
+	authority    = flag.String("ca", "", "Certificate authority")
+	authorityKey = flag.String("cakey", "", "Certificate authority")
 )
 
 func main() {
 	flag.Parse()
 
-	priv, err := rsa.GenerateKey(rand.Reader, 4096)
-	if err != nil {
-		log.Fatal(err)
+	var ca *x509.Certificate
+	var priv *rsa.PrivateKey
+	var err error
+	if *authority != "" {
+		if ca, priv, err = readCA(filepath.Join(*outDir, *authority), filepath.Join(*outDir, *authorityKey)); err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		priv, err = rsa.GenerateKey(rand.Reader, 4096)
+		if err != nil {
+			log.Fatal(err)
+		}
 	}
 
+	isCA := (ca == nil)
+
 	keyUsage := x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment
+	extKeyUsage := []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}
+	if isCA {
+		keyUsage = x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign
+		extKeyUsage = append(extKeyUsage, x509.ExtKeyUsageClientAuth)
+	}
+
 	notBefore := time.Now()
 	notAfter := notBefore.Add(*validFor)
 
 	template := x509.Certificate{
 		SerialNumber: big.NewInt(1),
 		Subject: pkix.Name{
-			CommonName: "coilv2-webhook-service",
+			CommonName: *commonName,
 		},
 		NotBefore:             notBefore,
 		NotAfter:              notAfter,
 		KeyUsage:              keyUsage,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		ExtKeyUsage:           extKeyUsage,
 		BasicConstraintsValid: true,
+		IsCA:                  isCA,
 		DNSNames:              dnsAliases(*host),
 	}
 
-	certBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, priv.Public(), priv)
+	parent := ca
+	if isCA {
+		parent = &template
+	}
+
+	certBytes, err := x509.CreateCertificate(rand.Reader, &template, parent, priv.Public(), priv)
 	if err != nil {
 		log.Fatalf("failed to create certificate: %v", err)
 	}
@@ -68,8 +97,8 @@ func main() {
 		log.Fatalf("failed to marshal private key: %v", err)
 	}
 
-	outputPEM(filepath.Join(*outDir, "cert.pem"), "CERTIFICATE", certBytes)
-	outputPEM(filepath.Join(*outDir, "key.pem"), "PRIVATE KEY", privBytes)
+	outputPEM(filepath.Join(*outDir, *outCert), "CERTIFICATE", certBytes)
+	outputPEM(filepath.Join(*outDir, *outKey), "PRIVATE KEY", privBytes)
 }
 
 func dnsAliases(host string) []string {
@@ -98,3 +127,35 @@ func outputPEM(fname string, pemType string, data []byte) {
 		log.Fatalf("failed to fsync: %v", err)
 	}
 }
+
+func readCA(certPath, keyPath string) (*x509.Certificate, *rsa.PrivateKey, error) {
+	certFile, err := os.ReadFile(certPath)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error reading file '%s': %w", certPath, err)
+	}
+
+	caData, _ := pem.Decode(certFile)
+
+	ca, err := x509.ParseCertificate(caData.Bytes)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	keyFile, err := os.ReadFile(keyPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	cakData, _ := pem.Decode(keyFile)
+	key, err := x509.ParsePKCS8PrivateKey(cakData.Bytes)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	privateKey, ok := key.(*rsa.PrivateKey)
+	if !ok {
+		return nil, nil, fmt.Errorf("type assertion error - object is not of type *rsa.PrivateKey")
+	}
+
+	return ca, privateKey, nil
+}

v2/config/cke/webhook-secret.yaml

@@ -1,7 +1,15 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: coilv2-webhook-server-cert
+  name: coilv2-ipam-webhook-server-cert
   namespace: system
   annotations:
-    cke.cybozu.com/issue-cert: coilv2-webhook-service
+    cke.cybozu.com/issue-cert: coilv2-ipam-webhook-service
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: coilv2-egress-webhook-server-cert
+  namespace: system
+  annotations:
+    cke.cybozu.com/issue-cert: coilv2-egress-webhook-service

v2/config/crd/egress/kustomization.yaml

@@ -0,0 +1,25 @@
+# This kustomization.yaml is not intended to be run by itself,
+# since it depends on service name and namespace that are out of this kustomize package.
+# It should be run by config/default
+resources:
+# [EGRESS] Following files should be uncommented to enable Egress NAT features.
+- ../bases/coil.cybozu.com_egresses.yaml
+# +kubebuilder:scaffold:crdkustomizeresource
+
+patchesStrategicMerge:
+- ../patches/egress/remove_status.yaml
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
+# patches here are for enabling the conversion webhook for each CRD
+# [EGRESS] Following files should be uncommented to enable Egress NAT features.
+#- patches/webhook_in_egresses.yaml
+# +kubebuilder:scaffold:crdkustomizewebhookpatch
+
+# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix.
+# patches here are for enabling the CA injection for each CRD
+# [EGRESS] Following files should be uncommented to enable Egress NAT features.
+#- patches/cainjection_in_egresses.yaml
+# +kubebuilder:scaffold:crdkustomizecainjectionpatch
+
+# the following config is for teaching kustomize how to do kustomization for CRDs.
+configurations:
+- ../kustomizeconfig.yaml

v2/config/crd/kustomization.yaml

@@ -2,27 +2,33 @@
 # since it depends on service name and namespace that are out of this kustomize package.
 # It should be run by config/default
 resources:
+# [IPAM] Following files should be uncommented to enable IPAM features.
 - bases/coil.cybozu.com_addresspools.yaml
 - bases/coil.cybozu.com_addressblocks.yaml
 - bases/coil.cybozu.com_blockrequests.yaml
+# [EGRESS] Following files should be uncommented to enable Egress NAT features.
 - bases/coil.cybozu.com_egresses.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:
 - patches/remove_status.yaml
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
 # patches here are for enabling the conversion webhook for each CRD
+# [IPAM] Following files should be uncommented to enable IPAM features.
 #- patches/webhook_in_addresspools.yaml
 #- patches/webhook_in_addressblocks.yaml
 #- patches/webhook_in_blockrequests.yaml
+# [EGRESS] Following files should be uncommented to enable Egress NAT features.
 #- patches/webhook_in_egresses.yaml
 # +kubebuilder:scaffold:crdkustomizewebhookpatch
 
 # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix.
 # patches here are for enabling the CA injection for each CRD
+# [IPAM] Following files should be uncommented to enable IPAM features.
 #- patches/cainjection_in_addresspools.yaml
 #- patches/cainjection_in_addressblocks.yaml
 #- patches/cainjection_in_blockrequests.yaml
+# [EGRESS] Following files should be uncommented to enable Egress NAT features.
 #- patches/cainjection_in_egresses.yaml
 # +kubebuilder:scaffold:crdkustomizecainjectionpatch
 

v2/config/crd/patches/egress/remove_status.yaml

@@ -0,0 +1,7 @@
+# [EGRESS] Following resources be uncommented to enable Egress NAT features.
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: egresses.coil.cybozu.com
+status: null
+---

v2/config/crd/patches/remove_status.yaml

@@ -1,21 +1,23 @@
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: addressblocks.coil.cybozu.com
-status: null
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: addresspools.coil.cybozu.com
-status: null
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: blockrequests.coil.cybozu.com
-status: null
----
+# [IPAM] Following resources should be uncommented to enable IPAM features.
+# apiVersion: apiextensions.k8s.io/v1
+# kind: CustomResourceDefinition
+# metadata:
+#   name: addressblocks.coil.cybozu.com
+# status: null
+# ---
+# apiVersion: apiextensions.k8s.io/v1
+# kind: CustomResourceDefinition
+# metadata:
+#   name: addresspools.coil.cybozu.com
+# status: null
+# ---
+# apiVersion: apiextensions.k8s.io/v1
+# kind: CustomResourceDefinition
+# metadata:
+#   name: blockrequests.coil.cybozu.com
+# status: null
+# ---
+# [EGRESS] Following resources be uncommented to enable Egress NAT features.
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:

v2/config/crd/patches/webhook_in_addressblocks.yaml

@@ -13,5 +13,5 @@ spec:
       caBundle: Cg==
       service:
         namespace: system
-        name: webhook-service
+        name: ipam-webhook-service
         path: /convert

v2/config/crd/patches/webhook_in_addresspools.yaml

@@ -13,5 +13,5 @@ spec:
       caBundle: Cg==
       service:
         namespace: system
-        name: webhook-service
+        name: ipam-webhook-service
         path: /convert

v2/config/crd/patches/webhook_in_blockrequests.yaml

@@ -13,5 +13,5 @@ spec:
       caBundle: Cg==
       service:
         namespace: system
-        name: webhook-service
+        name: ipam-webhook-service
         path: /convert

v2/config/crd/patches/webhook_in_egresses.yaml

@@ -13,5 +13,5 @@ spec:
       caBundle: Cg==
       service:
         namespace: system
-        name: webhook-service
+        name: egress-webhook-service
         path: /convert

v2/config/default/.gitignore

@@ -1,2 +1,6 @@
 cert.pem
 key.pem
+ipam-cert.pem
+ipam-key.pem
+egress-cert.pem
+egress-key.pem

v2/config/default/egress/v4/kustomization.yaml

@@ -0,0 +1,20 @@
+resources:
+- ../../../crd/egress
+- ../../../rbac/egress
+- ../../../pod/egress/v4
+- ../../../webhook/egress
+
+patchesStrategicMerge:
+- ../webhook_manifests_patch.yaml
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+secretGenerator:
+# [EGRESS] Following lines be uncommented to enable Egress NAT features.
+- name: coilv2-egress-webhook-server-cert
+  files:
+  - ca.crt=../../cert.pem
+  - tls.crt=../../egress-cert.pem
+  - tls.key=../../egress-key.pem
+  type: "kubernetes.io/tls"

v2/config/default/egress/v6/kustomization.yaml

@@ -0,0 +1,20 @@
+resources:
+- ../../../crd/egress
+- ../../../rbac/egress
+- ../../../pod/egress/v6
+- ../../../webhook/egress
+
+patchesStrategicMerge:
+- ../webhook_manifests_patch.yaml
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+secretGenerator:
+# [EGRESS] Following lines be uncommented to enable Egress NAT features.
+- name: coilv2-egress-webhook-server-cert
+  files:
+  - ca.crt=../../cert.pem
+  - tls.crt=../../egress-cert.pem
+  - tls.key=../../egress-key.pem
+  type: "kubernetes.io/tls"

v2/config/default/egress/webhook_manifests_patch.yaml.tmpl

@@ -0,0 +1,17 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: coilv2-mutating-egress-webhook-configuration
+webhooks:
+- name: megress.kb.io
+  clientConfig:
+    caBundle: "%CACERT%"
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: coilv2-validating-egress-webhook-configuration
+webhooks:
+- name: vegress.kb.io
+  clientConfig:
+    caBundle: "%CACERT%"

v2/config/default/webhook_manifests_patch.yaml.tmpl → v2/config/default/ipam/webhook_manifests_patch.yaml.tmpl

@@ -1,24 +1,17 @@
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
-  name: coilv2-mutating-webhook-configuration
+  name: coilv2-mutating-ipam-webhook-configuration
 webhooks:
 - name: maddresspool.kb.io
   clientConfig:
     caBundle: "%CACERT%"
-- name: megress.kb.io
-  clientConfig:
-    caBundle: "%CACERT%"
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
-  name: coilv2-validating-webhook-configuration
+  name: coilv2-validating-ipam-webhook-configuration
 webhooks:
 - name: vaddresspool.kb.io
   clientConfig:
     caBundle: "%CACERT%"
-- name: vegress.kb.io
-  clientConfig:
-    caBundle: "%CACERT%"
----

v2/config/default/kustomization.yaml

@@ -2,18 +2,28 @@ resources:
 - ../crd
 - ../rbac
 - ../pod
-- ../webhook
+- ../webhook/ipam
+- ../webhook/egress
 
 patchesStrategicMerge:
-- ./webhook_manifests_patch.yaml
+- egress/webhook_manifests_patch.yaml
+- ipam/webhook_manifests_patch.yaml
 
 generatorOptions:
   disableNameSuffixHash: true
 
 secretGenerator:
-- name: coilv2-webhook-server-cert
+# [IPAM] Following lines should be uncommented to enable IPAM features.
+- name: coilv2-ipam-webhook-server-cert
   files:
   - ca.crt=./cert.pem
-  - tls.crt=./cert.pem
-  - tls.key=./key.pem
+  - tls.crt=./ipam-cert.pem
+  - tls.key=./ipam-key.pem
   type: "kubernetes.io/tls"
+# [EGRESS] Following lines be uncommented to enable Egress NAT features.
+- name: coilv2-egress-webhook-server-cert
+  files:
+  - ca.crt=./cert.pem
+  - tls.crt=./egress-cert.pem
+  - tls.key=./egress-key.pem
+  type: "kubernetes.io/tls"

v2/config/pod/coil-egress-controller.yaml

@@ -0,0 +1,88 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coil-egress-controller
+  namespace: system
+  labels:
+    app.kubernetes.io/component: coil-egress-controller
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: coil-egress-controller
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: coil-egress-controller
+    spec:
+      hostNetwork: true
+      priorityClassName: system-cluster-critical
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/control-plane
+        effect: NoSchedule
+      - key: node.kubernetes.io/not-ready
+        effect: NoSchedule
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/component
+                  operator: In
+                  values: ["coil-egress-controller"]
+              topologyKey: kubernetes.io/hostname
+      securityContext:
+        runAsUser: 10000
+        runAsGroup: 10000
+      serviceAccountName: coil-egress-controller
+      terminationGracePeriodSeconds: 10
+      containers:
+      - name: coil-egress-controller
+        image: coil:dev
+        command: ["coil-egress-controller"]
+        args:
+          - --zap-stacktrace-level=panic
+        env:
+        - name: "COIL_POD_NAMESPACE"
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: "COIL_POD_NAME"
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        ports:
+        - name: metrics
+          containerPort: 9396
+          protocol: TCP
+        - name: health
+          containerPort: 9397
+          protocol: TCP
+        - name: webhook-server
+          containerPort: 9444
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: health
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: health
+        volumeMounts:
+        - mountPath: /certs
+          name: cert
+          readOnly: true
+      volumes:
+      - name: cert
+        secret:
+          defaultMode: 420
+          secretName: coilv2-egress-webhook-server-cert

v2/config/pod/coil-controller.yaml → v2/config/pod/coil-ipam-controller.yaml

@@ -1,19 +1,19 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: coil-controller
+  name: coil-ipam-controller
   namespace: system
   labels:
-    app.kubernetes.io/component: coil-controller
+    app.kubernetes.io/component: coil-ipam-controller
 spec:
   selector:
     matchLabels:
-      app.kubernetes.io/component: coil-controller
+      app.kubernetes.io/component: coil-ipam-controller
   replicas: 2
   template:
     metadata:
       labels:
-        app.kubernetes.io/component: coil-controller
+        app.kubernetes.io/component: coil-ipam-controller
     spec:
       hostNetwork: true
       priorityClassName: system-cluster-critical
@@ -33,17 +33,17 @@ spec:
                 matchExpressions:
                 - key: app.kubernetes.io/component
                   operator: In
-                  values: ["coil-controller"]
+                  values: ["coil-ipam-controller"]
               topologyKey: kubernetes.io/hostname
       securityContext:
         runAsUser: 10000
         runAsGroup: 10000
-      serviceAccountName: coil-controller
+      serviceAccountName: coil-ipam-controller
       terminationGracePeriodSeconds: 10
       containers:
-      - name: coil-controller
+      - name: coil-ipam-controller
         image: coil:dev
-        command: ["coil-controller"]
+        command: ["coil-ipam-controller"]
         args:
           - --zap-stacktrace-level=panic
         env:
@@ -87,4 +87,4 @@ spec:
       - name: cert
         secret:
           defaultMode: 420
-          secretName: coilv2-webhook-server-cert
+          secretName: coilv2-ipam-webhook-server-cert

v2/config/pod/coild.yaml

@@ -30,6 +30,8 @@ spec:
         command: ["coild"]
         args:
           - --zap-stacktrace-level=panic
+          - --enable-ipam=true
+          - --enable-egress=true
         env:
         - name: COIL_NODE_NAME
           valueFrom:

v2/config/pod/egress/v4/coild.yaml

@@ -0,0 +1,103 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: coild
+  namespace: system
+  labels:
+    app.kubernetes.io/component: coild
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: coild
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: coild
+    spec:
+      hostNetwork: true
+      hostPID: true    # to see netns file under /proc
+      priorityClassName: system-node-critical
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      serviceAccountName: coild
+      terminationGracePeriodSeconds: 1
+      containers:
+      - name: coild
+        image: coil:dev
+        command: ["coild"]
+        args:
+          - --zap-stacktrace-level=panic
+          - --enable-ipam=false
+          - --enable-egress=true
+          - --pod-table-id=0
+          - --protocol-id=2
+        env:
+        - name: COIL_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        securityContext:
+          privileged: true
+        ports:
+        - name: metrics
+          containerPort: 9384
+          protocol: TCP
+        - name: health
+          containerPort: 9385
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: health
+            host: localhost
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: health
+            host: localhost
+        volumeMounts:
+        - mountPath: /run
+          name: run
+          mountPropagation: HostToContainer  # to see bind mount netns file under /run/netns
+        - mountPath: /lib/modules
+          name: modules
+          readOnly: true
+      initContainers:
+      - name: coil-installer
+        image: coil:dev
+        command: ["coil-installer"]
+        env:
+        - name: CNI_NETCONF
+          valueFrom:
+            configMapKeyRef:
+              name: coil-config
+              key: cni_netconf
+        - name: CNI_CONF_NAME
+          value: "10-coil.conflist"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /host/opt/cni/bin
+          name: cni-bin-dir
+        - mountPath: /host/etc/cni/net.d
+          name: cni-net-dir
+      volumes:
+      - name: run
+        hostPath:
+          path: /run
+      - name: modules
+        hostPath:
+          path: /lib/modules
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-net-dir
+        hostPath:
+          path: /etc/cni/net.d

v2/config/pod/egress/v4/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+# [EGRESS] Following line should be uncommented to enable Egress NAT features.
+- ../../coil-egress-controller.yaml
+- coild.yaml

v2/config/pod/egress/v6/coild.yaml

@@ -0,0 +1,103 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: coild
+  namespace: system
+  labels:
+    app.kubernetes.io/component: coild
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: coild
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: coild
+    spec:
+      hostNetwork: true
+      hostPID: true    # to see netns file under /proc
+      priorityClassName: system-node-critical
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      serviceAccountName: coild
+      terminationGracePeriodSeconds: 1
+      containers:
+      - name: coild
+        image: coil:dev
+        command: ["coild"]
+        args:
+          - --zap-stacktrace-level=panic
+          - --enable-ipam=false
+          - --enable-egress=true
+          - --pod-table-id=255
+          - --protocol-id=2
+        env:
+        - name: COIL_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        securityContext:
+          privileged: true
+        ports:
+        - name: metrics
+          containerPort: 9384
+          protocol: TCP
+        - name: health
+          containerPort: 9385
+          protocol: TCP
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: health
+            host: localhost
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: health
+            host: localhost
+        volumeMounts:
+        - mountPath: /run
+          name: run
+          mountPropagation: HostToContainer  # to see bind mount netns file under /run/netns
+        - mountPath: /lib/modules
+          name: modules
+          readOnly: true
+      initContainers:
+      - name: coil-installer
+        image: coil:dev
+        command: ["coil-installer"]
+        env:
+        - name: CNI_NETCONF
+          valueFrom:
+            configMapKeyRef:
+              name: coil-config
+              key: cni_netconf
+        - name: CNI_CONF_NAME
+          value: "10-coil.conflist"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /host/opt/cni/bin
+          name: cni-bin-dir
+        - mountPath: /host/etc/cni/net.d
+          name: cni-net-dir
+      volumes:
+      - name: run
+        hostPath:
+          path: /run
+      - name: modules
+        hostPath:
+          path: /lib/modules
+      - name: cni-bin-dir
+        hostPath:
+          path: /opt/cni/bin
+      - name: cni-net-dir
+        hostPath:
+          path: /etc/cni/net.d

v2/config/pod/egress/v6/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+# [EGRESS] Following line should be uncommented to enable Egress NAT features.
+- ../../coil-egress-controller.yaml
+- coild.yaml

v2/config/pod/kustomization.yaml

@@ -1,3 +1,6 @@
 resources:
-- coil-controller.yaml
+# [IPAM] Next file should be uncommented to enable IPAM features.
+- coil-ipam-controller.yaml
+# [EGRESS] Following line should be uncommented to enable Egress NAT features.
+- coil-egress-controller.yaml
 - coild.yaml

v2/config/rbac/coil-controller_role.yaml → v2/config/rbac/coil-egress-controller_role.yaml

@@ -2,15 +2,8 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: coil-controller
+  name: coil-egress-controller
 rules:
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
 - apiGroups:
   - ""
   resources:
@@ -45,29 +38,6 @@ rules:
 - apiGroups:
   - coil.cybozu.com
   resources:
-  - addressblocks
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - coil.cybozu.com
-  resources:
-  - addresspools
-  verbs:
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - coil.cybozu.com
-  resources:
-  - blockrequests
   - egresses
   verbs:
   - get
@@ -76,7 +46,6 @@ rules:
 - apiGroups:
   - coil.cybozu.com
   resources:
-  - blockrequests/status
   - egresses/status
   verbs:
   - get

v2/config/rbac/coil-ipam-controller_role.yaml

@@ -0,0 +1,51 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: coil-ipam-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - coil.cybozu.com
+  resources:
+  - addressblocks
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - coil.cybozu.com
+  resources:
+  - addresspools
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - coil.cybozu.com
+  resources:
+  - blockrequests
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - coil.cybozu.com
+  resources:
+  - blockrequests/status
+  verbs:
+  - get
+  - patch
+  - update

v2/config/rbac/egress/kustomization.yaml

@@ -0,0 +1,10 @@
+resources:
+- ../serviceaccount.yaml
+- ../role_binding.yaml
+- ../leader_election_role.yaml
+- ../leader_election_role_binding.yaml
+- ../coild_role.yaml
+# [EGRESS] Following files should be uncommented to enable Egress NAT features.
+- ../coil-egress-controller_role.yaml
+- ../coil-egress_role.yaml
+- ../egress_viewer_role.yaml

v2/config/rbac/kustomization.yaml

@@ -1,13 +1,16 @@
 resources:
 - serviceaccount.yaml
-- coil-controller_role.yaml
-- coil-router_role.yaml
-- coild_role.yaml
-- coil-egress_role.yaml
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
+- coild_role.yaml
+# [IPAM] Following files should be uncommented to enable IPAM features.
+- coil-ipam-controller_role.yaml
+- coil-router_role.yaml
 - addressblock_viewer_role.yaml
 - addresspool_viewer_role.yaml
 - blockrequest_viewer_role.yaml
+# [EGRESS] Following files should be uncommented to enable Egress NAT features.
+- coil-egress-controller_role.yaml
+- coil-egress_role.yaml
 - egress_viewer_role.yaml

v2/config/rbac/leader_election_role_binding.yaml

@@ -8,5 +8,8 @@ roleRef:
   name: coil-leader-election
 subjects:
 - kind: ServiceAccount
-  name: coil-controller
+  name: coil-ipam-controller
   namespace: system
+- kind: ServiceAccount
+  name: coil-egress-controller
+  namespace: system

v2/config/rbac/role_binding.yaml

@@ -1,14 +1,27 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: coil-controller
+  name: coil-ipam-controller
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: coil-controller
+  name: coil-ipam-controller
 subjects:
 - kind: ServiceAccount
-  name: coil-controller
+  name: coil-ipam-controller
+  namespace: system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: coil-egress-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: coil-egress-controller
+subjects:
+- kind: ServiceAccount
+  name: coil-egress-controller
   namespace: system
 ---
 apiVersion: rbac.authorization.k8s.io/v1

v2/config/rbac/serviceaccount.yaml

@@ -1,7 +1,13 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: coil-controller
+  name: coil-ipam-controller
+  namespace: system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coil-egress-controller
   namespace: system
 ---
 apiVersion: v1

v2/config/webhook/kustomization.yaml → v2/config/webhook/egress/kustomization.yaml

@@ -5,4 +5,4 @@ resources:
 - service.yaml
 
 configurations:
-- kustomizeconfig.yaml
+- ../kustomizeconfig.yaml

v2/config/webhook/egress/manifests.yaml

@@ -0,0 +1,53 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: mutating-egress-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: egress-webhook-service
+      namespace: system
+      path: /mutate-coil-cybozu-com-v2-egress
+  failurePolicy: Fail
+  name: megress.kb.io
+  rules:
+  - apiGroups:
+    - coil.cybozu.com
+    apiVersions:
+    - v2
+    operations:
+    - CREATE
+    resources:
+    - egresses
+  sideEffects: None
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validating-egress-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: egress-webhook-service
+      namespace: system
+      path: /validate-coil-cybozu-com-v2-egress
+  failurePolicy: Fail
+  name: vegress.kb.io
+  rules:
+  - apiGroups:
+    - coil.cybozu.com
+    apiVersions:
+    - v2
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - egresses
+  sideEffects: None

v2/config/webhook/egress/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: egress-webhook-service
+  namespace: system
+spec:
+  ports:
+    - port: 443
+      targetPort: 9444
+      protocol: TCP
+  selector:
+    app.kubernetes.io/component: coil-egress-controller

v2/config/webhook/ipam/kustomization.yaml

@@ -0,0 +1,8 @@
+namePrefix: coilv2-
+
+resources:
+- manifests.yaml
+- service.yaml
+
+configurations:
+- ../kustomizeconfig.yaml

v2/config/webhook/manifests.v1.yaml → v2/config/webhook/ipam/manifests.yaml

@@ -2,14 +2,14 @@
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
-  name: mutating-webhook-configuration
+  name: mutating-ipam-webhook-configuration
 webhooks:
 - admissionReviewVersions:
   - v1
   - v1beta1
   clientConfig:
     service:
-      name: webhook-service
+      name: ipam-webhook-service
       namespace: system
       path: /mutate-coil-cybozu-com-v2-addresspool
   failurePolicy: Fail
@@ -24,38 +24,18 @@ webhooks:
     resources:
     - addresspools
   sideEffects: None
-- admissionReviewVersions:
-  - v1
-  - v1beta1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /mutate-coil-cybozu-com-v2-egress
-  failurePolicy: Fail
-  name: megress.kb.io
-  rules:
-  - apiGroups:
-    - coil.cybozu.com
-    apiVersions:
-    - v2
-    operations:
-    - CREATE
-    resources:
-    - egresses
-  sideEffects: None
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
-  name: validating-webhook-configuration
+  name: validating-ipam-webhook-configuration
 webhooks:
 - admissionReviewVersions:
   - v1
   - v1beta1
   clientConfig:
     service:
-      name: webhook-service
+      name: ipam-webhook-service
       namespace: system
       path: /validate-coil-cybozu-com-v2-addresspool
   failurePolicy: Fail
@@ -71,24 +51,3 @@ webhooks:
     resources:
     - addresspools
   sideEffects: None
-- admissionReviewVersions:
-  - v1
-  - v1beta1
-  clientConfig:
-    service:
-      name: webhook-service
-      namespace: system
-      path: /validate-coil-cybozu-com-v2-egress
-  failurePolicy: Fail
-  name: vegress.kb.io
-  rules:
-  - apiGroups:
-    - coil.cybozu.com
-    apiVersions:
-    - v2
-    operations:
-    - CREATE
-    - UPDATE
-    resources:
-    - egresses
-  sideEffects: None

v2/config/webhook/service.yaml → v2/config/webhook/ipam/service.yaml

@@ -2,12 +2,12 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: webhook-service
+  name: ipam-webhook-service
   namespace: system
 spec:
   ports:
     - port: 443
       targetPort: 9443
       protocol: TCP
   selector:
-    app.kubernetes.io/component: coil-controller
+    app.kubernetes.io/component: coil-ipam-controller

v2/controllers/egress_watcher.go

@@ -125,7 +125,7 @@ func (r *EgressWatcher) reconcileEgressClient(ctx context.Context, eg *coilv2.Eg
 			ipv6 = ip.To16()
 		}
 	}
-	if err := r.PodNet.Update(ipv4, ipv6, hook); err != nil {
+	if err := r.PodNet.Update(ipv4, ipv6, hook, pod); err != nil {
 		return fmt.Errorf("failed to update NAT configuration: %w", err)
 	}
 

v2/controllers/egress_watcher_test.go

@@ -180,11 +180,15 @@ func (p *mockPodNetwork) List() ([]*nodenet.PodNetConf, error) {
 	panic("not implemented")
 }
 
-func (p *mockPodNetwork) Setup(nsPath, podName, podNS string, conf *nodenet.PodNetConf, hook nodenet.SetupHook) (*current.Result, error) {
+func (p *mockPodNetwork) SetupIPAM(nsPath, podName, podNS string, conf *nodenet.PodNetConf) (*current.Result, error) {
 	panic("not implemented")
 }
 
-func (p *mockPodNetwork) Update(podIPv4, podIPv6 net.IP, hook nodenet.SetupHook) error {
+func (p *mockPodNetwork) SetupEgress(nsPath string, conf *nodenet.PodNetConf, hook nodenet.SetupHook) error {
+	panic("not implemented")
+}
+
+func (p *mockPodNetwork) Update(podIPv4, podIPv6 net.IP, hook nodenet.SetupHook, pod *corev1.Pod) error {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 

v2/e2e/Makefile

@@ -10,9 +10,19 @@ export KUBECTL
 
 KIND_CONFIG = kind-config.yaml
 ifeq ($(TEST_IPV6),true)
-KIND_CONFIG = kind-config_v6.yaml
+	ifeq ($(WITH_KINDNET),true)
+		KIND_CONFIG = kind-config_kindnet_v6.yaml
+	else 
+		KIND_CONFIG = kind-config_v6.yaml
+	endif
+else
+	ifeq ($(WITH_KINDNET),true)
+		KIND_CONFIG = kind-config_kindnet.yaml
+	endif
 endif
 
+TIMEOUT=0
+
 .PHONY: start
 start: $(KIND) $(KUBECTL) $(KUSTOMIZE)
 	$(KIND) create cluster --image kindest/node:v$(KUBERNETES_VERSION) --name coil --config $(KIND_CONFIG)
@@ -22,21 +32,66 @@ stop: $(KIND)
 	$(KIND) delete cluster --name coil
 
 .PHONY: install-coil
-install-coil:
+install-coil: setup-nodes
+	$(KIND) load docker-image --name coil coil:dev
+	$(KUSTOMIZE) build --load-restrictor=LoadRestrictionsNone . | $(KUBECTL) apply -f -
+	$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-ipam-controller
+	$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-egress-controller
+
+.PHONY: install-coil-egress-v4
+install-coil-egress-v4: setup-nodes
+	rm -rf tmp
+	mkdir tmp 2> /dev/null
+	CGO_ENABLED=0 go build -o kindnet-conf ./kindnet-configurer
+	$(KUBECTL) rollout status daemonset kindnet -n kube-system --timeout 120s
+	./kindnet-conf --action get
+	$(KIND) load docker-image --name coil coil:dev
+	$(KUSTOMIZE) build --load-restrictor=LoadRestrictionsNone configs/egress/v4 | $(KUBECTL) apply -f -
+	$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-egress-controller
+	$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-egress-controller
+	$(KUBECTL) rollout status daemonset coild -n kube-system --timeout=3m
+	./kindnet-conf --action set --file 10-coil.conflist
+	rm -rf tmp kindnet-conf
+
+.PHONY: install-coil-egress-v6
+install-coil-egress-v6: setup-nodes
+	rm -rf tmp
+	mkdir tmp 2> /dev/null
+	$(KUBECTL) rollout status daemonset kindnet -n kube-system --timeout 120s
+	CGO_ENABLED=0 go build -o kindnet-conf ./kindnet-configurer
+	./kindnet-conf --action get --protocol v6
+	$(KIND) load docker-image --name coil coil:dev
+	$(KUSTOMIZE) build --load-restrictor=LoadRestrictionsNone configs/egress/v6 | $(KUBECTL) apply -f -
+	$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-egress-controller
+	$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-egress-controller
+	$(KUBECTL) rollout status daemonset coild -n kube-system --timeout=3m
+	./kindnet-conf --action set --file 10-coil.conflist --protocol v6
+	rm -rf tmp kindnet-conf
+
+.PHONY: setup-nodes
+setup-nodes:
 	$(KUBECTL) taint nodes coil-worker test:NoSchedule
 	$(KUBECTL) taint nodes coil-worker2 test:NoSchedule
 	$(KUBECTL) label nodes coil-worker test=coil
 	$(KUBECTL) label nodes coil-worker2 test=coil
-	$(KIND) load docker-image --name coil coil:dev
-	$(KUSTOMIZE) build --load-restrictor=LoadRestrictionsNone . | $(KUBECTL) apply -f -
-	$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-controller
 
 .PHONY: test
-test:
+test: setup-echotest
+	TEST_IPAM=$(TEST_IPAM) TEST_EGRESS=$(TEST_EGRESS) go test -count 1 -v . -args -ginkgo.progress -ginkgo.v
+
+.PHONY: test-ipam
+test-ipam: setup-echotest
+	TEST_IPAM=$(TEST_IPAM) TEST_EGRESS=$(TEST_EGRESS) go test -count 1 -v . -args -ginkgo.progress -ginkgo.v
+
+.PHONY: test-egress
+test-egress: setup-echotest
+	TEST_IPAM=$(TEST_IPAM) TEST_EGRESS=$(TEST_EGRESS) go test -count 1 -v . -args -ginkgo.progress -ginkgo.v
+
+.PHONY: setup-echotest
+setup-echotest:
 	CGO_ENABLED=0 go build -o echotest ./echo-server
 	docker cp echotest coil-control-plane:/usr/local/bin
 	rm echotest
-	go test -count 1 -v . -args -ginkgo.progress -ginkgo.v
 
 .PHONY: logs
 logs:

v2/e2e/coil-controller_patch.yaml

@@ -1,11 +1,11 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: coil-controller
+  name: coil-ipam-controller
   namespace: system
 spec:
   template:
     spec:
       containers:
-      - name: coil-controller
+      - name: coil-ipam-controller
         args: ["--gc-interval=10s"]

v2/e2e/coil_test.go

@@ -18,9 +18,23 @@ import (
 	policyv1 "k8s.io/api/policy/v1"
 )
 
-var testIPv6 = os.Getenv("TEST_IPV6") == "true"
+var (
+	enableIPv6Tests   = os.Getenv(testIPv6Key) == "true"
+	enableIPAMTests   = os.Getenv(testIPAMKey) == "true"
+	enableEgressTests = os.Getenv(testEgressKey) == "true"
+)
+
+var _ = Describe("coil", func() {
+	if enableIPAMTests {
+		Context("when the IPAM features are enabled", testIPAM)
+	}
+	if enableEgressTests {
+		Context("when egress feature is enabled", testEgress)
+	}
+	Context("when coild is deployed", testCoild)
+})
 
-var _ = Describe("Coil", func() {
+func testIPAM() {
 	It("should run health probe servers", func() {
 		By("checking all pods get ready")
 		Eventually(func() error {
@@ -51,24 +65,6 @@ var _ = Describe("Coil", func() {
 		}).Should(Succeed())
 	})
 
-	It("should export metrics", func() {
-		By("checking port 9384 for coild")
-		out, err := runOnNode("coil-worker", "curl", "-sf", "http://localhost:9384/metrics")
-		Expect(err).ShouldNot(HaveOccurred())
-
-		mfs, err := (&expfmt.TextParser{}).TextToMetricFamilies(bytes.NewReader(out))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(mfs).NotTo(BeEmpty())
-
-		By("checking port 9388 for coil-router")
-		out, err = runOnNode("coil-worker", "curl", "-sf", "http://localhost:9388/metrics")
-		Expect(err).ShouldNot(HaveOccurred())
-
-		mfs, err = (&expfmt.TextParser{}).TextToMetricFamilies(bytes.NewReader(out))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(mfs).NotTo(BeEmpty())
-	})
-
 	// This series of tests confirms the following things:
 	// - coil can call coild gRPC method appropriately
 	// - coild runs gRPC server
@@ -77,7 +73,7 @@ var _ = Describe("Coil", func() {
 	It("should allow pods on different nodes to communicate", func() {
 		By("creating the default pool")
 		manifest := "manifests/default_pool.yaml"
-		if testIPv6 {
+		if enableIPv6Tests {
 			manifest = "manifests/default_pool_v6.yaml"
 		}
 		kubectlSafe(nil, "apply", "-f", manifest)
@@ -122,7 +118,7 @@ var _ = Describe("Coil", func() {
 
 		By("checking communication between pods on different nodes")
 		var testURL string
-		if testIPv6 {
+		if enableIPv6Tests {
 			testURL = fmt.Sprintf("http://[%s]:8000", httpdIP)
 		} else {
 			testURL = fmt.Sprintf("http://%s:8000", httpdIP)
@@ -204,7 +200,7 @@ var _ = Describe("Coil", func() {
 
 	It("should export routes to routing table 119", func() {
 		var ipOpt string
-		if testIPv6 {
+		if enableIPv6Tests {
 			ipOpt = "-6"
 		} else {
 			ipOpt = "-4"
@@ -244,6 +240,56 @@ var _ = Describe("Coil", func() {
 		}).Should(Succeed())
 	})
 
+	It("should export metrics", func() {
+		By("checking port 9388 for coil-router")
+		out, err := runOnNode("coil-worker", "curl", "-sf", "http://localhost:9388/metrics")
+		Expect(err).ShouldNot(HaveOccurred())
+
+		mfs, err := (&expfmt.TextParser{}).TextToMetricFamilies(bytes.NewReader(out))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(mfs).NotTo(BeEmpty())
+
+	})
+
+	It("should delete address pool", func() {
+		By("creating a dummy address pool")
+		_, err := kubectl(nil, "apply", "-f", "manifests/dummy_pool.yaml")
+		Expect(err).NotTo(HaveOccurred())
+
+		Eventually(func() error {
+			ap := coilv2.AddressPool{}
+			out, err := kubectl(nil, "get", "addresspool", "dummy", "-o", "json")
+			if err != nil {
+				return err
+			}
+			if err := json.Unmarshal(out, &ap); err != nil {
+				return err
+			}
+			return nil
+		}).Should(Succeed())
+
+		By("deleting the dummy address pool")
+		_, err = kubectl(nil, "delete", "-f", "manifests/dummy_pool.yaml")
+		Expect(err).NotTo(HaveOccurred())
+
+		Consistently(func() error {
+			apList := coilv2.AddressPoolList{}
+			out, err := kubectl(nil, "get", "addresspool", "-o", "json")
+			if err != nil {
+				return err
+			}
+			if err := json.Unmarshal(out, &apList); err != nil {
+				return err
+			}
+			if len(apList.Items) == 1 {
+				return nil
+			}
+			return fmt.Errorf("the number of AddressPool must be 1")
+		}).Should(Succeed())
+	})
+}
+
+func testEgress() {
 	It("should be able to run Egress pods", func() {
 		By("defining Egress in the internet namespace")
 		kubectlSafe(nil, "apply", "-f", "manifests/egress.yaml")
@@ -345,7 +391,7 @@ var _ = Describe("Coil", func() {
 
 	It("should allow NAT traffic over foo-over-udp tunnel", func() {
 		var fakeIP, fakeURL, ipOpt string
-		if testIPv6 {
+		if enableIPv6Tests {
 			fakeIP = "2606:4700:4700::9999"
 			fakeURL = fmt.Sprintf("http://[%s]", fakeIP)
 			ipOpt = "-6"
@@ -360,39 +406,50 @@ var _ = Describe("Coil", func() {
 		Expect(err).NotTo(HaveOccurred())
 		_, err = runOnNode("coil-control-plane", "ip", "link", "set", "dummy-fake", "up")
 		Expect(err).NotTo(HaveOccurred())
-		if testIPv6 {
+		if enableIPv6Tests {
 			_, err = runOnNode("coil-control-plane", "ip", "address", "add", fakeIP+"/128", "dev", "dummy-fake", "nodad")
 		} else {
 			_, err = runOnNode("coil-control-plane", "ip", "address", "add", fakeIP+"/32", "dev", "dummy-fake")
 		}
 		Expect(err).NotTo(HaveOccurred())
 
+		natAddresses := []string{}
+		if !enableIPAMTests {
+			natAddresses = getNATAddresses("egress")
+		}
+
 		By("running HTTP server on coil-control-plane")
-		go runOnNode("coil-control-plane", "/usr/local/bin/echotest")
+		if enableIPAMTests {
+			go runOnNode("coil-control-plane", "/usr/local/bin/echotest")
+		} else {
+			go runOnNode("coil-control-plane", "/usr/local/bin/echotest", "--reply-remote")
+		}
+
 		time.Sleep(100 * time.Millisecond)
 
 		By("sending and receiving HTTP request from nat-client")
 		data := make([]byte, 1<<20) // 1 MiB
-		resp := kubectlSafe(data, "exec", "-i", "nat-client", "--", "curl", "-sf", "-T", "-", fakeURL)
-		Expect(resp).To(HaveLen(1 << 20))
+		testNAT(data, "nat-client", fakeURL, natAddresses, enableIPAMTests)
 
 		By("running the same test 100 times")
 		for i := 0; i < 100; i++ {
 			time.Sleep(1 * time.Millisecond)
-			resp := kubectlSafe(data, "exec", "-i", "nat-client", "--", "curl", "-sf", "-T", "-", fakeURL)
-			Expect(resp).To(HaveLen(1 << 20))
+			testNAT(data, "nat-client", fakeURL, natAddresses, enableIPAMTests)
+		}
+
+		natAddresses = []string{}
+		if !enableIPAMTests {
+			natAddresses = getNATAddresses("egress-sport-auto")
 		}
 
 		By("sending and receiving HTTP request from nat-client-sport-auto")
 		data = make([]byte, 1<<20) // 1 MiB
-		resp = kubectlSafe(data, "exec", "-i", "nat-client-sport-auto", "--", "curl", "-sf", "-T", "-", fakeURL)
-		Expect(resp).To(HaveLen(1 << 20))
+		testNAT(data, "nat-client-sport-auto", fakeURL, natAddresses, enableIPAMTests)
 
 		By("running the same test 100 times with nat-client-sport-auto")
 		for i := 0; i < 100; i++ {
 			time.Sleep(1 * time.Millisecond)
-			resp := kubectlSafe(data, "exec", "-i", "nat-client-sport-auto", "--", "curl", "-sf", "-T", "-", fakeURL)
-			Expect(resp).To(HaveLen(1 << 20))
+			testNAT(data, "nat-client-sport-auto", fakeURL, natAddresses, enableIPAMTests)
 		}
 
 		By("creating a dummy pod don't use egress")
@@ -492,53 +549,58 @@ var _ = Describe("Coil", func() {
 		Expect(fouCount).To(Equal(1))
 
 		By("sending and receiving HTTP request from nat-client")
+
+		natAddresses = []string{}
+		if !enableIPAMTests {
+			natAddresses = getNATAddresses("egress")
+		}
+
 		data = make([]byte, 1<<20) // 1 MiB
-		resp = kubectlSafe(data, "exec", "-i", "nat-client", "--", "curl", "-sf", "-T", "-", fakeURL)
-		Expect(resp).To(HaveLen(1 << 20))
+		testNAT(data, "nat-client", fakeURL, natAddresses, enableIPAMTests)
 
 		By("running the same test 100 times")
 		for i := 0; i < 100; i++ {
 			time.Sleep(1 * time.Millisecond)
-			resp := kubectlSafe(data, "exec", "-i", "nat-client", "--", "curl", "-sf", "-T", "-", fakeURL)
-			Expect(resp).To(HaveLen(1 << 20))
+			testNAT(data, "nat-client", fakeURL, natAddresses, enableIPAMTests)
 		}
 	})
+}
 
-	It("should delete address pool", func() {
-		By("creating a dummy address pool")
-		_, err := kubectl(nil, "apply", "-f", "manifests/dummy_pool.yaml")
-		Expect(err).NotTo(HaveOccurred())
-
-		Eventually(func() error {
-			ap := coilv2.AddressPool{}
-			out, err := kubectl(nil, "get", "addresspool", "dummy", "-o", "json")
-			if err != nil {
-				return err
-			}
-			if err := json.Unmarshal(out, &ap); err != nil {
-				return err
-			}
-			return nil
-		}).Should(Succeed())
+func testCoild() {
+	It("should export metrics", func() {
+		By("checking port 9384 for coild")
+		out, err := runOnNode("coil-worker", "curl", "-sf", "http://localhost:9384/metrics")
+		Expect(err).ShouldNot(HaveOccurred())
 
-		By("deleting the dummy address pool")
-		_, err = kubectl(nil, "delete", "-f", "manifests/dummy_pool.yaml")
+		mfs, err := (&expfmt.TextParser{}).TextToMetricFamilies(bytes.NewReader(out))
 		Expect(err).NotTo(HaveOccurred())
-
-		Consistently(func() error {
-			apList := coilv2.AddressPoolList{}
-			out, err := kubectl(nil, "get", "addresspool", "-o", "json")
-			if err != nil {
-				return err
-			}
-			if err := json.Unmarshal(out, &apList); err != nil {
-				return err
-			}
-			if len(apList.Items) == 1 {
-				return nil
-			}
-			return fmt.Errorf("the number of AddressPool must be 1")
-		}).Should(Succeed())
-
+		Expect(mfs).NotTo(BeEmpty())
 	})
-})
+}
+
+func testNAT(data []byte, clientPod, fakeURL string, natAddresses []string, ipamEnabled bool) {
+	resp := kubectlSafe(data, "exec", "-i", clientPod, "--", "curl", "-sf", "-T", "-", fakeURL)
+
+	if !ipamEnabled {
+		respStr := string(resp)
+		idx := strings.Index(respStr, "|")
+		ipAddr := respStr[:idx]
+		resp = []byte(respStr[idx+1:])
+		Expect(natAddresses).To(ContainElement(ipAddr))
+	}
+	Expect(resp).To(HaveLen(1 << 20))
+}
+
+func getNATAddresses(name string) []string {
+	eps := &corev1.Endpoints{}
+	err := getResource("internet", "endpoints", name, "", eps)
+	Expect(err).ToNot(HaveOccurred())
+
+	natAddresses := []string{}
+	for _, s := range eps.Subsets {
+		for _, a := range s.Addresses {
+			natAddresses = append(natAddresses, a.IP)
+		}
+	}
+	return natAddresses
+}

v2/e2e/configs/egress/v4/kustomization.yaml

@@ -0,0 +1,15 @@
+resources:
+- ../../../../config/default/egress/v4
+
+configMapGenerator:
+- name: coil-config
+  namespace: system
+  files:
+  - cni_netconf=../../../netconf/netconf-kindnet-v4.json
+
+# Adds namespace to all resources.
+namespace: kube-system
+
+# Labels to add to all resources and selectors.
+commonLabels:
+  app.kubernetes.io/name: coil

v2/e2e/configs/egress/v6/kustomization.yaml

@@ -0,0 +1,15 @@
+resources:
+- ../../../../config/default/egress/v6
+
+configMapGenerator:
+- name: coil-config
+  namespace: system
+  files:
+  - cni_netconf=../../../netconf/netconf-kindnet-v6.json
+
+# Adds namespace to all resources.
+namespace: kube-system
+
+# Labels to add to all resources and selectors.
+commonLabels:
+  app.kubernetes.io/name: coil

v2/e2e/controller_test.go

@@ -2,6 +2,8 @@ package e2e
 
 import (
 	"bytes"
+	"fmt"
+	"os"
 
 	coilv2 "github.com/cybozu-go/coil/v2/api/v2"
 	. "github.com/onsi/ginkgo/v2"
@@ -10,9 +12,24 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
-var _ = Describe("coil-controller", func() {
+const (
+	testIPv6Key   = "TEST_IPV6"
+	testIPAMKey   = "TEST_IPAM"
+	testEgressKey = "TEST_EGRESS"
+)
+
+var _ = Describe("coil controllers", func() {
+	if os.Getenv(testIPAMKey) == "true" {
+		Context("when the IPAM features are enabled", testCoilIPAMController)
+	}
+	if os.Getenv(testEgressKey) == "true" {
+		Context("when egress feature is enabled", testCoilEgressController)
+	}
+})
+
+func testCoilIPAMController() {
 	It("should elect a leader instance of coil-controller", func() {
-		kubectlSafe(nil, "-n", "kube-system", "get", "leases", "coil-leader")
+		kubectlSafe(nil, "-n", "kube-system", "get", "leases", "coil-ipam-leader")
 	})
 
 	It("should run the admission webhook", func() {
@@ -23,7 +40,7 @@ var _ = Describe("coil-controller", func() {
 
 	It("should export metrics", func() {
 		pods := &corev1.PodList{}
-		getResourceSafe("kube-system", "pods", "", "app.kubernetes.io/component=coil-controller", pods)
+		getResourceSafe("kube-system", "pods", "", "app.kubernetes.io/component=coil-ipam-controller", pods)
 		Expect(pods.Items).Should(HaveLen(2))
 
 		node := pods.Items[0].Spec.NodeName
@@ -49,4 +66,32 @@ var _ = Describe("coil-controller", func() {
 			return abl.Items
 		}, 20).Should(BeEmpty())
 	})
-})
+}
+
+func testCoilEgressController() {
+	Context("when the egress features are enabled", func() {
+		It("should elect a leader instance of coil-controller", func() {
+			kubectlSafe(nil, "-n", "kube-system", "get", "leases", "coil-egress-leader")
+		})
+
+		It("should export metrics", func() {
+			pods := &corev1.PodList{}
+			getResourceSafe("kube-system", "pods", "", "app.kubernetes.io/component=coil-egress-controller", pods)
+			Expect(pods.Items).Should(HaveLen(2))
+
+			node := pods.Items[0].Spec.NodeName
+
+			address := fmt.Sprintf("http://%s:9396/metrics", pods.Items[0].Status.PodIP)
+			if enableIPv6Tests {
+				address = fmt.Sprintf("http://[%s]:9396/metrics", pods.Items[0].Status.PodIP)
+			}
+
+			out, err := runOnNode(node, "curl", "-sf", address)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			mfs, err := (&expfmt.TextParser{}).TextToMetricFamilies(bytes.NewReader(out))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(mfs).NotTo(BeEmpty())
+		})
+	})
+}

v2/e2e/echo-server/main.go

@@ -1,26 +1,46 @@
 package main
 
 import (
+	"flag"
 	"io"
+	"net"
 	"net/http"
 )
 
-type echoHandler struct{}
+type echoHandler struct {
+	withRemoteAddrReply bool
+}
 
-func (echoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+func (h echoHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	body, err := io.ReadAll(req.Body)
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
 	w.Header().Set("content-type", "application/octet-stream")
+
+	if h.withRemoteAddrReply {
+		remote, _, err := net.SplitHostPort(req.RemoteAddr)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		remote += "|"
+		w.Write([]byte(remote))
+	}
 	w.Write(body)
 }
 
 func main() {
+	var withRemoteAddress bool
+	flag.BoolVar(&withRemoteAddress, "reply-remote", false, "if set, echo-server will reply with remote host address (default: false)")
+	flag.Parse()
+
 	s := &http.Server{
-		Handler: echoHandler{},
+		Handler: echoHandler{
+			withRemoteAddrReply: withRemoteAddress,
+		},
 	}
 	s.ListenAndServe()
 }

v2/e2e/kind-config_kindnet.yaml

@@ -0,0 +1,7 @@
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+nodes:
+- role: control-plane
+- role: worker
+- role: worker
+- role: worker

v2/e2e/kind-config_kindnet_v6.yaml

@@ -0,0 +1,9 @@
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+networking:
+  ipFamily: ipv6
+nodes:
+- role: control-plane
+- role: worker
+- role: worker
+- role: worker

v2/e2e/kindnet-configurer/main.go

@@ -0,0 +1,143 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	containerName   = "coil-worker"
+	ipv4            = "v4"
+	ipv6            = "v6"
+	getAction       = "get"
+	setAction       = "set"
+	defaultConflist = "10-kindnet.conflist"
+	numOfNodes      = 4
+)
+
+func main() {
+	action := flag.String("action", getAction, "Action to perform (get/set)")
+	container := flag.String("container", containerName, "Base name of the container to use")
+	protocol := flag.String("protocol", ipv4, "Version of IP protocol to use")
+	file := flag.String("file", defaultConflist, "CNI config file to edit")
+
+	flag.Parse()
+
+	if *protocol != ipv4 && *protocol != ipv6 {
+		log.Fatalf("invalid protocol [%s]", *protocol)
+	}
+
+	switch *action {
+	case getAction:
+		if err := get(*file, *protocol, *container); err != nil {
+			log.Fatal(err)
+		}
+	case setAction:
+		if err := set(*file, *protocol, *container); err != nil {
+			log.Fatal(err)
+		}
+	default:
+		log.Fatalf("command [%s] not supported", *action)
+	}
+}
+
+func get(conflistName, protoVer, contianerBase string) error {
+	path := filepath.Join("tmp", "networks")
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	address := "10.244."
+	if protoVer == ipv6 {
+		address = "fd00:10:244:"
+	}
+
+	for i := 1; i < numOfNodes; i++ {
+		container := contianerBase
+		if i > 1 {
+			container += strconv.Itoa(i)
+		}
+		var err error
+		var output string
+		var errOutput string
+		for i := 0; i < 120; i++ {
+			cmd := exec.Command("docker", "exec", container, "cat", "/etc/cni/net.d/"+conflistName)
+			var buffer bytes.Buffer
+			cmd.Stdout = &buffer
+			var bufferErr bytes.Buffer
+			cmd.Stderr = &bufferErr
+			if err = cmd.Run(); err != nil {
+				errOutput = bufferErr.String()
+				fmt.Printf("Error: %s: %s\n", err.Error(), errOutput)
+				fmt.Println("Retrying...")
+				time.Sleep(time.Second)
+			} else {
+				output = buffer.String()
+				break
+			}
+		}
+
+		if err != nil {
+			return fmt.Errorf("error: %w: %s", err, errOutput)
+		}
+
+		start := strings.Index(output, address)
+		end := start + 10
+		if protoVer == ipv6 {
+			end = strings.Index(output, "/64")
+		}
+
+		network := output[start:end]
+		key := strings.ToUpper(container) + "_NETWORK"
+		if err := os.Setenv(key, network); err != nil {
+			return fmt.Errorf("failed to set env [%s]: %w", key, err)
+		}
+		if _, err := fmt.Fprintln(f, network); err != nil {
+			return fmt.Errorf("failed to write temporary file %s: %w", path, err)
+		}
+	}
+
+	return nil
+}
+
+func set(conflistName, protoVer, contianerBase string) error {
+	f, err := os.Open(filepath.Join("tmp", "networks"))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+
+	for i := 1; i < numOfNodes; i++ {
+		container := contianerBase
+		if i > 1 {
+			container += strconv.Itoa(i)
+		}
+		scanner.Scan()
+		network := scanner.Text()
+		fmt.Printf("%s: %s\n", strings.ToUpper(container)+"_NETWORK", network)
+		reg := fmt.Sprintf("s/10\\.244\\.0\\.0/%s/", network)
+		if protoVer == ipv6 {
+			reg = fmt.Sprintf("s/fd00\\:10\\:244\\:\\:/%s/", network)
+		}
+		cmd := exec.Command("docker", "exec", container, "sed", "-i", reg, "/etc/cni/net.d/"+conflistName)
+		var bufferErr bytes.Buffer
+		cmd.Stderr = &bufferErr
+		if err := cmd.Run(); err != nil {
+			return fmt.Errorf("error: %w: %s", err, bufferErr.String())
+		}
+	}
+	return nil
+}

v2/e2e/netconf/netconf-kindnet-v4.json

@@ -0,0 +1,33 @@
+{
+  "cniVersion": "0.3.1",
+  "name": "kindnet",
+  "plugins": [
+    {
+      "type": "ptp",
+      "ipMasq": false,
+      "ipam": {
+              "type": "host-local",
+              "dataDir": "/run/cni-ipam-state",
+              "routes": [
+                      { "dst": "0.0.0.0/0" }
+              ],
+              "ranges": [
+                      [ { "subnet": "10.244.0.0/24" } ]
+              ]
+      }
+      ,
+      "mtu": 1500
+
+    }, 
+    {
+      "type": "coil",
+      "socket": "/run/coild.sock"
+    },
+    {
+      "type": "portmap",
+      "capabilities": {
+        "portMappings": true
+      }
+    }
+  ]
+}

v2/e2e/netconf/netconf-kindnet-v6.json

@@ -0,0 +1,31 @@
+{
+  "cniVersion": "0.3.1",
+  "name": "kindnet",
+  "plugins": [
+    {
+      "type": "ptp",
+      "ipMasq": false,
+      "ipam": {
+              "type": "host-local",
+              "dataDir": "/run/cni-ipam-state",
+              "routes": [
+                      { "dst": "::/0" }
+              ],
+              "ranges": [
+                      [ { "subnet": "fd00:10:244::/64" } ]
+              ]
+      },
+      "mtu": 1500
+    }, 
+    {
+      "type": "coil",
+      "socket": "/run/coild.sock"
+    },
+    {
+      "type": "portmap",
+      "capabilities": {
+        "portMappings": true
+      }
+    }
+  ]
+}

v2/kustomization.yaml

@@ -5,7 +5,7 @@ images:
 
 resources:
 - config/default
-# If you are using CKE (github.com/cybozu-go/cke) and wwant to use
+# If you are using CKE (github.com/cybozu-go/cke) and want to use
 # its webhook installation feature, comment the above line and
 # uncomment the below line.
 #- config/cke

v2/pkg/cnirpc/cni.proto

@@ -14,6 +14,8 @@ message CNIArgs {
   map<string,string> args = 4;  // Key-Value pairs parsed from CNI_ARGS
   string path = 5;
   bytes stdin_data = 6;
+  repeated string ips = 7;
+  map<string,bool> interfaces = 8;
 }
 
 // ErrorCode enumerates errors for CNIError