From 42ec518e10d70d4e8f45bb5041a4518077729aa8 Mon Sep 17 00:00:00 2001 From: Jason Rokeach <1076569+jrokeach@users.noreply.github.com> Date: Fri, 5 Jan 2024 16:20:19 -0500 Subject: [PATCH] v0.2.0 Pre-release (#27) * Sidecar/init containers, 23.1, core example update * Clean up unused env vars * Distinct overrides/ dir for ro/configmap mounts * 2regions-hrr ds/lothlorien-a for sidecar/init * 2regions-hrr ds/lothlorien-b for sidecar/init * Updated image paths * Don't restart meshrr sidecars pending crpd * Mirkwood on LBs and multiple containers * Image pull cleanup * Restructuring * Initial YAML-driven config; 2region-hrr core only * bgpgroups to j2 as dict * Mirkwood example and configmap compatibility * pylint * Lothlorien example w/ YAML * Documentation updates * Minor doc update * 2regions-hrr docs update * Timestamps * GitHub Action to push container image * Docs update * Minor docs update * Docs updates * Container image tag normalization * Add cRPD image import instructions * Fix Junos config syntax error for evpnrs * Update A side evpnrs use case * b side evpnrs * Remove regions from routeserver example * 3 client groups (#24) * Base 3clientgroups example * Junos config cleanup * Routeservers readme update * Removed references to regions from RS example * Update manifests to v0.2 image tag --------- Co-authored-by: Jason R. Rokeach --- .github/workflows/publish-image.yml | 62 +++ LICENSE | 2 +- README.md | 137 +++-- examples/2regions-hrr/README.md | 131 ++--- examples/2regions-hrr/meshrr-core.yaml | 114 +++- examples/2regions-hrr/meshrr-lothlorien.yaml | 234 +++++--- examples/2regions-hrr/meshrr-mirkwood.yaml | 306 ++++++++--- .../templates/lothlorien-config.j2 | 29 +- .../2regions-hrr/templates/mirkwood-config.j2 | 29 +- examples/3clientgroups/bgppeer-global.yml | 10 + .../3clientgroups/meshrr-3clientgroups.yaml | 513 ++++++++++++++++++ .../meshrr-defaultonly-clients.yml | 26 + .../meshrr-fulltable-clients.yml | 25 + .../meshrr-partialtable-clients.yml | 25 + .../load-balanced-route-servers/README.md | 100 +++- .../bgppeer-global.yml | 10 +- .../meshrr-core.service.yml | 44 +- .../routeserver-1-a.ss.yml | 208 ------- .../routeserver-1-b.ss.yml | 186 ------- .../routeserver-2-a.ss.yml | 208 ------- .../routeserver-2-b.ss.yml | 186 ------- .../routeserver-a.ss.yml | 247 +++++++++ .../routeserver-b.ss.yml | 203 +++++++ meshrr/Dockerfile | 42 +- meshrr/LICENSE | 21 + meshrr/config.py | 91 ++++ meshrr/{runit-init.sh => connect_wait.py} | 54 +- meshrr/defaults/juniper-evpnrs.conf.j2 | 87 +++ meshrr/defaults/juniper-ipv4rr.conf.j2 | 76 +++ meshrr/defaults/meshrr.conf.yml | 22 + meshrr/juniper.conf.j2 | 81 --- meshrr/render_config.py | 36 +- meshrr/requirements.txt | 42 +- meshrr/run.sh | 77 +++ meshrr/update_peers.py | 200 +++---- 35 files changed, 2417 insertions(+), 1447 deletions(-) create mode 100644 .github/workflows/publish-image.yml create mode 100644 examples/3clientgroups/bgppeer-global.yml create mode 100644 examples/3clientgroups/meshrr-3clientgroups.yaml create mode 100644 examples/3clientgroups/meshrr-defaultonly-clients.yml create mode 100644 examples/3clientgroups/meshrr-fulltable-clients.yml create mode 100644 examples/3clientgroups/meshrr-partialtable-clients.yml delete mode 100644 examples/load-balanced-route-servers/routeserver-1-a.ss.yml delete mode 100644 examples/load-balanced-route-servers/routeserver-1-b.ss.yml delete mode 100644 examples/load-balanced-route-servers/routeserver-2-a.ss.yml delete mode 100644 examples/load-balanced-route-servers/routeserver-2-b.ss.yml create mode 100644 examples/load-balanced-route-servers/routeserver-a.ss.yml create mode 100644 examples/load-balanced-route-servers/routeserver-b.ss.yml create mode 100644 meshrr/LICENSE create mode 100755 meshrr/config.py rename meshrr/{runit-init.sh => connect_wait.py} (63%) create mode 100644 meshrr/defaults/juniper-evpnrs.conf.j2 create mode 100644 meshrr/defaults/juniper-ipv4rr.conf.j2 create mode 100644 meshrr/defaults/meshrr.conf.yml delete mode 100644 meshrr/juniper.conf.j2 create mode 100644 meshrr/run.sh diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml new file mode 100644 index 0000000..9848b7d --- /dev/null +++ b/.github/workflows/publish-image.yml @@ -0,0 +1,62 @@ +# +name: Publish container image + +on: + push: + branches: + - 'main' + - 'next' + tags: + - 'v*' + pull_request: + branches: + - 'main' + - 'next' + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + # + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: "{{defaultContext}}:meshrr" + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/LICENSE b/LICENSE index f0d93c9..3e5398d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) Juniper Networks, Inc. 2020 +Copyright (c) Juniper Networks, Inc. 2023 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index af8eddb..39f9c6f 100644 --- a/README.md +++ b/README.md @@ -9,106 +9,103 @@ At this time and in the project's raw form, *meshrr* should not be considered fo - [Introduction](#introduction) - [Instructions](#instructions) - [Prerequisites](#prerequisites) - - [Quickstart](#quickstart) + - [Usage](#usage) - [Environment Variables](#environment-variables) - - [Methodology](#methodology) + - [Containers](#containers) + - [BGP Group Types](#bgp-group-types) - [Examples](#examples) + - [Example Commands](#example-commands) ## Instructions ### Prerequisites -1. An operational Kubernetes cluster with sufficient resources for the topology you wish to build. -2. A *private* container registry accessible to your Kubernetes cluster. - - You'll need to be logged in to your registry using `docker login` to push the image you'll build. - - You'll need to store your registry credentials in a secret in your cluster to pull from this registry. In this project, all examples use a secret named `regcred`. There are a few ways you can do this. - 1. If you already have a simple means of generating the secret manifest (e.g. using `doctl`), you can do this in one line: - ``` - doctl registry kubernetes-manifest --name regcred | kubectl apply -f - - ``` - 2. You can generate the secret manually with all the parameters: - ``` - kubectl create secret docker-registry regcred \ - --docker-server= --docker-username= \ - --docker-password= --docker-email= - ``` - 3. Any number of [other reasonable approaches](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). +1. An operational Kubernetes cluster with sufficient resources for the topology you wish to build. +2. The cRPD software. The current tested version is **23.2R1.13**. The software must be available via a private repository or preloaded onto all nodes it may be run on. If using k3s, this can be accomplished with `k3s ctr images import junos-routing-crpd-docker-amd64-23.2R1.13.tar`. + - If the import fails, you may need to convert the tarfile into a format that can be imported. You can do so with + ```sh + docker load -i junos-routing-crpd-docker-amd64-23.2R1.13.tgz \ + && docker image tag crpd:23.2R1.13 localhost/juniper/crpd:23.2R1.13 \ + && docker image save localhost/juniper/crpd:23.2R1.13 --output=junos-routing-crpd-docker-amd64-23.2R1.13.tar + ``` 3. A cRPD license for the number of nodes you wish to deploy. At the time of writing, Juniper offers [free trial licenses](https://www.juniper.net/us/en/dm/crpd-trial/). Standard licenses are limited to 16 BGP peers and 4M RIB entries. -### Quickstart -1. (If required) modify [`juniper.conf.j2`](meshrr/juniper.conf.j2) -2. Build and push your image. **Do not push to a public registry.** +### Usage - ```bash - docker build -t meshrr - docker push - ``` +1. (If required) copy a configuration file template from [`the default templates`](meshrr/defaults/) and edit it to your liking. - e.g. - ```bash - docker build meshrr -t registry.example.com/meshrr/meshrr:latest - docker push registry.example.com/meshrr/meshrr:latest - ``` -3. Either: +2. Either: 1. Pick an example topology from [`examples`](examples/) and modify the YAML files as required for your topology. Details for how to use examples and reasonable modifications are below in the [Examples](#Examples) section. 2. Create your own YAML files if you need a completely custom topology. -4. Populate the YAML files with the required information. You will need to, at a minimum, replace the following: - 1. Names - 1. Service - 2. Labels (if following the 2regions-hrr example, your regions probably are not in Middle Earth) + +3. Populate the Kubernetes manifest YAML files with the required information. You will need to, at a minimum, replace the following: + 1. Names of elements 2. [Environment Variables](#Environment-Variables) - 3. Licensing mechanism. Examples here currently use a secret mounted as a volume mapped to `/config/license/safenet/junos_sfnt.lic`. This may be appropriate for bundle licenses where it is appropriate to use the same license file for many similar devices in a deployment or daemonset. You can create this using: + 3. Licensing mechanism. Examples here currently use a secret exposed as an environment variable in the meshrr-init container which will populate the license into the config. This may be appropriate for bundle licenses where it is appropriate to use the same license file for many similar devices in a deployment or daemonset. You can create this using: ``` - kubectl create secret generic crpd-license --from-file=junos_sfnt.lic= + kubectl create secret generic crpd-license --from-file=crpd-license= ``` - 4. Custom configuration Jinja2 templates loaded into ConfigMaps and mapped as volumes. See [Examples](#Examples). - 5. Port mapping IP addresses (`hostIP`). No `hostIP` must be specified for instances only accessible within the cluster. Detailed strategy information to be defined in [Examples](#Examples). -5. Apply appropriate labels to the nodes: + Note that `` must point to a file that contains the singular license line and not an entire license file. + 4. (If required) Custom configuration Jinja2 templates loaded into ConfigMaps and mapped as volumes. See [Examples](#Examples). +4. (If required - e.g., for [2regions-hrr](examples/2regions-hrr/) where only certain nodes should host certain clusters of RRs) Apply appropriate labels to the nodes: ```bash kubectl label nodes = = ``` -6. Apply your configuration: +5. Apply your configuration: ```bash kubectl [-n namespace] apply -f kubectl [-n namespace] apply -f ``` ### Environment Variables -| Variable | Required? | Description | -| --------------------- | --------- | ------------------------------------------------------------ | -| POD_IP | Yes | The pod's IP address. Should be set by Kubernetes (`valueFrom: fieldRef: fieldPath: status.podIP`) | -| MESH_SERVICE_NAME | Yes* | The name of the mesh service. Set to the name of the headless Kubernetes service used for mesh BGP neighbor discovery. *Usually, a `MESH_SERVICE_NAME` is desirable. However, it may be skipped if there is an `UPSTREAM_SERVICE_NAME` in cases such as unmeshed regions learning routes from upstream HRRs. | -| UPSTREAM_SERVICE_NAME | No | The name of the upstream service. Set to the name of the headless Kubernetes service used for upstream BGP neighbor discovery. Defaults to `None`. | -| KUBE_NAMESPACE | No | Optional name of the Kubernetes namespace. Defaults to `default`. | -| ENCRYPTED_ROOT_PW | Yes | Encrypted ($6) root password for cRPD | -| AUTONOMOUS_SYSTEM | Yes | ASN for the router. | -| MESHRR_CLIENTRANGE | Yes | Range to allow. Currently, this accepts only one CIDR block. Format: `network/mask-length` | -| MESHRR_MODE | No | `routereflector` or `routeserver`. Defaults to `routereflector`. | -| MESHRR_ASRANGE | No | Range of ASNs to allow for `routeserver` mode. Defaults to `65001-65500` | -| MESHRR_FAMILY_INET | No | `true` or `false`. Defaults to `true` | -| MESHRR_FAMILY_EVPN | No | `true` or `false`. Defaults to `false` | -| SERVICE_ROOT_DOMAIN | No | Defaults to `svc.cluster.local`. You probably don't need to change this. | - -## Methodology -- Build container image based on crpd. - - Requires additional packages installed via `apt-get`: - - cron - - python3 - - Builds crontab - - Sets up `runit-init.sh` -- `runit-init.sh` initializes the environment: - - Saves environment variables, including the necessary pod's IP address, to `/etc/envvars` - - Sets up the cRPD configuration based on the template `juniper.conf.template` - - Calls `render_config.py` to create configuration file from Jinja2 template. -- `update_peers.py` called every minute via cron. - - Uses a Kubernetes headless service DNS A records to detect peers. + +| Variable | Required for | Optional for | Description | +| -------------- | ------------------- | ------------ | ------------------------------------------------------------ | +| LICENSE_KEY | meshrr-init | | License key to be used for the cRPD container; expected to be a single line. | +| POD_IP | meshrr-init, meshrr | | The pod's IP address. Must be set by Kubernetes manifest in all pod templates for all meshrr containers. This does not need to be set for the cRPD containers. (`valueFrom: fieldRef: fieldPath: status.podIP`) | +| UPDATE_SECONDS | | meshrr | Frequency in seconds that `meshrr` container will attempt to update `crpd` container with changes to peers. (Default: 30) | + + +## Containers + +- Init Container - `meshrr-init`: + - `run.sh` with arg `init` + - Creates configuration from default template or mounted template or derives from existing `/config/juniper.conf` if pod uses persistent storage. +- Container - `crpd`: + - Unmodified cRPD image running Juniper cRPD. +- Container - `meshrr`: + - Conducts periodic BGP peer configuration changes on `crpd` container via Netconf. + +## BGP Group Types + +- `mesh` + - Discovers peers and connects to all or a limited number of them. + - Currently, the only BGP peer discovery mode is `dns`, which uses a Kubernetes headless service DNS A records to detect peers. New peers are added to config, removed peers are removed from config. - - Only occurs once a minute, so, given BGP timers, assume pod readiness 100 seconds from initiation. + - Supports a `max_peers` setting, which limits the number of peers added in this group. This is suitable for connections to a higher tier in a hierarchical route reflector / route server topology. +- `subtractive` + - This can be seen as a "wildcard". This is suitable for an environment in which not all peers are strictly defined and uses Junos BGP group `allow` config to permit connections from a range. + - The `allow` config is dynamically generated based on the list of all prefixes in the meshrr configuration with all peers from any mesh groups removed. ## Examples + - [2regions-hrr](examples/2regions-hrr) - Hierarchicial route reflectors broken into two regions with a single core region unifying them. - Reachability via static routes and Kubernetes NodeIP Services referencing additional loopbacks on the Kubernetes nodes. - [load-balanced-route-servers](examples/load-balanced-route-servers) - EVPN route servers deployed in a full iBGP mesh with each other serving eBGP peers. Intended to scale DCI for multi-region deployment. - - Reachability for external devices achieved through use of MetalLB in BGP mode. \ No newline at end of file + - Reachability for external devices achieved through use of MetalLB in BGP mode. + +## Example Commands + +| Command | Description | +| ------------------------------------------------------------ | ------------------------------------------------------------ | +| `kubectl [-n NAMESPACE] get pods -o wide` | List pods and the nodes on which they run | +| `kubectl [-n NAMESPACE] exec -it POD -c crpd -- cli` | Access the CLI of cRPD | +| `kubectl [-n NAMESPACE] exec POD -c crpd -- cli show bgp summary` | See the `show bgp summary` output of a pod | +| `kubectl [-n NAMESPACE] exec POD -c crpd -- cli show bgp group summary \|except \"Allow\|orlonger\|^Default\|^$\"` | See the status of the neighbor groups of a pod | +| `kubectl [-n NAMESPACE] logs [-f] POD -c meshrr` | View the logs from the meshrr sidecar container. `-f` will follow the logs. | +| `kubectl [-n NAMESPACE] delete pod POD` | Delete POD. Because pods should be created by DaemonSet, StatefulSet, or Deployment, a new pod should be recreated in its place; in this context, this may be considered functionally more similar to a "restart" than to a "delete". | + + + diff --git a/examples/2regions-hrr/README.md b/examples/2regions-hrr/README.md index d6baca2..30b3090 100644 --- a/examples/2regions-hrr/README.md +++ b/examples/2regions-hrr/README.md @@ -7,37 +7,31 @@ * Within a region, all cRPDs are fully meshed via iBGP to provide maximum visibility within the region. * All cRPDs in a region other than `core` have BGP peerings with up to 2 `core` cRPDs. The `core` cRPDs serve as route reflectors for the non-core regions. (The limit of 2 is hard coded on upstream peer groups.) * **Redundancy groups and anycast addressing:** - * Each node is assigned to redundancy group `a` or `b`. + * Each node is assigned to redundancy group `a` or `b`. (In a production environment, two separate Kubernetes clusters may be desirable.) * For each region with neighbors outside the cluster, separate DaemonSets are created for `a` and `b`, each with a unique IP address for that [meshrr_region:redundancy_group] combination. This IP address is used for iBGP peering with neighbors outside the cluster. - * For Lothlorien: - * Kubernetes nodes run MetalLB. - * MetalLB eBGP peers to each connected router on a loopback with the same IP address (10.0.0.0). - * meshrr pods are assigned to a Kubernetes service using the MetalLB load balancer. This service is configured with `externalTrafficPolicy: local` and MetalLB announces the /32 of the service only from nodes with a corresponding meshrr pod, which will act as a route reflector for routers outside the cluster. - * This is the preferred method of distributing routes to the route reflector as it dynamically advertises based upon where route reflectors exist. - * For Mirkwood: - * Each /32 is assigned to the loopback interface of *every* node in the [meshrr_region:redundancy_group] combination. - * Routers connecting to the node have a static route to the /32 that is redistributed into the IGP. - * This method is discouraged but illustrates an alternative option. It provides no assurance that the route reflector is operational before advertising routes, as it relies on static routes from the connected router. Such a method could be enhanced using event scripts or similar, but the method used in Lothlorien provides more native dynamic routing. + * Kubernetes nodes run MetalLB. + * MetalLB eBGP peers to each connected router on a loopback with the same IP address (10.0.0.0). + * meshrr pods are assigned to a Kubernetes service using the MetalLB load balancer. This service is configured with `externalTrafficPolicy: local` and MetalLB announces the /32 of the service only from nodes with a corresponding meshrr pod, which will act as a route reflector for routers outside the cluster. + * This is the preferred method of distributing routes to the route reflector as it dynamically advertises based upon where route reflectors exist. It is possible, however, to use loopbacks with NodeIPs assigned to pods, but this lacks a native means of failover. * In the above topology: - | Node Region | MESHRR_REGION(s) | Redundancy Group | Loopback Address(es) | MetalLB IP(s) | - | ----------- | -------------------------- | ---------------- | ---------------------- | ------------- | - | Lothlorien | lothlorien | a | | 172.19.1.1 | - | Lothlorien | lothlorien | b | | 172.19.1.2 | - | Core | core, mirkwood, lothlorien | a | 172.19.2.1 | 172.19.1.1 | - | Core | core, mirkwood, lothlorien | b | 172.19.2.2 | 172.19.1.2 | - | Mirkwood | mirkwood | a | 172.19.2.1 | | - | Mirkwood | mirkwood | b | 172.19.2.2 | | + | Node | MESHRR_REGION(s) | Redundancy Group | MetalLB IP(s) | + | -------------- | -------------------- | ---------------- | ---------------------- | + | lothlorien-vm1 | core, lothlorien | a | 172.19.1.1 | + | lothlorien-vm2 | core, lothlorien | b | 172.19.1.2 | + | lothlorien-vm3 | mirkwood, lothlorien | a | 172.19.1.1, 172.19.2.1 | + | lothlorien-vm4 | mirkwood, lothlorien | b | 172.19.1.2, 172.19.2.2 | + | mirkwood-vm1 | core, mirkwood | a | 172.19.2.1 | + | mrikwood-vm2 | mirkwood | b | 172.19.2.2 | -### Usage -1. Follow the instructions in [Quickstart](../../README.md#Quickstart) using the example YAML files in [examples/2regions-hrr](.). -2. Create the necessary loopback IPs on each of the nodes based on redundancy group and region. Internal-only MESHRR_REGIONs do not require a configured loopback IP. - ```bash - sudo ip address add 172.19.1.1 dev lo +### Usage +1. Review the instructions in [the project's main README](../../README.md#Usage). Create a license secret: + ``` + kubectl create secret generic crpd-license --from-file=crpd-license= ``` -3. Configure the Lothlorien routers connected to the nodes with: +2. Configure the routers connected to the nodes with: * The anycast peering address for MetalLB to peer to on the loopback * Hardcoded router ID (to ensure that the anycast peering address does not become the router ID) * BGP peering @@ -56,70 +50,57 @@ set policy-options policy-statement FILTER-RRLBPEER then reject set policy-options community no-advertise members no-advertise set policy-options policy-statement NO-ADVERTISE then community add no-advertise - set policy-options policy-statement REDISTRIBUTE-RRS from protocol bgp route-filter 172.19.1.0/24 prefix-length-range /32-/32 + set policy-options policy-statement REDISTRIBUTE-RRS from protocol bgp route-filter 172.19.0.0/16 prefix-length-range /32-/32 set policy-options policy-statement REDISTRIBUTE-RRS then accept ``` -4. Configure the Mirkwood routers connected to the nodes with static routes redistributed into your IGP for the node loopback addresses. -Configuration on the router servicing a Mirkwood A node may look like: - - ##### Junos - ```junos - routing-options { - static { - route 172.19.2.1/32 next-hop ; - } - } - policy-options { - policy-statement RRSTATIC-TO-ISIS { - from { - protocol static; - route-filter 172.19.2.1/32 exact; - } - then accept; - } - } - protocols { - isis { - export RRSTATIC-TO-ISIS; - } - } - ``` - - ##### IOS-XR - ```iox-xr - router static - address-family ipv4 unicast - 172.19.2.1/32 - ! - route-policy STATIC-TO-ISIS - if destination in (172.19.2.1/32) then - pass - endif - end-policy - ! - router isis ISIS - redistribute static level-2 route-policy STATIC-TO-ISIS - ! - ``` - -5. Modify configuration templates as necessary. [`meshrr/juniper.conf.j2`](../../meshrr/juniper.conf.j2) will be loaded to all instances by default, but customizations on a per-deployment/per-daemonset basis should be performed on other J2 files (see [`mirkwood-config.j2`](templates/mirkwood-config.j2) and [`lothlorien-config.j2`](templates/lothlorien-config.j2). +3. Modify configuration templates as necessary. [`meshrr/juniper.conf.j2`](../../meshrr/defaults/juniper-ipv4rr.conf.j2) will be loaded to all instances by default, but customizations on a per-deployment/per-daemonset basis should be performed in most cases files (see [`mirkwood-config.j2`](templates/mirkwood-config.j2) and [`lothlorien-config.j2`](templates/lothlorien-config.j2)). Apply these configuration templates as ConfigSets for any cases that require customization as so: ```bash - k create configmap mirkwood-config \ + kubectl create configmap mirkwood-config \ --from-file=config=examples/2regions-hrr/templates/mirkwood-config.j2 \ -o yaml --dry-run=client | - k apply -f - + kubectl apply -f - ``` ```bash - k create configmap lothlorien-config \ + kubectl create configmap lothlorien-config \ --from-file=config=examples/2regions-hrr/templates/lothlorien-config.j2 \ -o yaml --dry-run=client | - k apply -f - + kubectl apply -f - + ``` + + These ConfigMaps are mounted as volumes in the corresponding Deployments/DaemonSets. + +4. Label the nodes. You may need to adjust the node names. + ```bash + kubectl label nodes lothlorien-vm1 meshrr_region_core="true" meshrr_region_lothlorien="true" redundancy_group=a + kubectl label nodes lothlorien-vm2 meshrr_region_core="true" meshrr_region_lothlorien="true" redundancy_group=b + kubectl label nodes lothlorien-vm3 meshrr_region_lothlorien="true" meshrr_region_mirkwood="true" redundancy_group=a + kubectl label nodes lothlorien-vm4 meshrr_region_lothlorien="true" meshrr_region_mirkwood="true" redundancy_group=b + kubectl label nodes mirkwood-vm1 meshrr_region_core="true" meshrr_region_mirkwood="true" redundancy_group=a + kubectl label nodes mirkwood-vm2 meshrr_region_mirkwood="true" redundancy_group=b ``` - These ConfigMaps are mounted as volumes in the corresponding DaemonSets. +5. Apply the Kubernetes manifests: + ```bash + k apply -f examples/2regions-hrr/bgppeer-global.yml -f examples/2regions-hrr/meshrr-core.yaml -f examples/2regions-hrr/meshrr-lothlorien.yaml -f examples/2regions-hrr/meshrr-mirkwood.yaml + ``` + + See [Manifests and Objects Used](#Manifests-and-Objects-Used) for detail on what this does. + +### Manifests and Objects Used -6. Modify the YAML files to your needs. At the least, `` will need to be replaced to reference your private registry. Load the YAML files for the DaemonSets and Services into Kubernetes as per [Quickstart](../../README.md#Quickstart). \ No newline at end of file +* [`bgppeer-global.yml`]() + * `BGPPeer/asn65000-global-lo1`: Defines a single BGP peer that will be used from every node in the Kubernetes cluster out every interface. In this example, we use 10.0.0.0/32 as an anycast IP on every router that connects to the Kubernetes cluster. This must not be advertised in any routing protocol. +* [`meshrr-core.yaml`]() + * `ConfigMap/meshrr-core-conf`: Contains the YAML to configure meshrr for the Core Deployment. + * `Service/meshrr-core`: Headless service used for inter-pod BGP service discovery. + * `Deployment/meshrr-core`: Defines the Deployment for the meshrr Core region. +* [`meshrr-lothlorien.yaml`]() and [`meshrr-mirkwood.yaml`]() + * `ConfigMap/meshrr--conf`: Contains the YAML to configure meshrr for both the DaemonSets. + * `IPAddressPool/meshrr-`: The address pool used for both the A side and B side load balancers in . + * `BGPAdvertisement/meshrr-`: Configures MetalLB to advertise the load balancer addresses when a pod is reachable via a LoadBalancer service on the node. + * `Service/meshrr--`: Configures a MetalLB LoadBalancer to connect external route reflector clients to a containerized route reflector in redundancy group . Note that `externalTrafficPolicy: Local` is of importance here to ensure that traffic to the service does not traverse the Kubernetes cluster to reach a Pod but only is advertised via BGP if a pod in the service exists on the node. + * `DaemonSet/meshrr--`: The DaemonSet for the redundancy group nodes in . Note the use of DaemonSets vs Deployments: Deployments ensure that a certain number of pods exist within the cluster. DaemonSets ensure that a pod exists on each node within the cluster that matches the parameters defined in `nodeAffinity`. diff --git a/examples/2regions-hrr/meshrr-core.yaml b/examples/2regions-hrr/meshrr-core.yaml index 38081d5..d1cd104 100644 --- a/examples/2regions-hrr/meshrr-core.yaml +++ b/examples/2regions-hrr/meshrr-core.yaml @@ -1,5 +1,26 @@ --- apiVersion: v1 +kind: ConfigMap +metadata: + name: meshrr-core-conf +data: + meshrr.conf.yml: |+ + encrypted_root_pw: NOLOGIN + asn: "65000" + mode: ipv4rr + bgpgroups: + - name: MESHRR-MESH + type: mesh + source: + sourcetype: dns + hostname: meshrr-core + - name: MESHRR-CLIENTS + type: subtractive + prefixes: + - 10.42.0.0/16 + +--- +apiVersion: v1 kind: Service metadata: creationTimestamp: null @@ -32,7 +53,7 @@ spec: selector: matchLabels: app: meshrr - replicas: 2 + replicas: 3 strategy: type: RollingUpdate rollingUpdate: @@ -47,16 +68,47 @@ spec: dnsPolicy: ClusterFirst terminationGracePeriodSeconds: 30 volumes: - - name: crpd-license - secret: - secretName: crpd-license + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 items: - - key: junos_sfnt.lic - path: junos_sfnt.lic + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-core-conf + optional: false + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license containers: - - name: meshrr-core - image: /meshrr:latest - imagePullPolicy: Always + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -79,30 +131,36 @@ spec: - name: bgp containerPort: 179 protocol: TCP - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 volumeMounts: - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' + - name: config + mountPath: /config/ securityContext: allowPrivilegeEscalation: true privileged: true runAsNonRoot: false - imagePullSecrets: - - name: regcred + - name: meshrr + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/examples/2regions-hrr/meshrr-lothlorien.yaml b/examples/2regions-hrr/meshrr-lothlorien.yaml index 5dda168..824188e 100644 --- a/examples/2regions-hrr/meshrr-lothlorien.yaml +++ b/examples/2regions-hrr/meshrr-lothlorien.yaml @@ -1,4 +1,30 @@ --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: meshrr-lothlorien-conf +data: + meshrr.conf.yml: |+ + encrypted_root_pw: NOLOGIN + asn: "65000" + mode: ipv4rr + bgpgroups: + - name: MESHRR-MESH + type: mesh + source: + sourcetype: dns + hostname: meshrr-lothlorien + - name: MESHRR-CLIENTS + type: subtractive + prefixes: + - 0.0.0.0/0 + - name: MESHRR-UPSTREAM + type: mesh + source: + sourcetype: dns + hostname: meshrr-core + max_peers: 2 +--- apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: @@ -117,12 +143,19 @@ spec: dnsPolicy: ClusterFirst terminationGracePeriodSeconds: 30 volumes: - - name: crpd-license - secret: - secretName: crpd-license + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 items: - - key: junos_sfnt.lic - path: junos_sfnt.lic + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-lothlorien-conf + optional: false - configMap: defaultMode: 256 items: @@ -131,11 +164,39 @@ spec: path: juniper.conf.j2 name: lothlorien-config optional: false - name: config + name: override-config + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + - mountPath: /opt/meshrr/conf/juniper.conf.j2 + name: override-config + readOnly: true + subPath: juniper.conf.j2 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license containers: - - name: meshrr-lothlorien-a - image: /meshrr:latest - imagePullPolicy: Always + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: Never livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -158,37 +219,36 @@ spec: - name: bgp containerPort: 179 protocol: TCP - hostIP: 172.19.1.1 - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-lothlorien - - name: UPSTREAM_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 volumeMounts: - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - - mountPath: /root/juniper.conf.j2 - name: config - readOnly: true - subPath: juniper.conf.j2 + - name: config + mountPath: /config/ securityContext: allowPrivilegeEscalation: true privileged: true runAsNonRoot: false - imagePullSecrets: - - name: regcred + - name: meshrr + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -231,12 +291,19 @@ spec: dnsPolicy: ClusterFirst terminationGracePeriodSeconds: 30 volumes: - - name: crpd-license - secret: - secretName: crpd-license + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 items: - - key: junos_sfnt.lic - path: junos_sfnt.lic + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-lothlorien-conf + optional: false - configMap: defaultMode: 256 items: @@ -245,11 +312,39 @@ spec: path: juniper.conf.j2 name: lothlorien-config optional: false - name: config + name: override-config + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + - mountPath: /opt/meshrr/conf/juniper.conf.j2 + name: override-config + readOnly: true + subPath: juniper.conf.j2 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license containers: - - name: meshrr-lothlorien-b - image: /meshrr:latest - imagePullPolicy: Always + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: Never livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -272,37 +367,36 @@ spec: - name: bgp containerPort: 179 protocol: TCP - hostIP: 172.19.1.2 - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-lothlorien - - name: UPSTREAM_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 172.18.0.0/16 volumeMounts: - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - - mountPath: /root/juniper.conf.j2 - name: config - readOnly: true - subPath: juniper.conf.j2 + - name: config + mountPath: /config/ securityContext: allowPrivilegeEscalation: true privileged: true runAsNonRoot: false - imagePullSecrets: - - name: regcred + - name: meshrr + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/examples/2regions-hrr/meshrr-mirkwood.yaml b/examples/2regions-hrr/meshrr-mirkwood.yaml index a45789b..0015a19 100644 --- a/examples/2regions-hrr/meshrr-mirkwood.yaml +++ b/examples/2regions-hrr/meshrr-mirkwood.yaml @@ -1,5 +1,101 @@ --- apiVersion: v1 +kind: ConfigMap +metadata: + name: meshrr-mirkwood-conf +data: + meshrr.conf.yml: |+ + encrypted_root_pw: NOLOGIN + asn: "65000" + mode: ipv4rr + bgpgroups: + - name: MESHRR-MESH + type: mesh + source: + sourcetype: dns + hostname: meshrr-mirkwood + - name: MESHRR-CLIENTS + type: subtractive + prefixes: + - 0.0.0.0/0 + - name: MESHRR-UPSTREAM + type: mesh + source: + sourcetype: dns + hostname: meshrr-core + max_peers: 2 +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: meshrr-mirkwood + namespace: metallb +spec: + addresses: + - 172.19.2.1/32 + - 172.19.2.2/32 + autoAssign: false +--- +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: meshrr-mirkwood + namespace: metallb +spec: + ipAddressPools: + - meshrr-mirkwood +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + name: meshrr-mirkwood-a + selfLink: /api/v1/namespaces/default/services/meshrr-mirkwood-a + annotations: + metallb.universe.tf/address-pool: meshrr-mirkwood +spec: + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr + meshrr_region_mirkwood: "true" + redundancy_group: a + sessionAffinity: None + type: LoadBalancer + loadBalancerIP: 172.19.2.1 + externalTrafficPolicy: Local +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + name: meshrr-mirkwood-b + selfLink: /api/v1/namespaces/default/services/meshrr-mirkwood-b + annotations: + metallb.universe.tf/address-pool: meshrr-mirkwood +spec: + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr + meshrr_region_mirkwood: "true" + redundancy_group: b + sessionAffinity: None + type: LoadBalancer + loadBalancerIP: 172.19.2.2 + externalTrafficPolicy: Local +status: + loadBalancer: {} +--- +apiVersion: v1 kind: Service metadata: creationTimestamp: null @@ -48,12 +144,19 @@ spec: dnsPolicy: ClusterFirst terminationGracePeriodSeconds: 30 volumes: - - name: crpd-license - secret: - secretName: crpd-license + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 items: - - key: junos_sfnt.lic - path: junos_sfnt.lic + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-mirkwood-conf + optional: false - configMap: defaultMode: 256 items: @@ -62,11 +165,39 @@ spec: path: juniper.conf.j2 name: mirkwood-config optional: false - name: config + name: override-config + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + - mountPath: /opt/meshrr/conf/juniper.conf.j2 + name: override-config + readOnly: true + subPath: juniper.conf.j2 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license containers: - - name: meshrr-mirkwood-a - image: /meshrr:latest - imagePullPolicy: Always + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: Never livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -89,38 +220,36 @@ spec: - name: bgp containerPort: 179 protocol: TCP - hostIP: 172.19.2.1 - hostPort: 179 - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-mirkwood - - name: UPSTREAM_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 172.18.0.0/16 volumeMounts: - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - - mountPath: /root/juniper.conf.j2 - name: config - readOnly: true - subPath: juniper.conf.j2 + - name: config + mountPath: /config/ securityContext: allowPrivilegeEscalation: true privileged: true runAsNonRoot: false - imagePullSecrets: - - name: regcred + - name: meshrr + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -163,12 +292,19 @@ spec: dnsPolicy: ClusterFirst terminationGracePeriodSeconds: 30 volumes: - - name: crpd-license - secret: - secretName: crpd-license + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 items: - - key: junos_sfnt.lic - path: junos_sfnt.lic + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-mirkwood-conf + optional: false - configMap: defaultMode: 256 items: @@ -177,11 +313,39 @@ spec: path: juniper.conf.j2 name: mirkwood-config optional: false - name: config + name: override-config + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + - mountPath: /opt/meshrr/conf/juniper.conf.j2 + name: override-config + readOnly: true + subPath: juniper.conf.j2 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license containers: - - name: meshrr-mirkwood-b - image: /meshrr:latest - imagePullPolicy: Always + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: Never livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -204,38 +368,36 @@ spec: - name: bgp containerPort: 179 protocol: TCP - hostIP: 172.19.2.2 - hostPort: 179 - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-mirkwood - - name: UPSTREAM_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 172.18.0.0/16 volumeMounts: - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - - mountPath: /root/juniper.conf.j2 - name: config - readOnly: true - subPath: juniper.conf.j2 + - name: config + mountPath: /config/ securityContext: allowPrivilegeEscalation: true privileged: true runAsNonRoot: false - imagePullSecrets: - - name: regcred + - name: meshrr + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/examples/2regions-hrr/templates/lothlorien-config.j2 b/examples/2regions-hrr/templates/lothlorien-config.j2 index a01e44e..25a2502 100644 --- a/examples/2regions-hrr/templates/lothlorien-config.j2 +++ b/examples/2regions-hrr/templates/lothlorien-config.j2 @@ -1,6 +1,25 @@ system { root-authentication { - encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA + encrypted-password "{{ encrypted_root_pw }}"; ## SECRET-DATA + } + login { + user meshrr { + class super-user; + uid 100; + authentication { + ssh-ed25519 "ssh-ed25519 TBD"; ## SECRET-DATA + } + } + } + services { + netconf { + ssh; + } + } + license { + keys { + key "{{ LICENSE_KEY }}"; + } } processes { routing { @@ -21,12 +40,12 @@ groups { group MESHRR-CLIENTS { type internal; cluster {{ POD_IP }}; - allow {{ MESHRR_CLIENTRANGE }}; - }{% if UPSTREAM_SERVICE_NAME is not none %} + allow [ {% for prefix in bgpgroups_subtractive['MESHRR-CLIENTS'].prefixes %}{{ prefix }} {% endfor %}]; + } group MESHRR-UPSTREAM { type internal; export UPSTREAM-OUT; - }{% endif %} + } } } } @@ -49,7 +68,7 @@ policy-options { community INREGION-PREFERRED members 65000:102; } routing-options { - autonomous-system {{ AUTONOMOUS_SYSTEM }}; + autonomous-system {{ asn }}; router-id {{ POD_IP }}; } protocols { diff --git a/examples/2regions-hrr/templates/mirkwood-config.j2 b/examples/2regions-hrr/templates/mirkwood-config.j2 index a01e44e..25a2502 100644 --- a/examples/2regions-hrr/templates/mirkwood-config.j2 +++ b/examples/2regions-hrr/templates/mirkwood-config.j2 @@ -1,6 +1,25 @@ system { root-authentication { - encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA + encrypted-password "{{ encrypted_root_pw }}"; ## SECRET-DATA + } + login { + user meshrr { + class super-user; + uid 100; + authentication { + ssh-ed25519 "ssh-ed25519 TBD"; ## SECRET-DATA + } + } + } + services { + netconf { + ssh; + } + } + license { + keys { + key "{{ LICENSE_KEY }}"; + } } processes { routing { @@ -21,12 +40,12 @@ groups { group MESHRR-CLIENTS { type internal; cluster {{ POD_IP }}; - allow {{ MESHRR_CLIENTRANGE }}; - }{% if UPSTREAM_SERVICE_NAME is not none %} + allow [ {% for prefix in bgpgroups_subtractive['MESHRR-CLIENTS'].prefixes %}{{ prefix }} {% endfor %}]; + } group MESHRR-UPSTREAM { type internal; export UPSTREAM-OUT; - }{% endif %} + } } } } @@ -49,7 +68,7 @@ policy-options { community INREGION-PREFERRED members 65000:102; } routing-options { - autonomous-system {{ AUTONOMOUS_SYSTEM }}; + autonomous-system {{ asn }}; router-id {{ POD_IP }}; } protocols { diff --git a/examples/3clientgroups/bgppeer-global.yml b/examples/3clientgroups/bgppeer-global.yml new file mode 100644 index 0000000..acd6e75 --- /dev/null +++ b/examples/3clientgroups/bgppeer-global.yml @@ -0,0 +1,10 @@ +apiVersion: metallb.io/v1beta2 +kind: BGPPeer +metadata: + name: global-lo1 + namespace: metallb +spec: + myASN: 65001 + peerASN: 65000 + peerAddress: 10.0.0.0 + ebgpMultiHop: True diff --git a/examples/3clientgroups/meshrr-3clientgroups.yaml b/examples/3clientgroups/meshrr-3clientgroups.yaml new file mode 100644 index 0000000..f213033 --- /dev/null +++ b/examples/3clientgroups/meshrr-3clientgroups.yaml @@ -0,0 +1,513 @@ +--- +apiVersion: v1 +data: + config: | + system { + /* Required, though use of a variable is optional */ + root-authentication { + encrypted-password "{{ encrypted_root_pw }}"; ## SECRET-DATA + } + login { + /* Required for meshrr authentication */ + user meshrr { + class super-user; + uid 100; + authentication { + ssh-ed25519 "ssh-ed25519 TBD"; ## SECRET-DATA + } + } + } + services { + netconf { + ssh; + } + } + /* Required for licensing unless using an alternative approach. */ + license { + keys { + key "{{ LICENSE_KEY }}"; + } + } + processes { + routing { + bgp { + rib-sharding; + update-threading; + } + } + } + } + groups { + MESHRR { + protocols { + bgp { + /* Mesh {% if bgpgroups_mesh['MESHRR-MESH'].max_peers is defined %}max_peers:{{ bgpgroups_mesh['MESHRR-MESH'].max_peers }} {% endif %}group from {{ bgpgroups_mesh['MESHRR-MESH'].source.sourcetype }}:{{ bgpgroups_mesh['MESHRR-MESH'].source.hostname }} */ + group MESHRR-MESH { + type internal + } + /* Mesh {% if bgpgroups_mesh['MESHRR-FULLTABLE-CLIENTS'].max_peers is defined %}max_peers:{{ bgpgroups_mesh['MESHRR-FULLTABLE-CLIENTS'].max_peers }} {% endif %}group from {{ bgpgroups_mesh['MESHRR-FULLTABLE-CLIENTS'].source.sourcetype }}:{{ bgpgroups_mesh['MESHRR-FULLTABLE-CLIENTS'].source.hostname }} */ + group MESHRR-FULLTABLE-CLIENTS { + type internal; + passive; + cluster {{ POD_IP }}; + } + /* Mesh {% if bgpgroups_mesh['MESHRR-PARTIALTABLE-CLIENTS'].max_peers is defined %}max_peers:{{ bgpgroups_mesh['MESHRR-PARTIALTABLE-CLIENTS'].max_peers }} {% endif %}group from {{ bgpgroups_mesh['MESHRR-PARTIALTABLE-CLIENTS'].source.sourcetype }}:{{ bgpgroups_mesh['MESHRR-PARTIALTABLE-CLIENTS'].source.hostname }} */ + group MESHRR-PARTIALTABLE-CLIENTS { + type internal; + passive; + export PARTIALTABLE-OUT; + cluster {{ POD_IP }}; + } + /* Mesh {% if bgpgroups_mesh['MESHRR-DEFAULTONLY-CLIENTS'].max_peers is defined %}max_peers:{{ bgpgroups_mesh['MESHRR-DEFAULTONLY-CLIENTS'].max_peers }} {% endif %}group from {{ bgpgroups_mesh['MESHRR-DEFAULTONLY-CLIENTS'].source.sourcetype }}:{{ bgpgroups_mesh['MESHRR-DEFAULTONLY-CLIENTS'].source.hostname }} */ + group MESHRR-DEFAULTONLY-CLIENTS { + type internal; + passive; + export DEFAULTONLY-OUT; + cluster {{ POD_IP }}; + } + } + } + } + } + policy-options { + policy-statement DEFAULTONLY-OUT { + from { + route-filter 0.0.0.0/0 exact; + } + then accept; + } + policy-statement PARTIALTABLE-OUT { + term SAME-AS { + from { + as-path-calc-length 0 orlower; + } + then accept; + } + term DEFAULT { + from { + route-filter 0.0.0.0/0 exact; + } + then accept; + } + } + } + routing-options { + autonomous-system {{ asn }}; + router-id {{ POD_IP }}; + } + protocols { + bgp { + apply-groups MESHRR; + family inet { + unicast { + nexthop-resolution { + no-resolution; + } + no-install; + } + } + } + } +kind: ConfigMap +metadata: + creationTimestamp: null + name: 3clientgroups-config +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: meshrr-3clientgroups-conf +data: + meshrr.conf.yml: |+ + encrypted_root_pw: NOLOGIN + asn: "65000" + mode: ipv4rr + bgpgroups: + - name: MESHRR-MESH + type: mesh + source: + sourcetype: dns + hostname: meshrr-3clientgroups + - name: MESHRR-FULLTABLE-CLIENTS + type: mesh + source: + sourcetype: dns + hostname: meshrr-fulltable-clients + - name: MESHRR-PARTIALTABLE-CLIENTS + type: mesh + source: + sourcetype: dns + hostname: meshrr-partialtable-clients + - name: MESHRR-DEFAULTONLY-CLIENTS + type: mesh + source: + sourcetype: dns + hostname: meshrr-defaultonly-clients +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: meshrr-3clientgroups + namespace: metallb +spec: + addresses: + - 172.19.1.1/32 + - 172.19.1.2/32 + autoAssign: false +--- +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: meshrr-3clientgroups + namespace: metallb +spec: + ipAddressPools: + - meshrr-3clientgroups +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + name: meshrr-3clientgroups-a + annotations: + metallb.universe.tf/address-pool: meshrr-3clientgroups +spec: + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr + redundancy_group: a + sessionAffinity: None + type: LoadBalancer + loadBalancerIP: 172.19.1.1 + externalTrafficPolicy: Local +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + name: meshrr-3clientgroups-b + annotations: + metallb.universe.tf/address-pool: meshrr-3clientgroups +spec: + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr + redundancy_group: b + sessionAffinity: None + type: LoadBalancer + loadBalancerIP: 172.19.1.2 + externalTrafficPolicy: Local +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + name: meshrr-3clientgroups +spec: + clusterIP: None + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: meshrr-3clientgroups-a + labels: + app: meshrr + redundancy_group: a +spec: + selector: + matchLabels: + app: meshrr + minReadySeconds: 5 + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app: meshrr + redundancy_group: a + spec: + dnsPolicy: ClusterFirst + terminationGracePeriodSeconds: 30 + volumes: + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-3clientgroups-conf + optional: false + - configMap: + defaultMode: 256 + items: + - key: config + mode: 256 + path: juniper.conf.j2 + name: 3clientgroups-config + optional: false + name: override-config + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + - mountPath: /opt/meshrr/conf/juniper.conf.j2 + name: override-config + readOnly: true + subPath: juniper.conf.j2 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + containers: + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: Never + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 2 + successThreshold: 1 + tcpSocket: + port: bgp + timeoutSeconds: 3 + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 2 + tcpSocket: + port: bgp + timeoutSeconds: 3 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + ports: + - name: bgp + containerPort: 179 + protocol: TCP + volumeMounts: + - name: config + mountPath: /config/ + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsNonRoot: false + - name: meshrr + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: redundancy_group + operator: In + values: + - a + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: meshrr-3clientgroups-b + labels: + app: meshrr + redundancy_group: b +spec: + selector: + matchLabels: + app: meshrr + minReadySeconds: 5 + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app: meshrr + redundancy_group: b + spec: + dnsPolicy: ClusterFirst + terminationGracePeriodSeconds: 30 + volumes: + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-3clientgroups-conf + optional: false + - configMap: + defaultMode: 256 + items: + - key: config + mode: 256 + path: juniper.conf.j2 + name: 3clientgroups-config + optional: false + name: override-config + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + - mountPath: /opt/meshrr/conf/juniper.conf.j2 + name: override-config + readOnly: true + subPath: juniper.conf.j2 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + containers: + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: Never + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 2 + successThreshold: 1 + tcpSocket: + port: bgp + timeoutSeconds: 3 + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 2 + tcpSocket: + port: bgp + timeoutSeconds: 3 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + ports: + - name: bgp + containerPort: 179 + protocol: TCP + volumeMounts: + - name: config + mountPath: /config/ + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsNonRoot: false + - name: meshrr + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: redundancy_group + operator: In + values: + - b \ No newline at end of file diff --git a/examples/3clientgroups/meshrr-defaultonly-clients.yml b/examples/3clientgroups/meshrr-defaultonly-clients.yml new file mode 100644 index 0000000..2fc7715 --- /dev/null +++ b/examples/3clientgroups/meshrr-defaultonly-clients.yml @@ -0,0 +1,26 @@ +--- +kind: "Service" +apiVersion: "v1" +metadata: + name: "meshrr-defaultonly-clients" +spec: + clusterIP: None + ports: + - name: "bgp" + protocol: "TCP" + port: 179 + targetPort: 179 +--- +kind: "Endpoints" +apiVersion: "v1" +metadata: + name: "meshrr-defaultonly-clients" +subsets: + - addresses: + - ip: "192.0.2.4" + - ip: "192.0.2.5" + - ip: "192.0.2.56" + ports: + - + port: 179 + name: "bgp" \ No newline at end of file diff --git a/examples/3clientgroups/meshrr-fulltable-clients.yml b/examples/3clientgroups/meshrr-fulltable-clients.yml new file mode 100644 index 0000000..fb66040 --- /dev/null +++ b/examples/3clientgroups/meshrr-fulltable-clients.yml @@ -0,0 +1,25 @@ +--- +kind: "Service" +apiVersion: "v1" +metadata: + name: "meshrr-fulltable-clients" +spec: + clusterIP: None + ports: + - name: "bgp" + protocol: "TCP" + port: 179 + targetPort: 179 +--- +kind: "Endpoints" +apiVersion: "v1" +metadata: + name: "meshrr-fulltable-clients" +subsets: + - addresses: + - ip: "1.2.3.4" + - ip: "4.5.6.7" + ports: + - + port: 179 + name: "bgp" \ No newline at end of file diff --git a/examples/3clientgroups/meshrr-partialtable-clients.yml b/examples/3clientgroups/meshrr-partialtable-clients.yml new file mode 100644 index 0000000..18dfce6 --- /dev/null +++ b/examples/3clientgroups/meshrr-partialtable-clients.yml @@ -0,0 +1,25 @@ +--- +kind: "Service" +apiVersion: "v1" +metadata: + name: "meshrr-partialtable-clients" +spec: + clusterIP: None + ports: + - name: "bgp" + protocol: "TCP" + port: 179 + targetPort: 179 +--- +kind: "Endpoints" +apiVersion: "v1" +metadata: + name: "meshrr-partialtable-clients" +subsets: + - addresses: + - ip: "2.3.4.5" + - ip: "6.7.8.9" + ports: + - + port: 179 + name: "bgp" \ No newline at end of file diff --git a/examples/load-balanced-route-servers/README.md b/examples/load-balanced-route-servers/README.md index 32dd914..340c129 100644 --- a/examples/load-balanced-route-servers/README.md +++ b/examples/load-balanced-route-servers/README.md @@ -1,15 +1,13 @@ -# Example: Fully-meshed Multi-region Route Servers +# Example: Fully-meshed Route Servers ## Description -* This topology has two regions, `1` and `2`. * cRPD is used as route servers for EVPN connectivity to minimize peering requirements in a many-DC environment. -* The two regions have route servers fully meshed together. -* Route servers, and Kubernetes nodes, are defined as "a" or "b" side. -* These route servers are deployed as StatefulSets for configuration persistence. +* Route servers, and Kubernetes nodes, are defined as "a" or "b" side. In this example, they exist within a single Kubernetes cluster using labels to differentiate between nodes. However, it is feasible to structure this with two entirely separate Kubernetes clusters. +* These route servers are deployed as StatefulSets; these could be used to provide for configuration persistence but do not in this example. * **Redundancy groups and anycast addressing:** * Each node is assigned to redundancy group ("side") `a` or `b`. - * cRPD pods are scheduled via StatefulSets, and will only be scheduled on a node of their appropriate side. They will prefer to be scheduled on a node of their region. With minor modification to the manifest, backup region(s) can be configured to avoid them being scheduled to just any side-compliant node. - * In this example, the same IP address is configured as a loopback on each device connecting to a k8s node. This enables to use of only one MetalLB `BGPPeer` manifest globally. + * cRPD pods are scheduled via StatefulSets, and will only be scheduled on a node of their appropriate side. + * In this example, the same IP address is configured as a loopback on each device connecting to a k8s node. Serving as an anycast address, this enables to use of only one MetalLB `BGPPeer` manifest globally. * MetalLB is used to provide external addressing for BGP connectivity to the cRPD route servers, as well as load balancing if the StatefulSet's `replicas` setting is greater than 1 (or multiple deployments provide endpoints for the same service). * MetalLB is also used in this example to provide external addressing for traditional management connectivity (SSH) to the cRPD route servers. * `LoadBalancer` services use a `Local` `externalTrafficPolicy`. This is important to ensure that: @@ -17,12 +15,7 @@ * cRPD's view of the peer shows the physical router's peering IP address (not critical for function but important for operational clarity) ## Requirements -* Kubernetes environment with at least two nodes (one `side=a` and one `side=b`) and with coredns active. For this example, we used microk8s. -* Available storage class. Longhorn was used for this example: - ```zsh - helm repo add longhorn https://charts.longhorn.io - helm install longhorn longhorn/longhorn --namespace longhorn-system --create-namespace --version 1.4.0 --set defaultSettings.defaultDataPath="/longhorn" --set csi.kubeletRootDir="/var/snap/microk8s/common/var/lib/kubelet" - ``` +* Kubernetes environment with at least two nodes (one `side=a` and one `side=b`) and with coredns active. This example has been tested on k3s. * MetalLB: ```zsh helm repo add metallb https://metallb.github.io/metallb @@ -40,7 +33,74 @@ Configuration on the router may look like: - ##### Junos + ##### Junos - If RS is in a DC fabric + ```junos + interfaces { + lo0 { + unit 0 { + family inet { + # Primary loopback address + address 192.168.1.3/32 { + primary; + preferred; + } + # MetalLB Anycast Peer + address 192.168.255.0/32/32; + } + } + } + ge-0/0/2 { + description "meshrr-kube1 eth1"; + unit 0 { + family inet { + address 172.16.1.12/31 + } + } + } + } + policy-options { + policy-statement ADVERTISE-LOOPBACKS { + term DENY-MESHRR-LB { + from { + route-filter 192.168.255.0/32 exact; + } + then reject; + } + from { + family inet; + protocol direct; + interface lo0.0; + } + then accept; + } + policy-statement ADVERTISE-MESHRR { + from { + family inet; + protocol direct; + interface ge-0/0/2.0; + } + then accept; + } + } + protocols { + bgp { + group MESHRR-LB { + type external; + local-address 192.168.255.0; + ttl 1; + family inet { + unicast; + } + peer-as 65000.3; + local-as 65000.2 private; + allow 172.16.0.0/16; + } + export [ ADVERTISE-LOOPBACKS ADVERTISE-MESHRR ]; + } + } + ``` + + ##### Junos - If RS is on the MPLS network in a VRF ```junos interfaces lo0 { unit 100 { @@ -70,7 +130,7 @@ } ``` - ##### IOS-XR + ##### IOS-XR - If RS is on the MPLS network in a VRF ```ios-xr interface Loopback100 vrf DCI @@ -105,8 +165,8 @@ - `Service/meshrr-core` - Provides a headless service coordinating the meshrr function of automatically forming full-mesh iBGP peerings between route servers. Also includes the IPAddressPool and corresponding L2Advertisement for the management network connectivity of the route servers. (In a lab environment, a single L2 domain for cRPD management was sufficient.) - `metallb-bgppeer-global.yml` - `bgppeers.metallb.io/asn100-global-lo100` - Peers MetalLB to the loopback deployed for on each router connecting to the Kubernetes cluster. -- `routeserver--.yml` - - `ipaddresspools.metallb.io/routeserver--` - Creates a pool containing the single address for the service per region per side. `autoAssign: false` ensures that the address is not allocated unless specifically requested by the service. - - `bgpadvertisements.metallb.io/routeserver--` - Advertises the address to all peers (by default) from all nodes that host and endpoint for the service. - - `Service/routeserver--` - Allocates the address based on the pool defined previously and uses it as an external address load balancing BGP to all healthy pods matching the criteria. - - `Deployment/routeserver--` - Creates a deployment of the service for the region and side. In the `routerserver-1-b` example, `replicas: 2`, but for most production deployments, 1 should be sufficient and operationally simpler. \ No newline at end of file +- `routeserver-.ss.yml` + - `ipaddresspools.metallb.io/routeserver-` - Creates a pool containing the single address for the service per side. `autoAssign: false` ensures that the address is not allocated unless specifically requested by the service. + - `bgpadvertisements.metallb.io/routeserver-` - Advertises the address to all peers (by default) from all nodes that host and endpoint for the service. + - `Service/routeserver-` - Allocates the address based on the pool defined previously and uses it as an external address load balancing BGP to all healthy pods matching the criteria. + - `StatefulSet/routeserver-` - Creates a StatefulSet of the service for the side. In the `routerserver-b` example, `replicas: 2`, but for most production deployments, 1 should be sufficient and operationally simpler. \ No newline at end of file diff --git a/examples/load-balanced-route-servers/bgppeer-global.yml b/examples/load-balanced-route-servers/bgppeer-global.yml index b86c96e..6c3312f 100644 --- a/examples/load-balanced-route-servers/bgppeer-global.yml +++ b/examples/load-balanced-route-servers/bgppeer-global.yml @@ -1,10 +1,10 @@ apiVersion: metallb.io/v1beta2 kind: BGPPeer metadata: - name: asn100-global-lo100 + name: global-lo1 namespace: metallb spec: - myASN: 65000 - peerASN: 100 - peerAddress: 10.0.0.0 - + myASN: 4259840003 + peerASN: 4259840002 + peerAddress: 192.168.255.0 + ebgpMultiHop: True diff --git a/examples/load-balanced-route-servers/meshrr-core.service.yml b/examples/load-balanced-route-servers/meshrr-core.service.yml index aa2d3b7..2bdf018 100644 --- a/examples/load-balanced-route-servers/meshrr-core.service.yml +++ b/examples/load-balanced-route-servers/meshrr-core.service.yml @@ -12,26 +12,26 @@ spec: protocol: TCP targetPort: bgp selector: - app: routeserver + app: meshrr-evpnrs type: ClusterIP ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: routeserver-mgt - namespace: metallb -spec: - addresses: - - 192.168.18.0/24 - autoAssign: false ---- -apiVersion: metallb.io/v1beta1 -kind: L2Advertisement -metadata: - name: routeserver-mgt - namespace: metallb -spec: - ipAddressPools: - - routeserver-mgt - interfaces: - - ens3 +# --- +# apiVersion: metallb.io/v1beta1 +# kind: IPAddressPool +# metadata: +# name: routeserver-mgt +# namespace: metallb +# spec: +# addresses: +# - 192.168.18.0/24 +# autoAssign: false +# --- +# apiVersion: metallb.io/v1beta1 +# kind: L2Advertisement +# metadata: +# name: routeserver-mgt +# namespace: metallb +# spec: +# ipAddressPools: +# - routeserver-mgt +# interfaces: +# - ens3 diff --git a/examples/load-balanced-route-servers/routeserver-1-a.ss.yml b/examples/load-balanced-route-servers/routeserver-1-a.ss.yml deleted file mode 100644 index 8de943d..0000000 --- a/examples/load-balanced-route-servers/routeserver-1-a.ss.yml +++ /dev/null @@ -1,208 +0,0 @@ ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: routeserver-1-a - namespace: metallb -spec: - addresses: - - 10.0.0.1/32 - autoAssign: false ---- -apiVersion: metallb.io/v1beta1 -kind: BGPAdvertisement -metadata: - name: routeserver-1-a - namespace: metallb -spec: - ipAddressPools: - - routeserver-1-a - communities: - - 65000:0 ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-1-a - annotations: - metallb.universe.tf/address-pool: routeserver-1-a -spec: - ports: - - name: bgp - port: 179 - protocol: TCP - targetPort: bgp - selector: - app: routeserver - region: "1" - side: a - type: LoadBalancer - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-1-a-0-mgt - annotations: - metallb.universe.tf/address-pool: routeserver-mgt -spec: - ports: - - name: ssh - port: 22 - protocol: TCP - targetPort: ssh - - name: netconf - port: 830 - protocol: TCP - targetPort: netconf - selector: - statefulset.kubernetes.io/pod-name: routeserver-1-a-0 - type: LoadBalancer - loadBalancerIP: 192.168.18.3 - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-1-a-1-mgt - annotations: - metallb.universe.tf/address-pool: routeserver-mgt -spec: - ports: - - name: ssh - port: 22 - protocol: TCP - targetPort: ssh - - name: netconf - port: 830 - protocol: TCP - targetPort: netconf - selector: - statefulset.kubernetes.io/pod-name: routeserver-1-a-1 - type: LoadBalancer - loadBalancerIP: 192.168.18.6 - externalTrafficPolicy: Local ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: routeserver-1-a - labels: - app: routeserver - region: "1" - side: a -spec: - serviceName: routeserver - podManagementPolicy: "Parallel" - selector: - matchLabels: - app: routeserver - region: "1" - side: a - replicas: 1 - template: - metadata: - labels: - app: routeserver - region: "1" - side: a - spec: - volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic - containers: - - name: crpd - image: /meshrr:latest - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - initialDelaySeconds: 60 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: bgp - timeoutSeconds: 3 - readinessProbe: - failureThreshold: 5 - initialDelaySeconds: 30 - periodSeconds: 5 - successThreshold: 2 - tcpSocket: - port: bgp - timeoutSeconds: 3 - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - ports: - - name: bgp - containerPort: 179 - protocol: TCP - - name: ssh - containerPort: 22 - protocol: TCP - - name: netconf - containerPort: 830 - protocol: TCP - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-core - - name: MESHRR_MODE - value: routeserver - - name: MESHRR_ASRANGE - value: 65001-65535 - - name: MESHRR_CLIENTRANGE - value: 10.0/16 - - name: MESHRR_FAMILY_EVPN - value: "true" - - name: MESHRR_FAMILY_INET - value: "false" - volumeMounts: - - name: config - mountPath: /config - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - securityContext: - allowPrivilegeEscalation: true - privileged: true - runAsNonRoot: false - imagePullSecrets: - - name: regcred - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: side - operator: In - values: - - "a" - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: region - operator: In - values: - - "1" - volumeClaimTemplates: - - metadata: - name: config - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi - diff --git a/examples/load-balanced-route-servers/routeserver-1-b.ss.yml b/examples/load-balanced-route-servers/routeserver-1-b.ss.yml deleted file mode 100644 index da96dff..0000000 --- a/examples/load-balanced-route-servers/routeserver-1-b.ss.yml +++ /dev/null @@ -1,186 +0,0 @@ ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: routeserver-1-b - namespace: metallb -spec: - addresses: - - 10.0.0.2/32 - autoAssign: false ---- -apiVersion: metallb.io/v1beta1 -kind: BGPAdvertisement -metadata: - name: routeserver-1-b - namespace: metallb -spec: - ipAddressPools: - - routeserver-1-b - communities: - - 65000:0 ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-1-b - annotations: - metallb.universe.tf/address-pool: routeserver-1-b -spec: - ports: - - name: bgp - port: 179 - protocol: TCP - targetPort: bgp - selector: - app: routeserver - region: "1" - side: b - type: LoadBalancer - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-1-b-0-mgt - annotations: - metallb.universe.tf/address-pool: routeserver-mgt -spec: - ports: - - name: ssh - port: 22 - protocol: TCP - targetPort: ssh - - name: netconf - port: 830 - protocol: TCP - targetPort: netconf - selector: - statefulset.kubernetes.io/pod-name: routeserver-1-b-0 - type: LoadBalancer - loadBalancerIP: 192.168.18.4 - externalTrafficPolicy: Local ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: routeserver-1-b - labels: - app: routeserver - region: "1" - side: b -spec: - serviceName: routeserver - podManagementPolicy: "Parallel" - selector: - matchLabels: - app: routeserver - region: "1" - side: b - replicas: 1 - template: - metadata: - labels: - app: routeserver - region: "1" - side: b - spec: - volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic - containers: - - name: crpd - image: /meshrr:latest - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - initialDelaySeconds: 60 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: bgp - timeoutSeconds: 3 - readinessProbe: - failureThreshold: 5 - initialDelaySeconds: 30 - periodSeconds: 5 - successThreshold: 2 - tcpSocket: - port: bgp - timeoutSeconds: 3 - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - ports: - - name: bgp - containerPort: 179 - protocol: TCP - - name: ssh - containerPort: 22 - protocol: TCP - - name: netconf - containerPort: 830 - protocol: TCP - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-core - - name: MESHRR_MODE - value: routeserver - - name: MESHRR_ASRANGE - value: 65001-65535 - - name: MESHRR_CLIENTRANGE - value: 10.0/16 - - name: MESHRR_FAMILY_EVPN - value: "true" - - name: MESHRR_FAMILY_INET - value: "false" - volumeMounts: - - name: config - mountPath: /config - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - securityContext: - allowPrivilegeEscalation: true - privileged: true - runAsNonRoot: false - imagePullSecrets: - - name: regcred - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: side - operator: In - values: - - "b" - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: region - operator: In - values: - - "1" - volumeClaimTemplates: - - metadata: - name: config - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi - diff --git a/examples/load-balanced-route-servers/routeserver-2-a.ss.yml b/examples/load-balanced-route-servers/routeserver-2-a.ss.yml deleted file mode 100644 index c715d18..0000000 --- a/examples/load-balanced-route-servers/routeserver-2-a.ss.yml +++ /dev/null @@ -1,208 +0,0 @@ ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: routeserver-2-a - namespace: metallb -spec: - addresses: - - 10.0.0.3/32 - autoAssign: false ---- -apiVersion: metallb.io/v1beta1 -kind: BGPAdvertisement -metadata: - name: routeserver-2-a - namespace: metallb -spec: - ipAddressPools: - - routeserver-2-a - communities: - - 65000:0 ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-2-a - annotations: - metallb.universe.tf/address-pool: routeserver-2-a -spec: - ports: - - name: bgp - port: 179 - protocol: TCP - targetPort: bgp - selector: - app: routeserver - region: "2" - side: a - type: LoadBalancer - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-2-a-0-mgt - annotations: - metallb.universe.tf/address-pool: routeserver-mgt -spec: - ports: - - name: ssh - port: 22 - protocol: TCP - targetPort: ssh - - name: netconf - port: 830 - protocol: TCP - targetPort: netconf - selector: - statefulset.kubernetes.io/pod-name: routeserver-2-a-0 - type: LoadBalancer - loadBalancerIP: 192.168.18.5 - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-2-a-1-mgt - annotations: - metallb.universe.tf/address-pool: routeserver-mgt -spec: - ports: - - name: ssh - port: 22 - protocol: TCP - targetPort: ssh - - name: netconf - port: 830 - protocol: TCP - targetPort: netconf - selector: - statefulset.kubernetes.io/pod-name: routeserver-2-a-1 - type: LoadBalancer - loadBalancerIP: 192.168.18.6 - externalTrafficPolicy: Local ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: routeserver-2-a - labels: - app: routeserver - region: "2" - side: a -spec: - serviceName: routeserver - podManagementPolicy: "Parallel" - selector: - matchLabels: - app: routeserver - region: "2" - side: a - replicas: 1 - template: - metadata: - labels: - app: routeserver - region: "2" - side: a - spec: - volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic - containers: - - name: crpd - image: /meshrr:latest - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - initialDelaySeconds: 60 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: bgp - timeoutSeconds: 3 - readinessProbe: - failureThreshold: 5 - initialDelaySeconds: 30 - periodSeconds: 5 - successThreshold: 2 - tcpSocket: - port: bgp - timeoutSeconds: 3 - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - ports: - - name: bgp - containerPort: 179 - protocol: TCP - - name: ssh - containerPort: 22 - protocol: TCP - - name: netconf - containerPort: 830 - protocol: TCP - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-core - - name: MESHRR_MODE - value: routeserver - - name: MESHRR_ASRANGE - value: 65001-65535 - - name: MESHRR_CLIENTRANGE - value: 10.0/16 - - name: MESHRR_FAMILY_EVPN - value: "true" - - name: MESHRR_FAMILY_INET - value: "false" - volumeMounts: - - name: config - mountPath: /config - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - securityContext: - allowPrivilegeEscalation: true - privileged: true - runAsNonRoot: false - imagePullSecrets: - - name: regcred - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: side - operator: In - values: - - "a" - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: region - operator: In - values: - - "2" - volumeClaimTemplates: - - metadata: - name: config - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi - diff --git a/examples/load-balanced-route-servers/routeserver-2-b.ss.yml b/examples/load-balanced-route-servers/routeserver-2-b.ss.yml deleted file mode 100644 index 60b69a3..0000000 --- a/examples/load-balanced-route-servers/routeserver-2-b.ss.yml +++ /dev/null @@ -1,186 +0,0 @@ ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: routeserver-2-b - namespace: metallb -spec: - addresses: - - 10.0.0.4/32 - autoAssign: false ---- -apiVersion: metallb.io/v1beta1 -kind: BGPAdvertisement -metadata: - name: routeserver-2-b - namespace: metallb -spec: - ipAddressPools: - - routeserver-2-b - communities: - - 65000:0 ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-2-b - annotations: - metallb.universe.tf/address-pool: routeserver-2-b -spec: - ports: - - name: bgp - port: 179 - protocol: TCP - targetPort: bgp - selector: - app: routeserver - region: "2" - side: b - type: LoadBalancer - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-2-b-0-mgt - annotations: - metallb.universe.tf/address-pool: routeserver-mgt -spec: - ports: - - name: ssh - port: 22 - protocol: TCP - targetPort: ssh - - name: netconf - port: 830 - protocol: TCP - targetPort: netconf - selector: - statefulset.kubernetes.io/pod-name: routeserver-2-b-0 - type: LoadBalancer - loadBalancerIP: 192.168.18.7 - externalTrafficPolicy: Local ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: routeserver-2-b - labels: - app: routeserver - region: "2" - side: b -spec: - serviceName: routeserver - podManagementPolicy: "Parallel" - selector: - matchLabels: - app: routeserver - region: "2" - side: b - replicas: 1 - template: - metadata: - labels: - app: routeserver - region: "2" - side: b - spec: - volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic - containers: - - name: crpd - image: /meshrr:latest - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - initialDelaySeconds: 60 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: bgp - timeoutSeconds: 3 - readinessProbe: - failureThreshold: 5 - initialDelaySeconds: 30 - periodSeconds: 5 - successThreshold: 2 - tcpSocket: - port: bgp - timeoutSeconds: 3 - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - ports: - - name: bgp - containerPort: 179 - protocol: TCP - - name: ssh - containerPort: 22 - protocol: TCP - - name: netconf - containerPort: 830 - protocol: TCP - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-core - - name: MESHRR_MODE - value: routeserver - - name: MESHRR_ASRANGE - value: 65001-65535 - - name: MESHRR_CLIENTRANGE - value: 10.0/16 - - name: MESHRR_FAMILY_EVPN - value: "true" - - name: MESHRR_FAMILY_INET - value: "false" - volumeMounts: - - name: config - mountPath: /config - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - securityContext: - allowPrivilegeEscalation: true - privileged: true - runAsNonRoot: false - imagePullSecrets: - - name: regcred - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: side - operator: In - values: - - "b" - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: region - operator: In - values: - - "2" - volumeClaimTemplates: - - metadata: - name: config - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi - diff --git a/examples/load-balanced-route-servers/routeserver-a.ss.yml b/examples/load-balanced-route-servers/routeserver-a.ss.yml new file mode 100644 index 0000000..f8b61d4 --- /dev/null +++ b/examples/load-balanced-route-servers/routeserver-a.ss.yml @@ -0,0 +1,247 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: meshrr-conf +data: + meshrr.conf.yml: |+ + encrypted_root_pw: NOLOGIN + asn: "65000.1" + mode: evpnrs + bgpgroups: + - name: MESHRR-CORE + type: mesh + source: + sourcetype: dns + hostname: meshrr-core + - name: MESHRR-CLIENTS + type: subtractive + prefixes: + - 192.168.0.0/16 + asranges: + - 65001-65535 +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: routeserver-a + namespace: metallb +spec: + addresses: + - 192.168.255.1/32 + autoAssign: false +--- +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: routeserver-a + namespace: metallb +spec: + ipAddressPools: + - routeserver-a + communities: + - 65000:0 +--- +apiVersion: v1 +kind: Service +metadata: + name: routeserver-a + annotations: + metallb.universe.tf/address-pool: routeserver-a +spec: + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr-evpnrs + side: a + type: LoadBalancer + externalTrafficPolicy: Local +# --- +# apiVersion: v1 +# kind: Service +# metadata: +# name: routeserver-1-a-0-mgt +# annotations: +# metallb.universe.tf/address-pool: routeserver-mgt +# spec: +# ports: +# - name: ssh +# port: 22 +# protocol: TCP +# targetPort: ssh +# - name: netconf +# port: 830 +# protocol: TCP +# targetPort: netconf +# selector: +# statefulset.kubernetes.io/pod-name: routeserver-1-a-0 +# type: LoadBalancer +# loadBalancerIP: 192.168.18.3 +# externalTrafficPolicy: Local +# --- +# apiVersion: v1 +# kind: Service +# metadata: +# name: routeserver-1-a-1-mgt +# annotations: +# metallb.universe.tf/address-pool: routeserver-mgt +# spec: +# ports: +# - name: ssh +# port: 22 +# protocol: TCP +# targetPort: ssh +# - name: netconf +# port: 830 +# protocol: TCP +# targetPort: netconf +# selector: +# statefulset.kubernetes.io/pod-name: routeserver-1-a-1 +# type: LoadBalancer +# loadBalancerIP: 192.168.18.6 +# externalTrafficPolicy: Local +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: routeserver-a + labels: + app: meshrr-evpnrs + side: a +spec: + serviceName: routeserver + podManagementPolicy: "Parallel" + selector: + matchLabels: + app: meshrr-evpnrs + side: a + replicas: 2 + template: + metadata: + labels: + app: meshrr-evpnrs + side: a + spec: + volumes: + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-conf + optional: false + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + - name: MESHRR_MODE + value: evpnrs + containers: + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: Never + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 60 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: bgp + timeoutSeconds: 3 + readinessProbe: + failureThreshold: 5 + initialDelaySeconds: 30 + periodSeconds: 5 + successThreshold: 2 + tcpSocket: + port: bgp + timeoutSeconds: 3 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + ports: + - name: bgp + containerPort: 179 + protocol: TCP + - name: ssh + containerPort: 22 + protocol: TCP + - name: netconf + containerPort: 830 + protocol: TCP + volumeMounts: + - name: config + mountPath: /config/ + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsNonRoot: false + - name: meshrr + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: side + operator: In + values: + - "a" + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - meshrr-evpnrs + topologyKey: kubernetes.io/hostname diff --git a/examples/load-balanced-route-servers/routeserver-b.ss.yml b/examples/load-balanced-route-servers/routeserver-b.ss.yml new file mode 100644 index 0000000..56ee83e --- /dev/null +++ b/examples/load-balanced-route-servers/routeserver-b.ss.yml @@ -0,0 +1,203 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: meshrr-conf +data: + meshrr.conf.yml: |+ + encrypted_root_pw: NOLOGIN + asn: "65000.1" + mode: evpnrs + bgpgroups: + - name: MESHRR-CORE + type: mesh + source: + sourcetype: dns + hostname: meshrr-core + - name: MESHRR-CLIENTS + type: subtractive + prefixes: + - 192.168.0.0/16 + asranges: + - 65001-65535 +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: routeserver-b + namespace: metallb +spec: + addresses: + - 192.168.255.2/32 + autoAssign: false +--- +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: routeserver-b + namespace: metallb +spec: + ipAddressPools: + - routeserver-b + communities: + - 65000:0 +--- +apiVersion: v1 +kind: Service +metadata: + name: routeserver-b + annotations: + metallb.universe.tf/address-pool: routeserver-b +spec: + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr-evpnrs + side: b + type: LoadBalancer + externalTrafficPolicy: Local +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: routeserver-b + labels: + app: meshrr-evpnrs + side: b +spec: + serviceName: routeserver + podManagementPolicy: "Parallel" + selector: + matchLabels: + app: meshrr-evpnrs + side: b + replicas: 2 + template: + metadata: + labels: + app: meshrr-evpnrs + side: b + spec: + volumes: + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-conf + optional: false + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + - name: MESHRR_MODE + value: evpnrs + containers: + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: Never + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 60 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: bgp + timeoutSeconds: 3 + readinessProbe: + failureThreshold: 5 + initialDelaySeconds: 30 + periodSeconds: 5 + successThreshold: 2 + tcpSocket: + port: bgp + timeoutSeconds: 3 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + ports: + - name: bgp + containerPort: 179 + protocol: TCP + - name: ssh + containerPort: 22 + protocol: TCP + - name: netconf + containerPort: 830 + protocol: TCP + volumeMounts: + - name: config + mountPath: /config/ + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsNonRoot: false + - name: meshrr + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: side + operator: In + values: + - "b" + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - meshrr-evpnrs + topologyKey: kubernetes.io/hostname diff --git a/meshrr/Dockerfile b/meshrr/Dockerfile index 01f0797..b9ed877 100644 --- a/meshrr/Dockerfile +++ b/meshrr/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) Juniper Networks, Inc., 2020. All rights reserved. +# Copyright (c) Juniper Networks, Inc., 2023. All rights reserved. # # Notice and Disclaimer: This code is licensed to you under the MIT License (the # "License"). You may not use this code except in compliance with the License. @@ -27,32 +27,30 @@ # components is subject to the terms and conditions of the respective license as # noted in the Third-Party source code file. -FROM crpd:22.4R1.10 as prebuild +FROM alpine:3.18 as meshrr-prebuild -WORKDIR /root +WORKDIR /opt/meshrr -COPY requirements.txt /root/ +COPY requirements.txt /opt/meshrr/ -# This blocks cron startup if not changed -RUN printf '#!/bin/sh\nexit 0\n' > /usr/sbin/policy-rc.d - -# Install requirements. Note that cron must be reconfigured later to create user. -RUN apt-get update \ - && apt-get -y --no-install-recommends install cron python3-minimal python3-setuptools python3-pip \ - && rm -fr /var/lib/apt/lists/* \ - && pip3 install wheel \ +RUN apk add --update --no-cache python3 py3-pip openssh \ && pip3 install -r requirements.txt -FROM prebuild +FROM meshrr-prebuild + +RUN mkdir -p /config /secret/ssh conf -COPY render_config.py /root/ -COPY runit-init.sh /sbin/ -COPY juniper.conf.j2 /root/ -COPY update_peers.py /root/ +COPY config.py /opt/meshrr/ +COPY render_config.py /opt/meshrr/ +COPY run.sh /opt/meshrr/ +COPY defaults/ /opt/meshrr/defaults/ +COPY update_peers.py /opt/meshrr/ +COPY connect_wait.py /opt/meshrr/ +COPY ../LICENSE /opt/meshrr/ -RUN chmod +x /sbin/runit-init.sh -RUN chmod +x /root/update_peers.py -RUN chmod +x /root/render_config.py +RUN chmod +x /opt/meshrr/run.sh +RUN chmod +x /opt/meshrr/connect_wait.py +RUN chmod +x /opt/meshrr/update_peers.py +RUN chmod +x /opt/meshrr/render_config.py -ENTRYPOINT [ "/sbin/runit-init.sh" ] -STOPSIGNAL 35 +ENTRYPOINT [ "/opt/meshrr/run.sh" ] \ No newline at end of file diff --git a/meshrr/LICENSE b/meshrr/LICENSE new file mode 100644 index 0000000..3e5398d --- /dev/null +++ b/meshrr/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Juniper Networks, Inc. 2023 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/meshrr/config.py b/meshrr/config.py new file mode 100755 index 0000000..e771705 --- /dev/null +++ b/meshrr/config.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 + +# Copyright (c) Juniper Networks, Inc., 2023. All rights reserved. +# +# Notice and Disclaimer: This code is licensed to you under the MIT License (the +# "License"). You may not use this code except in compliance with the License. +# This code is not an official Juniper product. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Third-Party Code: This code may depend on other components under separate +# copyright notice and license terms. Your use of the source code for those +# components is subject to the terms and conditions of the respective license as +# noted in the Third-Party source code file. + +import yaml +from yaml import SafeLoader +from os import getenv +from dotenv import load_dotenv +from os.path import isfile + +load_dotenv(dotenv_path="/etc/envvars") + +class Meshrrconfig: + def __init__(self): + if not isfile('/opt/meshrr/conf/meshrr.conf.yml'): + # Start from default config + with open('/opt/meshrr/defaults/meshrr.conf.yml') as d: + c = yaml.load(d,Loader=SafeLoader) + + # Copy in defined environment variables. + c.update({ + 'encrypted_root_pw': getenv("ENCRYPTED_ROOT_PW", c['encrypted_root_pw']), + 'asn': getenv("AUTONOMOUS_SYSTEM", c['asn']), + 'mode': getenv("MESHRR_MODE", c['mode']), + }) + + # Save to live config + with open('/opt/meshrr/conf/meshrr.conf.yml','w') as w: + yaml.dump(c,w) + with open('/opt/meshrr/conf/meshrr.conf.yml') as f: + self._config = yaml.load(f,Loader=SafeLoader) + + self.encrypted_root_pw = self._config['encrypted_root_pw'] + self.asn = self._config['asn'] + self.mode = self._config['mode'] + + self.bgpgroups_mesh = list() + self.bgpgroups_subtractive = list() + for group in self._config['bgpgroups']: + if 'type' not in group or 'name' not in group: + raise Exception("All bgpgroups require 'type' and 'name' definitions. Offending bgpgroup: "+str(group)) + elif group['type'].casefold() == 'mesh': + if 'source' not in group: + raise Exception("Mesh bgpgroups require 'source.hostname' definitions. Offending bgpgroup: "+str(group)) + self.bgpgroups_mesh.append(group) + elif group['type'].casefold() == 'subtractive': + if 'prefixes' not in group: + raise(Exception("Subtractive bgpgroups require 'source.hostname' definitions. Offending bgpgroup: "+str(group))) + self.bgpgroups_subtractive.append(group) + else: + raise(Exception("Invalid `type` in bgpgroup: "+str(group))) + + def get_bgpgroups_dict(self, grouptype): + if grouptype == 'mesh': + bgpgroups = self.bgpgroups_mesh + elif grouptype == 'subtractive': + bgpgroups = self.bgpgroups_subtractive + else: + raise Exception("Invalid grouptype: ${grouptype}") + retval = dict() + for bgpgroup in bgpgroups: + retval.update({bgpgroup.pop('name'): bgpgroup}) + return retval + \ No newline at end of file diff --git a/meshrr/runit-init.sh b/meshrr/connect_wait.py similarity index 63% rename from meshrr/runit-init.sh rename to meshrr/connect_wait.py index e2f9188..b262c73 100644 --- a/meshrr/runit-init.sh +++ b/meshrr/connect_wait.py @@ -1,21 +1,21 @@ -#!/bin/bash +#!/usr/bin/env python3 -# Copyright (c) Juniper Networks, Inc., 2020. All rights reserved. -# +# Copyright (c) Juniper Networks, Inc., 2023. All rights reserved. +# # Notice and Disclaimer: This code is licensed to you under the MIT License (the # "License"). You may not use this code except in compliance with the License. # This code is not an official Juniper product. -# +# # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: -# +# # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. -# +# # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -29,24 +29,24 @@ # components is subject to the terms and conditions of the respective license as # noted in the Third-Party source code file. -set -e - -printenv > /etc/envvars - -# Overwrite with existing configuration if it exists. -if [ -f "/config/juniper.conf" ]; then - sed "s/^\(.\+router-id\) [0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+/\1 {{ POD_IP }};/" /config/juniper.conf > juniper.conf.j2 -fi - -./render_config.py -i juniper.conf.j2 -o /config/juniper.conf - -# Install crontab. Expect non-zero status for crontab -l -line="* * * * * /root/update_peers.py >> /var/log/update_peers 2>&1" -set +e - -# crontab group doesn't seem to be created in docker build. -dpkg-reconfigure cron -(crontab -l 2>/dev/null; echo "$line" ) | crontab - -set -e - -exec /sbin/runit-init 0 \ No newline at end of file +from jnpr.junos import Device +from jnpr.junos.exception import ConnectTimeoutError,ConnectRefusedError + +from datetime import datetime +from time import sleep + +if __name__ == "__main__": + # Attempt to open a connection to the device + dev = Device(host="127.0.0.1",user="meshrr",ssh_private_key_file="/secret/ssh/id_ed25519") + while True: + try: + dev.open() + break + except ConnectTimeoutError: + print(f"[{datetime.now()}]","connect_wait.py: Connection timed out; retrying.") + except ConnectRefusedError: + print(f"[{datetime.now()}]","connect_wait.py: Connection refused; retrying.") + sleep(1) + # Create /tmp/connected-to-crpd to inform startup probes + open("/tmp/connected-to-crpd","x") + \ No newline at end of file diff --git a/meshrr/defaults/juniper-evpnrs.conf.j2 b/meshrr/defaults/juniper-evpnrs.conf.j2 new file mode 100644 index 0000000..107839f --- /dev/null +++ b/meshrr/defaults/juniper-evpnrs.conf.j2 @@ -0,0 +1,87 @@ +system { + /* Required, though use of a variable is optional */ + root-authentication { + encrypted-password "{{ encrypted_root_pw }}"; ## SECRET-DATA + } + login { + /* Required for meshrr authentication */ + user meshrr { + class super-user; + uid 100; + authentication { + ssh-ed25519 "ssh-ed25519 TBD"; ## SECRET-DATA + } + } + } + /* Required for configuration persistence */ + no-compress-configuration-files; + services { + netconf { + /* Required for meshrr access */ + ssh; + } + } + /* Required for licensing unless using an alternative approach. */ + license { + keys { + key "{{ LICENSE_KEY }}"; + } + } + processes { + routing { + bgp { + rib-sharding; + update-threading; + } + } + } +} +routing-options { + autonomous-system {{ asn }}; + router-id {{ POD_IP }}; +} +groups { + MESHRR { + policy-options { + {%- for bgpgroup in bgpgroups_subtractive %} + as-list MESHRR:{{ bgpgroup }} members [ {% for asrange in bgpgroups_subtractive[bgpgroup].asranges %}{{ asrange }} {% endfor %}]; + {%- endfor %} + } + protocols { + bgp { + family evpn { + signaling { + nexthop-resolution { + no-resolution; + } + no-install; + } + }{%- for bgpgroup in bgpgroups_mesh %} + /* Mesh {% if bgpgroups_mesh[bgpgroup].max_peers is defined %}max_peers:{{ bgpgroups_mesh[bgpgroup].max_peers }} {% endif %}group from {{ bgpgroups_mesh[bgpgroup].source.sourcetype }}:{{ bgpgroups_mesh[bgpgroup].source.hostname }} */ + group {{ bgpgroup }} { + type internal + family evpn { + signaling; + } + } + {%- endfor %} + {%- for bgpgroup in bgpgroups_subtractive %} + /* Subtractive group from {{ bgpgroups_subtractive[bgpgroup].prefixes }} */ + group {{ bgpgroup }} { + type external; + multihop { + ttl 10; + no-nexthop-change; + } + family evpn { + signaling; + } + peer-as-list MESHRR:{{ bgpgroup }}; + allow [ {% for prefix in bgpgroups_subtractive[bgpgroup].prefixes %}{{ prefix }} {% endfor %}]; + } + {%- endfor %} + } + } + } +} +apply-groups MESHRR; diff --git a/meshrr/defaults/juniper-ipv4rr.conf.j2 b/meshrr/defaults/juniper-ipv4rr.conf.j2 new file mode 100644 index 0000000..7f4131b --- /dev/null +++ b/meshrr/defaults/juniper-ipv4rr.conf.j2 @@ -0,0 +1,76 @@ +system { + /* Required, though use of a variable is optional */ + root-authentication { + encrypted-password "{{ encrypted_root_pw }}"; ## SECRET-DATA + } + login { + /* Required for meshrr authentication */ + user meshrr { + class super-user; + uid 100; + authentication { + ssh-ed25519 "ssh-ed25519 TBD"; ## SECRET-DATA + } + } + } + /* Required for configuration persistence */ + no-compress-configuration-files; + services { + netconf { + /* Required for meshrr access */ + ssh; + } + } + /* Required for licensing unless using an alternative approach. */ + license { + keys { + key "{{ LICENSE_KEY }}"; + } + } + processes { + routing { + bgp { + rib-sharding; + update-threading; + } + } + } +} +routing-options { + autonomous-system {{ asn }}; + router-id {{ POD_IP }}; +} +groups { + MESHRR { + protocols { + bgp { + family inet { + unicast { + nexthop-resolution { + no-resolution; + } + no-install; + } + } + {%- for bgpgroup in bgpgroups_mesh %} + /* Mesh {% if bgpgroups_mesh[bgpgroup].max_peers is defined %}max_peers:{{ bgpgroups_mesh[bgpgroup].max_peers }} {% endif %}group from {{ bgpgroups_mesh[bgpgroup].source.sourcetype }}:{{ bgpgroups_mesh[bgpgroup].source.hostname }} */ + group {{ bgpgroup }} { + type internal + family inet { + unicast; + } + } + {%- endfor %} + {%- for bgpgroup in bgpgroups_subtractive %} + /* Subtractive group from {{ bgpgroups_subtractive[bgpgroup].prefixes }} */ + group {{ bgpgroup }} { + type internal; + cluster {{ POD_IP }}; + allow [ {% for prefix in bgpgroups_subtractive[bgpgroup].prefixes %}{{ prefix }} {% endfor %}]; + } + {%- endfor %} + } + } + } +} +apply-groups MESHRR; diff --git a/meshrr/defaults/meshrr.conf.yml b/meshrr/defaults/meshrr.conf.yml new file mode 100644 index 0000000..fc2ad49 --- /dev/null +++ b/meshrr/defaults/meshrr.conf.yml @@ -0,0 +1,22 @@ +encrypted_root_pw: NOLOGIN +asn: "65000" +mode: ipv4rr +bgpgroups: +- name: MESHRR-MESH + type: mesh + source: + sourcetype: dns + hostname: meshrr # FQDN for svc required if not in same namespace +- name: MESHRR-CLIENTS + type: subtractive # Prefixes in multiple external-subtractive groups must not overlap + prefixes: + - 0.0.0.0/0 + # For routeserver use case, an AS range is needed; we don't set this for RR use case. + # asranges: + # - 65001-65500 +- name: MESHRR-UPSTREAM + type: mesh + source: + sourcetype: dns + hostname: meshrr.core.svc.cluster.local # FQDN required if svc not in same namespace + max_peers: 2 diff --git a/meshrr/juniper.conf.j2 b/meshrr/juniper.conf.j2 deleted file mode 100644 index 34af20b..0000000 --- a/meshrr/juniper.conf.j2 +++ /dev/null @@ -1,81 +0,0 @@ -system { - root-authentication { - encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA - } - /* Required for configuration persistence */ - no-compress-configuration-files; - processes { - routing { - bgp { - rib-sharding; - update-threading; - } - } - } -} -routing-options { - autonomous-system {{ AUTONOMOUS_SYSTEM }}; - router-id {{ POD_IP }}; -} -groups { - MESHRR { - {% if MESHRR_MODE == 'routeserver' %}policy-options { - as-list MESHRR-RSCLIENTS members {{ MESHRR_ASRANGE }}; - } - {% endif %}protocols { - bgp { - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast { - nexthop-resolution { - no-resolution; - } - no-install; - } - }{% endif %} - {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { - signaling { - nexthop-resolution { - no-resolution; - } - no-install; - } - }{% endif %} - group MESHRR-MESH { - type internal - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast; - }{% endif %} - {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { - signaling; - }{% endif %} - } - group MESHRR-CLIENTS { - {% if MESHRR_MODE == 'routeserver' %}type external; - multihop { - ttl 10; - no-nexthop-change; - } - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast; - }{% endif %} - {% if MESHRR_FAMILY_EVPN %}family evpn { - signaling; - }{% endif %} - peer-as-list MESHRR-RSCLIENTS;{% else %}type internal; - cluster {{ POD_IP }};{% endif %} - allow {{ MESHRR_CLIENTRANGE }}; - }{% if UPSTREAM_SERVICE_NAME is defined and UPSTREAM_SERVICE_NAME is not none %} - group MESHRR-UPSTREAM { - type internal; - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast; - }{% endif %} - {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { - signaling; - }{% endif %} - }{% endif %} - } - } - } -} -apply-groups MESHRR; diff --git a/meshrr/render_config.py b/meshrr/render_config.py index 7e0d70c..6558a22 100755 --- a/meshrr/render_config.py +++ b/meshrr/render_config.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (c) Juniper Networks, Inc., 2020. All rights reserved. +# Copyright (c) Juniper Networks, Inc., 2023. All rights reserved. # # Notice and Disclaimer: This code is licensed to you under the MIT License (the # "License"). You may not use this code except in compliance with the License. @@ -31,9 +31,12 @@ from argparse import ArgumentParser, FileType from os import getenv - from jinja2 import Template +from config import Meshrrconfig + +mconf=Meshrrconfig() + if __name__ == "__main__": parser = ArgumentParser( @@ -47,27 +50,22 @@ template = Template(args.inputfile.read()) configvars = dict() - configvars.update({"ENCRYPTED_ROOT_PW": getenv("ENCRYPTED_ROOT_PW", None)}) - if not configvars["ENCRYPTED_ROOT_PW"]: - raise (Exception("ENCRYPTED_ROOT_PW is not set.")) - configvars.update({"AUTONOMOUS_SYSTEM": getenv("AUTONOMOUS_SYSTEM", None)}) - if not configvars["AUTONOMOUS_SYSTEM"]: - raise (Exception("AUTONOMOUS_SYSTEM is not set.")) + + # Populate variables to be used in templates that must come from env + configvars.update({"LICENSE_KEY": getenv("LICENSE_KEY", None)}) + if not configvars["LICENSE_KEY"]: + raise (Exception("LICENSE_KEY is not set.")) configvars.update({"POD_IP": getenv("POD_IP", None)}) if not configvars["POD_IP"]: raise (Exception("POD_IP is not set.")) - configvars.update({"MESHRR_CLIENTRANGE": getenv("MESHRR_CLIENTRANGE", None)}) - if not configvars["MESHRR_CLIENTRANGE"]: - raise (Exception("MESHRR_CLIENTRANGE is not set.")) - configvars.update({"UPSTREAM_SERVICE_NAME": getenv("UPSTREAM_SERVICE_NAME", None)}) - # Default to Route Reflector Mode - configvars.update({"MESHRR_MODE": getenv("MESHRR_MODE", "routereflector")}) - # Default AS Range for Route Server mode is 65001 to 65000 - configvars.update({"MESHRR_ASRANGE": getenv("MESHRR_ASRANGE", "65001-65500")}) - # Default to inet unicast only - configvars.update({"MESHRR_FAMILY_INET": getenv("MESHRR_FAMILY_INET", "true")}) - configvars.update({"MESHRR_FAMILY_EVPN": getenv("MESHRR_FAMILY_EVPN", "false")}) + # Populate variables to be used in termplates that will come from Meshrrconfig / YAML + configvars.update({ + "encrypted_root_pw": mconf.encrypted_root_pw, + "asn": mconf.asn, + "bgpgroups_mesh": mconf.get_bgpgroups_dict('mesh'), + "bgpgroups_subtractive": mconf.get_bgpgroups_dict('subtractive') + }) crpd_config = template.render(configvars) args.outputfile.write(crpd_config) diff --git a/meshrr/requirements.txt b/meshrr/requirements.txt index 2982694..daccf97 100644 --- a/meshrr/requirements.txt +++ b/meshrr/requirements.txt @@ -1,26 +1,22 @@ -bcrypt==3.2.0 -cffi==1.15.1 +bcrypt==4.0.1 +cffi==1.16.0 cryptography==41.0.4 -dnspython==2.0.0 -importlib-resources==3.3.0 -Jinja2==2.11.3 -junos-eznc==2.6.6 -lxml==4.9.1 -MarkupSafe==1.1.1 +dnspython==2.4.2 +Jinja2==3.1.2 +junos-eznc==2.6.8 +lxml==4.9.3 +MarkupSafe==2.1.3 ncclient==0.6.13 -netaddr==0.8.0 -paramiko==2.10.1 -pip==22.3.1 -pycparser==2.20 -PyNaCl==1.4.0 -pyparsing==2.4.7 +netaddr==0.9.0 +packaging==23.1 +paramiko==3.3.1 +pycparser==2.21 +PyNaCl==1.5.0 +pyparsing==3.0.9 pyserial==3.5 -python-dotenv==0.15.0 -PyYAML>=6.0.1 -scp==0.13.3 -setuptools==65.5.1 -six==1.15.0 -transitions==0.8.5 -wheel==0.38.1 -yamlordereddictloader==0.4.0 -zipp==3.4.0 \ No newline at end of file +python-dotenv==1.0.0 +PyYAML==6.0.1 +scp==0.14.5 +six==1.16.0 +transitions==0.9.0 +yamlordereddictloader==0.4.2 \ No newline at end of file diff --git a/meshrr/run.sh b/meshrr/run.sh new file mode 100644 index 0000000..ab4d8a0 --- /dev/null +++ b/meshrr/run.sh @@ -0,0 +1,77 @@ +#!/bin/sh + +# Copyright (c) Juniper Networks, Inc., 2023. All rights reserved. +# +# Notice and Disclaimer: This code is licensed to you under the MIT License (the +# "License"). You may not use this code except in compliance with the License. +# This code is not an official Juniper product. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Third-Party Code: This code may depend on other components under separate +# copyright notice and license terms. Your use of the source code for those +# components is subject to the terms and conditions of the respective license as +# noted in the Third-Party source code file. + +set -e + +printenv > /etc/envvars + +if [ $# -eq 0 ]; then + echo "One of the following arguments required: init, sidecar" + exit 1 +elif [ $1 = 'init' ]; then + echo "Initializing pod" + # Overwrite with existing configuration if it exists. + if [ -f "/config/juniper.conf" ]; then + echo "Existing configuration detected; overwriting pod IP only." + sed "s/^\(.\+router-id\) [0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+/\1 {{ POD_IP }};/" /config/juniper.conf > conf/juniper.conf.j2 + elif ! test -f conf/juniper.conf.j2; then + if [[ "$MESHRR_MODE" == 'evpnrs' ]] || [[ "$MESHRR_MODE" == 'routeserver' ]] ; then + template='juniper-evpnrs.conf.j2' + elif [[ "$MESHRR_MODE" == 'ipv4rr' ]] || [[ "$MESHRR_MODE" == 'routereflector' ]] || [[ -z $MESHRR_MODE ]]; then + template='juniper-ipv4rr.conf.j2' + else + echo "Invalid MESHRR_MODE set. Must be empty or one of [ 'evpnrs' 'ipv4rr' ]." + exit 1 + fi + echo "Initializing fresh configuration from defaults/${template}." + cp defaults/${template} conf/juniper.conf.j2 + else # Don't overwrite template since it's been explicitly mounted. + echo "Custom configuration provided; using conf/juniper.conf.j2 template." + fi + # Generate a fresh SSH key and apply to configuration template. + ssh-keygen -q -t ed25519 -f /secret/ssh/id_ed25519 -P "" + PUBKEY=`cat \/secret\/ssh\/id_ed25519.pub | tr -d '\r\n'` + # Cannot overwrite in place as this may be a mounted configmap + sed "/user meshrr/,/SECRET-DATA/ s~ssh-ed25519.*~ssh-ed25519 \"$PUBKEY\"; ## SECRET-DATA~" conf/juniper.conf.j2 > /config/juniper.conf.j2 + ./render_config.py -i /config/juniper.conf.j2 -o /config/juniper.conf +elif [ $1 = 'sidecar' ]; then + # Wait for cRPD container to become available for netconf. + ./connect_wait.py + echo "Initializing peer maintenance every ${UPDATE_SECONDS:=30} seconds." + while true; do + ./update_peers.py + sleep ${UPDATE_SECONDS} + done +else + echo "Invalid argument: $1" +fi + + diff --git a/meshrr/update_peers.py b/meshrr/update_peers.py index 2410ef6..bd428fa 100644 --- a/meshrr/update_peers.py +++ b/meshrr/update_peers.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (c) Juniper Networks, Inc., 2020. All rights reserved. +# Copyright (c) Juniper Networks, Inc., 2023. All rights reserved. # # Notice and Disclaimer: This code is licensed to you under the MIT License (the # "License"). You may not use this code except in compliance with the License. @@ -32,28 +32,34 @@ from os import getenv from random import randrange +from datetime import datetime from dns import resolver -from dotenv import load_dotenv from jnpr.junos import Device from jnpr.junos.utils.config import Config -from netaddr import IPNetwork, IPAddress, IPSet, cidr_exclude, cidr_merge +from netaddr import IPSet -load_dotenv(dotenv_path="/etc/envvars") +from config import Meshrrconfig +mconf=Meshrrconfig() + +# Load necessary environment variables +pod_ip = getenv("POD_IP") +if not pod_ip: + raise (Exception("POD_IP environment variable not set")) class ConfigUpdate(Config): # Child class to maintain lists of selected peers in managed groups - def __init__(self, dev, mode=None, **kwargs): + def __init__(self, device, mode=None, **kwargs): self.__selected_peers = dict() - super().__init__(dev, mode, **kwargs) + super().__init__(device, mode, **kwargs) def initiate_group(self, group_name, force=False): """Initiate the group from live configuration only if it's not inventoried yet. Override with force=True""" if group_name not in self.__selected_peers or force: # Get the list of neighbors in the group - filter = f"MESHRR{group_name}" - data = dev.rpc.get_config(filter_xml=filter) + xmlfilter = f"MESHRR{group_name}" + data = dev.rpc.get_config(filter_xml=xmlfilter) configured_peers = data.xpath( f"groups/protocols/bgp/group[name='{group_name}']/neighbor/name/text()" ) @@ -62,7 +68,7 @@ def initiate_group(self, group_name, force=False): else: return False - def get_allowed_peers(self, allowed_base=getenv("MESHRR_CLIENTRANGE")): + def get_allowed_peers(self, bgpgroup): """ Returns the allowed_base minus any explicitly configured managed peers This is required, otherwise configured peers may end up in the wrong group. @@ -70,28 +76,70 @@ def get_allowed_peers(self, allowed_base=getenv("MESHRR_CLIENTRANGE")): """ disallowed = IPSet(cu.get_all_selected_peers()) - allowed = [IPNetwork(allowed_base)] - for disallowed_ip in disallowed: - for network in allowed: - if disallowed_ip in network: - allowed.remove(network) - allowed.extend(cidr_exclude(network, disallowed_ip)) - break - allowed = cidr_merge(allowed) + allowed = IPSet(bgpgroup['prefixes']) + allowed = allowed-disallowed return allowed - def commit_peerupdate(self, **kwargs): - allowed = self.get_allowed_peers() - allowed_string = " ".join([str(network) for network in allowed]) + def update_bgpgroup_mesh(self, bgpgroup): + detected_peers = list() + + # Get peers for DNS based BGP group [DEFAULT] + if 'sourcetype' not in bgpgroup['source'] or bgpgroup['source']['sourcetype'].casefold() == 'dns': + try: + result = resolver.resolve( + bgpgroup['source']['hostname'], "A", search=True + ) + except (resolver.NXDOMAIN, resolver.NoAnswer) as err: + print(f"[{datetime.now()}]", err.msg, f"Skipping processing of {bgpgroup['name']}.") + return + + for r in result: + if r.address != pod_ip: + detected_peers.append(r.address) + else: + raise(Exception(f"Invalid source type for group {bgpgroup['name']}: {bgpgroup['source']['sourcetype']}")) + + # Identify peers that should be active after this commit. + # Currently configured peers that are still detected via DNS are prioritized. + # Additional peers will be selected at random from detected_peers until `max_peers` are selected. + configured_peers = self.get_selected_peers(bgpgroup['name']) + selected_peers = list() + for peer_ip in configured_peers: + if 'max_peers' not in bgpgroup or len(selected_peers) < bgpgroup['max_peers']: + if peer_ip in detected_peers: + selected_peers.append(peer_ip) + detected_peers.remove(peer_ip) + else: + # The peer group is full with selected_peers. No need to continue this loop. + break + while len(detected_peers) and ( + 'max_peers' not in bgpgroup or len(selected_peers) < bgpgroup['max_peers'] + ): + # Add a peer at random from those detected to fill to max_peers. + selected_peers.append(detected_peers.pop(randrange(len(detected_peers)))) + + # Compare detected_peers (up-to-date list) with configured_peers. + # Add detected_peers not in configured_peers. + # Remove configured_peers not in detected_peers. + peers_to_add = set(selected_peers) - set(configured_peers) + for peer_ip in peers_to_add: + self.add_selected_peer(bgpgroup['name'], peer_ip) + + peers_to_remove = set(configured_peers) - set(selected_peers) + for peer_ip in peers_to_remove: + self.remove_selected_peer(bgpgroup['name'], peer_ip) + + def update_bgpgroup_subtractive(self, bgpgroup): + allowed = self.get_allowed_peers(bgpgroup) + allowed_string = " ".join([str(network) for network in allowed.iter_cidrs()]) self.load( - f"delete groups MESHRR protocols bgp group MESHRR-CLIENTS allow", + f"delete groups MESHRR protocols bgp group {bgpgroup['name']} allow", format="set", ) self.load( - f"set groups MESHRR protocols bgp group MESHRR-CLIENTS allow [ {allowed_string} ]", + f"set groups MESHRR protocols bgp group {bgpgroup['name']} allow [ {allowed_string} ]", format="set", ) - self.commit() def get_selected_peers(self, group_name): self.initiate_group(group_name) @@ -99,8 +147,8 @@ def get_selected_peers(self, group_name): def get_all_selected_peers(self): result = list() - for group in self.__selected_peers: - result.extend(self.__selected_peers[group]) + for groupname in self.__selected_peers: + result.extend(self.__selected_peers[groupname]) return result def add_selected_peer(self, group_name, peer_ip): @@ -120,101 +168,25 @@ def remove_selected_peer(self, group_name, peer_ip): ) -def update_peergroup(cu, group_name, service_name, max_peers=None): - detected_peers = list() - try: - result = resolver.resolve( - f"{service_name}.{kube_namespace}.{service_root_domain}", "A" - ) - except (resolver.NXDOMAIN, resolver.NoAnswer) as err: - print(err.msg, f"- Skipping processing of {group_name}.") - return cu - - for r in result: - if r.address != pod_ip: - detected_peers.append(r.address) - - # Identify peers that should be active after this commit. - # Currently configured peers that are still detected via DNS are prioritized. - # Additional peers will be selected at random from detected_peers until `max_peers` are selected. - configured_peers = cu.get_selected_peers(group_name) - selected_peers = list() - for peer_ip in configured_peers: - if max_peers is None or len(selected_peers) < max_peers: - if peer_ip in detected_peers: - selected_peers.append(peer_ip) - detected_peers.remove(peer_ip) - else: - # The peer group is full with selected_peers. No need to continue this loop. - break - while len(detected_peers) and ( - max_peers is None or len(selected_peers) < max_peers - ): - selected_peers.append(detected_peers.pop(randrange(len(detected_peers)))) - - # Compare detected_peers (up-to-date list) with configured_peers. - # Add detected_peers not in configured_peers. - # Remove configured_peers not in detected_peers. - peers_to_add = set(selected_peers) - set(configured_peers) - for peer_ip in peers_to_add: - cu.add_selected_peer(group_name, peer_ip) - - peers_to_remove = set(configured_peers) - set(selected_peers) - for peer_ip in peers_to_remove: - cu.remove_selected_peer(group_name, peer_ip) - return cu - - if __name__ == "__main__": - # Load necessary environment variables - pod_ip = getenv("POD_IP") - if not pod_ip: - raise (Exception("POD_IP environment variable not set")) - mesh_service_name = getenv("MESH_SERVICE_NAME") - - upstream_service_name = getenv("UPSTREAM_SERVICE_NAME") - - if not mesh_service_name and not upstream_service_name: - raise ( - Exception( - "MESH_SERVICE_NAME and UPSTREAM_SERVICE_NAME environment variables not set" - ) - ) - - kube_namespace = getenv("KUBE_NAMESPACE", "default") - service_root_domain = getenv("SERVICE_ROOT_DOMAIN", "svc.cluster.local") + # Confirm that a mesh BGP group has been defined. + if not mconf.bgpgroups_mesh: + raise(Exception("No mesh BGP groups defined.")) # Open a connection to the device - dev = Device() + dev = Device(host="127.0.0.1",user="meshrr",ssh_private_key_file="/secret/ssh/id_ed25519") dev.open() with ConfigUpdate(dev, mode="private") as cu: - - groups = list() - if mesh_service_name: - groups.append( - { - "name": "MESHRR-MESH", - "service_name": mesh_service_name, - "max_peers": None, - } + for group in mconf.bgpgroups_mesh: + cu.update_bgpgroup_mesh( + bgpgroup=group ) - - if upstream_service_name: - groups.append( - { - "name": "MESHRR-UPSTREAM", - "service_name": upstream_service_name, - "max_peers": 2, - } - ) - - for group in groups: - cu = update_peergroup( - cu, - group_name=group["name"], - service_name=group["service_name"], - max_peers=group["max_peers"], + for group in mconf.bgpgroups_subtractive: + cu.update_bgpgroup_subtractive( + bgpgroup=group ) if cu.diff(): - cu.commit_peerupdate() + print(f"[{datetime.now()}] Peer change detected. Writing changes:") + cu.pdiff() + cu.commit()