Add multi-region example

Signed-off-by: Byron Ruth <[email protected]>

examples/topologies/multi-region/cli/Dockerfile

@@ -0,0 +1,23 @@
+FROM golang:1.19-alpine3.17 AS build
+
+RUN apk update && apk add git
+
+RUN go install github.com/nats-io/nats-server/[email protected]
+RUN go install github.com/nats-io/natscli/nats@main
+RUN go install github.com/nats-io/nsc/[email protected]
+
+FROM alpine:3.17
+
+RUN apk add bash curl
+
+COPY --from=build /go/bin/nats-server /usr/local/bin/
+COPY --from=build /go/bin/nats /usr/local/bin/
+COPY --from=build /go/bin/nsc /usr/local/bin/
+
+WORKDIR /app
+
+COPY . .
+
+ENTRYPOINT ["bash"]
+
+CMD ["main.sh"]

examples/topologies/multi-region/cli/GLOBAL.json

@@ -0,0 +1,23 @@
+{
+  "name": "GLOBAL",
+  "subjects": [
+    "js.in.global.>"
+  ],
+  "retention": "limits",
+  "max_consumers": -1,
+  "max_msgs_per_subject": -1,
+  "max_msgs": -1,
+  "max_bytes": -1,
+  "max_age": 0,
+  "max_msg_size": -1,
+  "storage": "file",
+  "discard": "old",
+  "num_replicas": 3,
+  "duplicate_window": 120000000000,
+  "sealed": false,
+  "deny_delete": false,
+  "deny_purge": false,
+  "allow_rollup_hdrs": false,
+  "allow_direct": false,
+  "mirror_direct": false
+}

examples/topologies/multi-region/cli/ORDERS_CENTRAL.json

@@ -0,0 +1,39 @@
+{
+  "name": "ORDERS_CENTRAL",
+  "subjects": [
+    "js.in.orders_central"
+  ],
+  "retention": "limits",
+  "max_consumers": -1,
+  "max_msgs_per_subject": -1,
+  "max_msgs": -1,
+  "max_bytes": -1,
+  "max_age": 0,
+  "max_msg_size": -1,
+  "storage": "file",
+  "discard": "old",
+  "num_replicas": 3,
+  "duplicate_window": 120000000000,
+  "placement": {
+    "cluster": "",
+    "tags": [
+      "region:central"
+    ]
+  },
+  "sources": [
+    {
+      "name": "ORDERS_EAST",
+      "filter_subject": "js.in.orders_east"
+    },
+    {
+      "name": "ORDERS_WEST",
+      "filter_subject": "js.in.orders_west"
+    }
+  ],
+  "sealed": false,
+  "deny_delete": false,
+  "deny_purge": false,
+  "allow_rollup_hdrs": false,
+  "allow_direct": false,
+  "mirror_direct": false
+}

examples/topologies/multi-region/cli/ORDERS_EAST.json

@@ -0,0 +1,39 @@
+{
+  "name": "ORDERS_EAST",
+  "subjects": [
+    "js.in.orders_east"
+  ],
+  "retention": "limits",
+  "max_consumers": -1,
+  "max_msgs_per_subject": -1,
+  "max_msgs": -1,
+  "max_bytes": -1,
+  "max_age": 0,
+  "max_msg_size": -1,
+  "storage": "file",
+  "discard": "old",
+  "num_replicas": 3,
+  "duplicate_window": 120000000000,
+  "placement": {
+    "cluster": "",
+    "tags": [
+      "region:east"
+    ]
+  },
+  "sources": [
+    {
+      "name": "ORDERS_WEST",
+      "filter_subject": "js.in.orders_west"
+    },
+    {
+      "name": "ORDERS_CENTRAL",
+      "filter_subject": "js.in.orders_central"
+    }
+  ],
+  "sealed": false,
+  "deny_delete": false,
+  "deny_purge": false,
+  "allow_rollup_hdrs": false,
+  "allow_direct": false,
+  "mirror_direct": false
+}

examples/topologies/multi-region/cli/ORDERS_WEST.json

@@ -0,0 +1,39 @@
+{
+  "name": "ORDERS_WEST",
+  "subjects": [
+    "js.in.orders_west"
+  ],
+  "retention": "limits",
+  "max_consumers": -1,
+  "max_msgs_per_subject": -1,
+  "max_msgs": -1,
+  "max_bytes": -1,
+  "max_age": 0,
+  "max_msg_size": -1,
+  "storage": "file",
+  "discard": "old",
+  "num_replicas": 3,
+  "duplicate_window": 120000000000,
+  "placement": {
+    "cluster": "",
+    "tags": [
+      "region:west"
+    ]
+  },
+  "sources": [
+    {
+      "name": "ORDERS_EAST",
+      "filter_subject": "js.in.orders_east"
+    },
+    {
+      "name": "ORDERS_CENTRAL",
+      "filter_subject": "js.in.orders_central"
+    }
+  ],
+  "sealed": false,
+  "deny_delete": false,
+  "deny_purge": false,
+  "allow_rollup_hdrs": false,
+  "allow_direct": false,
+  "mirror_direct": false
+}

examples/topologies/multi-region/cli/README.md

@@ -0,0 +1,107 @@
+# Multi Region NATS Cluster
+
+This demonstrates a very basic NATS Cluster using only routes across 3 regions.
+
+This should be used in cases where a multi region setup is needed and one requires a stream to be globally deployed and have globally consistent message deduplication.
+
+## Constraints
+
+ * This should be deployed in networks with generally below 100ms latency. For example in GCP using Tier-1 network connectivity in US east, west and central.
+ * Streams should generally be R3 when stretched out of a single region to mitigate the big exposure they would have to latency if R5
+ * Multi-region streams should not be the default, ideally these are only used for those cases where global deduplication is needed
+ * For general streams where eventual consistency is acceptible streams should be bound to a region and replicated into others if desired
+
+## Configuration guidelines
+
+### Server Tags
+
+Each server is tagged with a regional tag, for example, `region:east` or `region:west`. The servers are configured with anti-affinity on this tag using the following configuration:
+
+```
+jetstream {
+  unique_tag: "region:"
+}
+```
+
+This ensures that streams that are not specifically bound to a single region will be split across the 3 regions.
+
+### Configuring single region streams
+
+```
+$ nats stream add --tag region:east --replicas 3 EAST
+...
+Cluster Information:
+
+                 Name: c1
+               Leader: n1-east
+              Replica: n2-east, current, seen 1ms ago
+              Replica: n3-east, current, seen 0s ago
+```
+
+Note the servers are `n1-east`, `n2-east`, `n3-east`.
+
+### Configuring a global stream
+
+```
+$ nats stream add --replicas 3 GLOBAL
+...
+Cluster Information:
+
+                 Name: c1
+               Leader: n1-central
+              Replica: n2-east, current, seen 0s ago
+              Replica: n3-west, current, seen 0s ago
+
+```
+
+Note here we give no placement directive so the server will spread it across regions due to the `unique_tag` setting.  Servers are `n1-central`, `n2-east` and `n3-west` - 1 per region.
+
+### Configuring replicated streams
+
+To facilitate an eventually consistent but single config set up we show how to create 3 streams:
+
+ * `ORDERS_EAST` listening on subjects `js.in.orders_east`
+ * `ORDERS_WEST` listening on subjects `js.in.orders_west`
+ * `ORDERS_CENTRAL` listening on subjects `js.in.orders.central`
+
+We then use server mappings in each region to map `js.in.orders` to the in-region subject:
+
+```
+accounts {
+  one: {
+    jetstream: enabled
+    mappings: {
+      js.in.orders: js.in.orders_central
+    }
+  }
+}
+```
+
+We can now publish to one subject and depend on which region we connect to the local stream will handle - then replicate - the data:
+
+```
+$ nats --context contexts/central-user.json req js.in.orders 1
+{"stream":"ORDERS_CENTRAL", "seq":4}
+
+$ nats --context contexts/east-user.json req js.in.orders 1
+{"stream":"ORDERS_EAST", "seq":5}
+```
+
+After a short while all streams hold the same data.
+
+```
+╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+│                                                             Stream Report                                                             │
+├────────────────┬─────────┬──────────────────────┬───────────┬──────────┬───────┬──────┬─────────┬─────────────────────────────────────┤
+│ Stream         │ Storage │ Placement            │ Consumers │ Messages │ Bytes │ Lost │ Deleted │ Replicas                            │
+├────────────────┼─────────┼──────────────────────┼───────────┼──────────┼───────┼──────┼─────────┼─────────────────────────────────────┤
+│ GLOBAL         │ File    │                      │         0 │ 0        │ 0 B   │ 0    │       0 │ n1-central, n2-west*, n3-east       │
+│ ORDERS_CENTRAL │ File    │ tags: region:central │         0 │ 5        │ 399 B │ 0    │       0 │ n1-central, n2-central*, n3-central │
+│ ORDERS_EAST    │ File    │ tags: region:east    │         0 │ 5        │ 405 B │ 0    │       0 │ n1-east*, n2-east, n3-east          │
+│ ORDERS_WEST    │ File    │ tags: region:west    │         0 │ 5        │ 456 B │ 0    │       0 │ n1-west*, n2-west, n3-west          │
+╰────────────────┴─────────┴──────────────────────┴───────────┴──────────┴───────┴──────┴─────────┴─────────────────────────────────────╯
+```
+
+This allows portable publishers to be built, consumers will need to know the region they are in and bind to the correct stream.
+
+

examples/topologies/multi-region/cli/configs/cluster-central.conf

@@ -0,0 +1,56 @@
+port: 4222
+monitor_port: 8222
+server_name: $NAME
+client_advertise: $ADVERTISE
+
+server_tags: [$GATEWAY, $REGION]
+
+cluster {
+  port: 6222
+
+  routes = [
+    nats-route://n1-east:6222
+    nats-route://n2-east:6222
+    nats-route://n3-east:6222
+    nats-route://n1-west:6222
+    nats-route://n2-west:6222
+    nats-route://n3-west:6222
+    nats-route://n1-central:6222
+    nats-route://n2-central:6222
+    nats-route://n3-central:6222
+  ]
+}
+
+leafnodes {
+  port: 7422
+}
+
+gateway {
+  name: $GATEWAY
+  port: 7222
+}
+
+jetstream {
+  store_dir: /data
+  unique_tag: "region:"
+}
+
+accounts {
+  one: {
+    jetstream: enabled
+    users = [
+      {user: one, password: secret}
+    ]
+    mappings: {
+      js.in.orders: js.in.orders_central
+    }
+  }
+
+  system: {
+    users = [
+      {user: system, password: secret}
+    ]
+  }
+}
+
+system_account: system

examples/topologies/multi-region/cli/configs/cluster-east.conf

@@ -0,0 +1,57 @@
+port: 4222
+monitor_port: 8222
+server_name: $NAME
+client_advertise: $ADVERTISE
+
+server_tags: [$GATEWAY, $REGION]
+
+cluster {
+  port: 6222
+
+  routes = [
+    nats-route://n1-east:6222
+    nats-route://n2-east:6222
+    nats-route://n3-east:6222
+    nats-route://n1-west:6222
+    nats-route://n2-west:6222
+    nats-route://n3-west:6222
+    nats-route://n1-central:6222
+    nats-route://n2-central:6222
+    nats-route://n3-central:6222
+  ]
+}
+
+leafnodes {
+  port: 7422
+}
+
+gateway {
+  name: $GATEWAY
+  port: 7222
+}
+
+jetstream {
+  store_dir: /data
+  unique_tag: "region:"
+}
+
+accounts {
+  one: {
+    jetstream: enabled
+    users = [
+      {user: one, password: secret}
+    ]
+
+    mappings: {
+      js.in.orders: js.in.orders_east
+    }
+  }
+
+  system: {
+    users = [
+      {user: system, password: secret}
+    ]
+  }
+}
+
+system_account: system