Everything from the earlier stories at once, across three NATS clusters in three Kubernetes clusters: a five-server one in the home cluster, a development NATS cluster pinned to one zone, and a production NATS cluster in another region whose placement tag is not its name. Four services run in two environments, each service an account.

Trust roots and NATS clusters #

The trust roots and the gateway list are the same in every Kubernetes cluster; GitOps keeps them alike. The gateway certificates come from the Issuer nats-gateway-ca in nats-system of each Kubernetes cluster, a private CA, as in the supercluster story.

01-natsoperatortrust.yaml

apiVersion: nats.mikluko.io/v1beta1
kind: NatsOperatorTrust
metadata:
  name: acme
  namespace: nats-system
spec:
  operatorJWT: eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ...
  systemAccountJWT: eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ...

kubectl --context prod-east apply -f https://nats-operator.io/docs/stories/09-acceptance/01-natsoperatortrust.yaml
kubectl --context dev-east apply -f https://nats-operator.io/docs/stories/09-acceptance/01-natsoperatortrust.yaml
kubectl --context prod-west apply -f https://nats-operator.io/docs/stories/09-acceptance/01-natsoperatortrust.yaml

01-prod-east.yaml

# One NatsCluster per Kubernetes cluster, each in its own GitOps tree.
# Only what differs between them is commented.
apiVersion: cluster.nats.mikluko.io/v1beta1
kind: NatsCluster
metadata:
  name: prod-east   # in the home cluster
  namespace: nats-system
spec:
  version: 2.15.0
  replicas: 5
  resources:
    requests:
      cpu: "6"
      memory: 29Gi
    limits:
      memory: 29Gi
  jetstream:
    # Overrides the memory store derived from limits.memory.
    limits:
      maxMemoryStore: 10Gi
    volumeClaimTemplate:
      spec:
        storageClassName: gp3
        resources:
          requests:
            storage: 500Gi
  serverTags:
    cluster: prod-east
    region: us-east-2
  auth:
    trustRef:
      name: acme
    systemCredentials:
      secretKeyRef:
        name: cluster-controller-creds
  gateway:
    discovery: Explicit
    remotes:
      - name: prod-east
        url: tls://nats.prod-east.acme.example:7222
      - name: dev-east
        url: tls://nats.dev-east.acme.example:7222
      - name: prod-west
        url: tls://nats.prod-west.acme.example:7222
    tls:
      certManager:
        issuerRef:
          kind: Issuer
          name: nats-gateway-ca
    service:
      type: LoadBalancer
      annotations:
        service.beta.kubernetes.io/aws-load-balancer-type: external
        service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
        external-dns.alpha.kubernetes.io/hostname: nats.prod-east.acme.example
    advertise: nats.prod-east.acme.example:7222

kubectl --context prod-east apply -f https://nats-operator.io/docs/stories/09-acceptance/01-prod-east.yaml

01-dev-east.yaml

apiVersion: cluster.nats.mikluko.io/v1beta1
kind: NatsCluster
metadata:
  name: dev-east
  namespace: nats-system
spec:
  version: 2.15.0
  replicas: 3
  resources:
    requests:
      cpu: "1500m"
      memory: 3Gi
    limits:
      memory: 3Gi
  jetstream:
    volumeClaimTemplate:
      spec:
        storageClassName: gp3
        resources:
          requests:
            storage: 100Gi
  serverTags:
    cluster: dev-east
    region: us-east-2
  # Scheduling passes through: dev pins every server to one zone.
  podTemplate:
    spec:
      nodeSelector:
        topology.kubernetes.io/zone: us-east-2a
  auth:
    trustRef:
      name: acme
    systemCredentials:
      secretKeyRef:
        name: dev-east-cluster-controller-creds
  gateway:
    discovery: Explicit
    remotes:
      - name: prod-east
        url: tls://nats.prod-east.acme.example:7222
      - name: dev-east
        url: tls://nats.dev-east.acme.example:7222
      - name: prod-west
        url: tls://nats.prod-west.acme.example:7222
    tls:
      certManager:
        issuerRef:
          kind: Issuer
          name: nats-gateway-ca
    service:
      type: LoadBalancer
      annotations:
        service.beta.kubernetes.io/aws-load-balancer-type: external
        service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
        external-dns.alpha.kubernetes.io/hostname: nats.dev-east.acme.example
    advertise: nats.dev-east.acme.example:7222

kubectl --context dev-east apply -f https://nats-operator.io/docs/stories/09-acceptance/01-dev-east.yaml

01-prod-west.yaml

apiVersion: cluster.nats.mikluko.io/v1beta1
kind: NatsCluster
metadata:
  name: prod-west
  namespace: nats-system
spec:
  version: 2.15.0
  replicas: 3
  resources:
    requests:
      cpu: "1500m"
      memory: 3Gi
    limits:
      memory: 3Gi
  jetstream:
    volumeClaimTemplate:
      spec:
        storageClassName: gp3
        resources:
          requests:
            storage: 100Gi
  # Tags are free: this NATS cluster's placement tag is not its name.
  serverTags:
    cluster: west
    region: us-west-2
  auth:
    trustRef:
      name: acme
    systemCredentials:
      secretKeyRef:
        name: prod-west-cluster-controller-creds
  gateway:
    discovery: Explicit
    remotes:
      - name: prod-east
        url: tls://nats.prod-east.acme.example:7222
      - name: dev-east
        url: tls://nats.dev-east.acme.example:7222
      - name: prod-west
        url: tls://nats.prod-west.acme.example:7222
    tls:
      certManager:
        issuerRef:
          kind: Issuer
          name: nats-gateway-ca
    service:
      type: LoadBalancer
      annotations:
        service.beta.kubernetes.io/aws-load-balancer-type: external
        service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
        external-dns.alpha.kubernetes.io/hostname: nats.prod-west.acme.example
    advertise: nats.prod-west.acme.example:7222

kubectl --context prod-west apply -f https://nats-operator.io/docs/stories/09-acceptance/01-prod-west.yaml

Each NatsCluster reports every other member’s gateways connected.

01-status-natscluster-prod-west.yaml

status:
  observedGeneration: 1
  conditions:
    - type: Ready
      status: "True"
      reason: AllServersReady
    - type: Settled
      status: "True"
      reason: AllGroupsCurrent
    - type: GatewaysConnected
      status: "True"
      reason: AllMembersReachable
      message: 2 of 2 remote members connected
  gateways:
    - name: dev-east
      connected: true
      inbound: 3
      outbound: 3
    - name: prod-east
      connected: true
      inbound: 5
      outbound: 3
  endpoints:
    client: nats://prod-west.nats-system.svc:4222
    gateway: nats.prod-west.acme.example:7222

The auth plane #

The NATS operator, the system account, and the production account chain: checks exports a service to monitoring, which exports streams and services to core and to the collector. The development chain repeats it under -dev names and is left out.

01-auth.yaml

# All of it in the home cluster, prod-east.
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsOperator
metadata:
  name: acme
  namespace: nats-system
spec:
  systemAccountRef:
    name: sys
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsSystemAccount
metadata:
  name: sys
  namespace: nats-system
spec:
  operatorRef:
    name: acme
---
# The production chain of four services; the development chain is left out.
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsAccount
metadata:
  name: checks-prod
  namespace: nats-system
spec:
  operatorRef:
    name: acme
  exports:
    - name: run
      type: Service
      subject: "checks.run.>"
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsAccount
metadata:
  name: monitoring-prod
  namespace: nats-system
spec:
  operatorRef:
    name: acme
  limits:
    jetstream:
      memoryStorage: 36Gi
      diskStorage: 300Gi
  exports:
    - name: executions
      type: Stream
      subject: "monitoring.execution.>"
    - name: state-changes
      type: Stream
      subject: "monitoring.state.>"
    - name: execute
      type: Service
      subject: "monitoring.execute"
    - name: configure
      type: Service
      subject: "monitoring.configure"
  imports:
    - accountRef:
        kind: NatsAccount
        name: checks-prod
      export: run
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsAccount
metadata:
  name: core-prod
  namespace: nats-system
spec:
  operatorRef:
    name: acme
  imports:
    - accountRef:
        kind: NatsAccount
        name: monitoring-prod
      export: state-changes
    - accountRef:
        kind: NatsAccount
        name: monitoring-prod
      export: configure
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsAccount
metadata:
  name: collector-prod
  namespace: nats-system
spec:
  operatorRef:
    name: acme
  limits:
    jetstream:
      diskStorage: 50Gi
  imports:
    - accountRef:
        kind: NatsAccount
        name: monitoring-prod
      export: executions

kubectl --context prod-east apply -f https://nats-operator.io/docs/stories/09-acceptance/01-auth.yaml

Every account reaches every server of the supercluster through the resolver.

01-status-natsaccount-monitoring-prod.yaml

status:
  observedGeneration: 1
  conditions:
    - type: Ready
      status: "True"
      reason: Distributed
    - type: Distributed
      status: "True"
      reason: AllServersCurrent
      message: 11 of 11 servers hold this JWT
  distribution:
    servers: 11
    current: 11

Every account carries a service and a readonly user, and every other Kubernetes cluster two controller users.

01-users.yaml

# Every account carries the same two users, `service` and `readonly`. The
# monitoring team's live in its own namespace, admitted by the grant below.
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsUser
metadata:
  name: monitoring-prod-service
  namespace: monitoring
spec:
  accountRef:
    kind: NatsAccount
    namespace: nats-system
    name: monitoring-prod
  permissions:
    publish:
      allow: [">"]
    subscribe:
      allow: [">"]
  credentials:
    secretKeyRef:
      name: nats-service-creds
---
# The readonly preset: publish only to the JetStream introspection
# subjects, and subscribe only to replies under `_INBOX.readonly`.
# `permissions` is refused beside a preset.
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsUser
metadata:
  name: monitoring-prod-readonly
  namespace: monitoring
spec:
  accountRef:
    kind: NatsAccount
    namespace: nats-system
    name: monitoring-prod
  preset: readonly
  credentials:
    secretKeyRef:
      name: nats-readonly-creds
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsUser
metadata:
  name: checks-prod-service
  namespace: nats-system
spec:
  accountRef:
    kind: NatsAccount
    name: checks-prod
  permissions:
    publish:
      allow: [">"]
    subscribe:
      allow: [">"]
  credentials:
    secretKeyRef:
      name: checks-prod-service-creds
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsUser
metadata:
  name: checks-prod-readonly
  namespace: nats-system
spec:
  accountRef:
    kind: NatsAccount
    name: checks-prod
  preset: readonly
  credentials:
    secretKeyRef:
      name: checks-prod-readonly-creds
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsUser
metadata:
  name: core-prod-service
  namespace: nats-system
spec:
  accountRef:
    kind: NatsAccount
    name: core-prod
  permissions:
    publish:
      allow: [">"]
    subscribe:
      allow: [">"]
  credentials:
    secretKeyRef:
      name: core-prod-service-creds
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsUser
metadata:
  name: core-prod-readonly
  namespace: nats-system
spec:
  accountRef:
    kind: NatsAccount
    name: core-prod
  preset: readonly
  credentials:
    secretKeyRef:
      name: core-prod-readonly-creds
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsUser
metadata:
  name: collector-prod-service
  namespace: nats-system
spec:
  accountRef:
    kind: NatsAccount
    name: collector-prod
  permissions:
    publish:
      allow: [">"]
    subscribe:
      allow: [">"]
  credentials:
    secretKeyRef:
      name: collector-prod-service-creds
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsUser
metadata:
  name: collector-prod-readonly
  namespace: nats-system
spec:
  accountRef:
    kind: NatsAccount
    name: collector-prod
  preset: readonly
  credentials:
    secretKeyRef:
      name: collector-prod-readonly-creds
---
# The controllers of the remote Kubernetes clusters, two per remote (story 6).
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsUser
metadata:
  name: dev-east-cluster-controller
  namespace: nats-system
spec:
  accountRef:
    kind: NatsSystemAccount
    name: sys
  preset: cluster-controller
  credentials:
    secretKeyRef:
      name: dev-east-cluster-controller-creds
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsUser
metadata:
  name: dev-east-jetstream-controller
  namespace: nats-system
spec:
  accountRef:
    kind: NatsSystemAccount
    name: sys
  preset: jetstream-controller
  credentials:
    secretKeyRef:
      name: dev-east-jetstream-controller-creds
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsUser
metadata:
  name: prod-west-cluster-controller
  namespace: nats-system
spec:
  accountRef:
    kind: NatsSystemAccount
    name: sys
  preset: cluster-controller
  credentials:
    secretKeyRef:
      name: prod-west-cluster-controller-creds
---
apiVersion: auth.nats.mikluko.io/v1beta1
kind: NatsUser
metadata:
  name: prod-west-jetstream-controller
  namespace: nats-system
spec:
  accountRef:
    kind: NatsSystemAccount
    name: sys
  preset: jetstream-controller
  credentials:
    secretKeyRef:
      name: prod-west-jetstream-controller-creds
---
# What lets the monitoring namespace's users reach the accounts in
# nats-system.
apiVersion: nats.mikluko.io/v1beta1
kind: NatsReferenceGrant
metadata:
  name: monitoring-users
  namespace: nats-system
spec:
  from:
    - group: auth.nats.mikluko.io
      kind: NatsUser
      namespace: monitoring
  to:
    - group: auth.nats.mikluko.io
      kind: NatsAccount
      name: monitoring-prod
    - group: auth.nats.mikluko.io
      kind: NatsAccount
      name: monitoring-dev

kubectl --context prod-east apply -f https://nats-operator.io/docs/stories/09-acceptance/01-users.yaml

JetStream and balancing #

The monitoring team adopts the streams its application created, pools them, and balances within its account; the platform team balances each NATS cluster.

02-jetstream.yaml

# The monitoring team's JetStream, in prod-east. Its streams were created by
# the application at runtime before any of this existed, so they are adopted.
apiVersion: nats.mikluko.io/v1beta1
kind: NatsConnection
metadata:
  name: prod
  namespace: monitoring
spec:
  servers: ["nats://prod-east.nats-system.svc:4222"]
  credentials:
    secretKeyRef:
      name: nats-service-creds
---
apiVersion: jetstream.nats.mikluko.io/v1beta1
kind: NatsStream
metadata:
  name: requests
  namespace: monitoring
  labels:
    traffic: requests
spec:
  connectionRef:
    name: prod
  adoptionPolicy: Adopt
  name: REQUESTS
---
apiVersion: jetstream.nats.mikluko.io/v1beta1
kind: NatsStream
metadata:
  name: responses
  namespace: monitoring
  labels:
    traffic: responses
spec:
  connectionRef:
    name: prod
  adoptionPolicy: Adopt
  name: RESPONSES
---
apiVersion: jetstream.nats.mikluko.io/v1beta1
kind: NatsBalancer
metadata:
  name: prod
  namespace: monitoring
spec:
  connectionRef:
    name: prod
  pools:
    - name: requests
      selector:
        matchLabels:
          traffic: requests
    - name: responses
      selector:
        matchLabels:
          traffic: responses
---
# The platform team's, one per NATS cluster; prod-east shown.
apiVersion: nats.mikluko.io/v1beta1
kind: NatsConnection
metadata:
  name: prod-east-sys
  namespace: nats-system
spec:
  servers: ["nats://prod-east.nats-system.svc:4222"]
  credentials:
    secretKeyRef:
      name: jetstream-controller-creds
---
apiVersion: jetstream.nats.mikluko.io/v1beta1
kind: NatsSystemBalancer
metadata:
  name: prod-east
  namespace: nats-system
spec:
  connectionRef:
    name: prod-east-sys
  moves:
    placement: true
    leader: true

kubectl --context prod-east apply -f https://nats-operator.io/docs/stories/09-acceptance/02-jetstream.yaml

02-status-natsstream-requests.yaml

status:
  observedGeneration: 2
  conditions:
    - type: Ready
      status: "True"
      reason: Synced
    - type: Adopted
      status: "True"
      reason: FoundUnowned
      message: adopted existing stream REQUESTS; spec written from the server
    - type: Synced
      status: "True"
      reason: MatchesSpec
  ownership:
    origin: Adopted
  server:
    leader: prod-east-2

02-status-natsbalancer.yaml

status:
  observedGeneration: 1
  conditions:
    - type: Ready
      status: "True"
      reason: Balancing
    - type: Holding
      status: "False"
      reason: Settled
    - type: Overlapping
      status: "False"
      reason: PoolsDisjoint
  # One stream a pool: skewed by one whichever server leads it. Every stream
  # of the account has a resource in a pool, so there is no (default) pool.
  pools:
    - name: requests
      streams: 1
      leaderSkew: 1
    - name: responses
      streams: 1
      leaderSkew: 1

02-status-natssystembalancer.yaml

status:
  observedGeneration: 1
  conditions:
    - type: Ready
      status: "True"
      reason: Balancing
    - type: Holding
      status: "False"
      reason: Settled
  # monitoring-prod, the one account with streams in prod-east, carries no
  # jetstream-stepdown export: no leader is moved.
  capabilities:
    placement: true
    leader: None
    leaderReason: 1 of 1 accounts carry no jetstream-stepdown export; their leaders are not moved