wip: slowly working thru cilium

arichtman · Jan 3, 2025 · 9916fe8 · 9916fe8
1 parent 62840ed
commit 9916fe8
Show file tree

Hide file tree

Showing 13 changed files with 88 additions and 44 deletions.
diff --git a/README.md b/README.md
@@ -435,17 +435,11 @@ References:
 #### Trust chain setup
 
 Arguably this mingles with substratum, as PKI/trust/TLS is required or very desirable for VPN/HTTPS etc.
+SPIFFE/SPIRE will address this somewhat.
 
 1. Create root CA
    `xkcdpass --delimiter - --numwords 4 > root-ca.pass`
    `step certificate create "[email protected]" ./root-ca.pem ./root-ca-key.pem --profile root-ca --password-file ./root-ca.pass`
-1. Make node directories cause this is going to get messy
-   `<nodes.txt xargs mkdir`
-1. Create password files
-   `<nodes.txt xargs -I% sh -c 'xkcdpass --delimiter - --numwords 4 > "./$1/$1-pass.txt"' -- %`
-1. Secure them `chmod 400 *.pass`
-1. Create intermediate CAs
-   `<nodes.txt xargs -I% step certificate create % ./%/%.pem ./%/%-key.pem --profile intermediate-ca --ca ./root-ca.pem --ca-key ./root-ca-key.pem --ca-password-file root-ca-pass.txt --password-file ./%/%-pass.txt`
 1. Distribute the intermediate certificates and keys
 1. Secure the root CA, it's a bit hidden but Bitwarden _does_ take attachments.
 1. Publish the root CA, with my current setup this meant uploading it to s3.

diff --git a/certificates/admin-client.sh b/certificates/admin-client.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+step certificate create cluster-admin admin-client.pem admin-client-key.pem \
+  --ca k8s-ca.pem --ca-key k8s-ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json \
+  --not-after 8760h --set organization=system:masters \
+  $@
+
+kubectl config set-credentials home-admin --client-certificate=admin-client.pem --client-key=admin-client-key.pem --embed-certs
+kubectl config set-cluster home --server=https://fat-controller.systems.richtman.au:6443 --certificate-authority=k8s-ca.pem --embed-certs
diff --git a/certificates/control-node-certs.sh b/certificates/control-node-certs.sh
@@ -1,26 +1,33 @@
 #!/bin/bash
 # Parameters:
 #   1. Node's host name/DNS name
+#   2+. Additional arguments to step commands, mostly "--force"
 
 export NODE_DNS_NAME="${1}"
+shift
 
 # etcd TLS
 step certificate create etcd etcd-tls.pem etcd-tls-key.pem --ca etcd.pem --ca-key etcd-key.pem \
   --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json --not-after 8760h --bundle \
-  --san "${NODE_DNS_NAME}" --san "${NODE_DNS_NAME}.internal" --san "${NODE_DNS_NAME}.systems.richtman.au" --san localhost --san 127.0.0.1 --san ::1
+  --san "${NODE_DNS_NAME}" --san "${NODE_DNS_NAME}.internal" --san "${NODE_DNS_NAME}.systems.richtman.au" --san localhost \
+  --san 127.0.0.1 --san ::1 \
+  $@
 
 # apiserver client to etcd
 step certificate create kube-apiserver-etcd-client kube-apiserver-etcd-client.pem kube-apiserver-etcd-client-key.pem \
   --ca etcd.pem --ca-key etcd-key.pem --insecure --no-password --not-after 8120h \
-  --template granular-dn-leaf.tpl --set-file dn-defaults.json
+  --template granular-dn-leaf.tpl --set-file dn-defaults.json \
+  $@
 
 # apiserver TLS
 # Note that your local domain and private IP for in-cluster may vary
-step certificate create kube-apiserver kube-apiserver-tls.pem kube-apiserver-tls-key.pem --ca ca.pem --ca-key ca-key.pem \
+step certificate create kube-apiserver kube-apiserver-tls.pem kube-apiserver-tls-key.pem --ca k8s-ca.pem --ca-key k8s-ca-key.pem \
   --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json --not-after 8760h --bundle \
-  --san "${NODE_DNS_NAME}" --san "${NODE_DNS_NAME}.systems.richtman.au" --san "${NODE_DNS_NAME}.internal" --san localhost --san 127.0.0.1 --san ::1 --san 10.0.0.1 \
+  --san "${NODE_DNS_NAME}" --san "${NODE_DNS_NAME}.systems.richtman.au" --san "${NODE_DNS_NAME}.internal" --san localhost \
+  --san 127.0.0.1 --san ::1 --san 10.0.0.1 \
   --san kubernetes --san kubernetes.default --san kubernetes.default.svc \
-  --san kubernetes.default.svc.cluster --san kubernetes.default.svc.cluster.systems.richtman.au
+  --san kubernetes.default.svc.cluster --san kubernetes.default.svc.cluster.systems.richtman.au \
+  $@
 
 # service account token signing
 openssl req -new -x509 -days 365 -newkey rsa:4096 -keyout service-account-key.pem -sha256 \
@@ -29,36 +36,44 @@ openssl req -new -x509 -days 365 -newkey rsa:4096 -keyout service-account-key.pe
 
 # Controller manager apiserver client
 step certificate create system:kube-controller-manager controllermanager-apiserver-client.pem controllermanager-apiserver-client-key.pem \
-  --ca ca.pem --ca-key ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json \
-  --not-after 8760h --set organization=system:kube-controller-manager
+  --ca k8s-ca.pem --ca-key k8s-ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json \
+  --not-after 8760h --set organization=system:kube-controller-manager \
+  $@
 
 # Controller manager TLS
-step certificate create kube-controllermanager controllermanager-tls-cert-file.pem controllermanager-tls-private-key-file.pem --ca ca.pem --ca-key ca-key.pem \
+step certificate create kube-controllermanager controllermanager-tls-cert-file.pem controllermanager-tls-private-key-file.pem \
+  --ca k8s-ca.pem --ca-key k8s-ca-key.pem \
   --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json --not-after 8760h --bundle \
-  --san "${NODE_DNS_NAME}" --san "${NODE_DNS_NAME}.systems.richtman.au" --san "${NODE_DNS_NAME}.internal" --san localhost --san 127.0.0.1 --san ::1
+  --san "${NODE_DNS_NAME}" --san "${NODE_DNS_NAME}.systems.richtman.au" --san "${NODE_DNS_NAME}.internal" --san localhost \
+  --san 127.0.0.1 --san ::1 \
+  $@
 
 # Scheduler apiserver client
 step certificate create system:kube-scheduler scheduler-apiserver-client.pem scheduler-apiserver-client-key.pem \
-  --ca ca.pem --ca-key ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json \
-  --not-after 8760h
+  --ca k8s-ca.pem --ca-key k8s-ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json \
+  --not-after 8760h \
+  $@
 
 # Scheduler TLS
-step certificate create scheduler scheduler-tls-cert-file.pem scheduler-tls-private-key-file.pem --ca ca.pem --ca-key ca-key.pem \
+step certificate create scheduler scheduler-tls-cert-file.pem scheduler-tls-private-key-file.pem --ca k8s-ca.pem --ca-key k8s-ca-key.pem \
   --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json --not-after 8760h --bundle \
-  --san "${NODE_DNS_NAME}" --san "${NODE_DNS_NAME}.systems.richtman.au" --san "${NODE_DNS_NAME}.internal" --san localhost --san 127.0.0.1 --san ::1
+  --san "${NODE_DNS_NAME}" --san "${NODE_DNS_NAME}.systems.richtman.au" --san "${NODE_DNS_NAME}.internal" --san localhost \
+  --san 127.0.0.1 --san ::1 \
+  $@
 
 # APIserver client to kubelet
 step certificate create "system:node:${NODE_DNS_NAME}" kubelet-apiserver-client.pem kubelet-apiserver-client-key.pem \
-  --ca ca.pem --ca-key ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json \
-  --not-after 8760h --set organization=system:nodes
+  --ca k8s-ca.pem --ca-key k8s-ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json \
+  --not-after 8760h --set organization=system:nodes \
+  $@
 
 # Copy everything over, using ~ so we don't hit permissions issues
 rsync service-account*.pem "${NODE_DNS_NAME}.systems.richtman.au:/home/nixos/secrets"
 rsync scheduler*.pem "${NODE_DNS_NAME}.systems.richtman.au:/home/nixos/secrets"
 rsync etcd*.pem "${NODE_DNS_NAME}.systems.richtman.au:/home/nixos/secrets"
 rsync controller*.pem "${NODE_DNS_NAME}.systems.richtman.au:/home/nixos/secrets"
 rsync kube*.pem "${NODE_DNS_NAME}.systems.richtman.au:/home/nixos/secrets"
-rsync ca*.pem "${NODE_DNS_NAME}.systems.richtman.au:/home/nixos/secrets"
+rsync k8s-ca*.pem "${NODE_DNS_NAME}.systems.richtman.au:/home/nixos/secrets"
 
 # Remove any existing secrets so it's just this run
 ssh "${NODE_DNS_NAME}.systems.richtman.au" sudo rm -fr /var/lib/kubernetes/secrets

diff --git a/certificates/k8s-int-ca.sh b/certificates/k8s-int-ca.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+step certificate create k8s-ca k8s-ca.pem k8s-ca-key.pem \
+  --ca root-ca.pem --ca-key root-ca-key.pem --insecure --ca-password-file root-ca-pass.txt --no-password --not-after 8120h \
+  --template granular-dn-intermediate.tpl --set-file dn-defaults.json \
+  $@
diff --git a/certificates/worker-node-certs.sh b/certificates/worker-node-certs.sh
@@ -1,31 +1,36 @@
 #!/bin/bash
 # Parameters:
 #   1. Node's host name/DNS name
+#   2+. Additional arguments to step commands, mostly "--force"
 
 export NODE_DNS_NAME="${1}"
+shift
 
 # kubelet client to apiserver, referenced in kubeconfig file
 step certificate create kubelet-kubeconfig-client-certificate kubelet-kubeconfig-client-certificate.pem kubelet-kubeconfig-client-key.pem \
-  --ca ca.pem --ca-key ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json --not-after 8760h \
-  --set organization=system:masters
+  --ca k8s-ca.pem --ca-key k8s-ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json --not-after 8760h \
+  --set organization=system:masters \
+  $@
 
 # kubelet TLS
-step certificate create kubelet kubelet-tls-cert-file.pem kubelet-tls-private-key-file.pem --ca ca.pem --ca-key ca-key.pem \
+step certificate create kubelet kubelet-tls-cert-file.pem kubelet-tls-private-key-file.pem --ca k8s-ca.pem --ca-key k8s-ca-key.pem \
   --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json --not-after 8760h --bundle \
-  --san "${NODE_DNS_NAME}" --san "${NODE_DNS_NAME}.systems.richtman.au" --san "${NODE_DNS_NAME}.internal" --san localhost --san 127.0.0.1 --san ::1
+  --san "${NODE_DNS_NAME}" --san "${NODE_DNS_NAME}.systems.richtman.au" --san "${NODE_DNS_NAME}.internal" \
+  --san localhost --san 127.0.0.1 --san ::1 \
+  $@
 # # For client authentication to the proxy services
 # step certificate create kube-apiserver-proxy-client kube-apiserver-proxy-client.pem kube-apiserver-proxy-client-key.pem \
-#   --ca ca.pem --ca-key ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json \
+#   --ca k8s-ca.pem --ca-key k8s-ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json \
 #   --not-after 8760h
 # # Kube-proxy apiserver client
 # step certificate create system:kube-proxy proxy-apiserver-client.pem proxy-apiserver-client-key.pem \
-#   --ca ca.pem --ca-key ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json \
+#   --ca k8s-ca.pem --ca-key k8s-ca-key.pem --insecure --no-password --template granular-dn-leaf.tpl --set-file dn-defaults.json \
 #   --not-after 8760h --set organization=system:node-proxier
 
 # rsync proxy-*.pem "${NODE_DNS_NAME}.systems.richtman.au:/home/nixos/secrets"
 
 rsync kubelet*.pem "${NODE_DNS_NAME}.systems.richtman.au:/home/nixos/secrets"
-rsync ca.pem "${NODE_DNS_NAME}.systems.richtman.au:/home/nixos/secrets"
+rsync k8s-ca.pem "${NODE_DNS_NAME}.systems.richtman.au:/home/nixos/secrets"
 
 # Kubelet needs to run as root so the specific files that it accesses should be owned by it.
 ssh "${NODE_DNS_NAME}.systems.richtman.au" sudo rm -fr /var/lib/kubelet/secrets/

diff --git a/certificates/worker-nodes.sh b/certificates/worker-nodes.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+./worker-node-certs.sh fat-controller --force
+./worker-node-certs.sh mum --force
+./worker-node-certs.sh patient-zero --force
+./worker-node-certs.sh dr-singh --force
+./worker-node-certs.sh smol-bat --force
+./worker-node-certs.sh tweedledum --force
+./worker-node-certs.sh tweedledee --force
diff --git a/cilium.yaml b/cilium.yaml
@@ -1,3 +1,4 @@
+#helm upgrade --install cilium cilium/cilium --namespace kube-system --values cilium.yaml --version 1.16.5
 # No kube-proxy
 kubeProxyReplacement: true
 # Enables healthz endpoint
@@ -29,8 +30,8 @@ debug:
 #   which contains the gateway LL address
 # failed to start: daemon creation failed: error while initializing daemon: failed while reinitializing datapath: failed to enable direct routes for ipv6: route to destination fd00::118 contains gateway fe80::4140:f9e8:ec8a:489e, must be directly reachable. Add `direct-routing-skip-unreachable` to skip unreachable routes" subsys=daemon
 # autoDirectNodeRoutes: true
-# routingMode: native
-# ipv6NativeRoutingCIDR: 2403:580a:e4b1::/64
+routingMode: native
+ipv6NativeRoutingCIDR: 2403:580a:e4b1::/64
 # ipam:
 #   operator:
 #     clusterPoolIPv6PodCIDRList:

diff --git a/modules/nixos/k8s/apiserver.nix b/modules/nixos/k8s/apiserver.nix
@@ -27,7 +27,7 @@
     "--cert-dir"
     "${cfg.secretsPath}"
     "--client-ca-file"
-    "${cfg.secretsPath}/ca.pem"
+    "${cfg.secretsPath}/k8s-ca.pem"
     "--etcd-cafile"
     "${cfg.secretsPath}/etcd.pem"
     "--etcd-certfile"
@@ -46,7 +46,7 @@
     "certificates.k8s.io/v1alpha1/clustertrustbundles=true"
     # TODO: deduplicate/couple this
     "--kubelet-certificate-authority"
-    "${cfg.secretsPath}/ca.pem"
+    "${cfg.secretsPath}/k8s-ca.pem"
     "--kubelet-client-certificate"
     "${cfg.secretsPath}/kubelet-apiserver-client.pem"
     "--kubelet-client-key"

diff --git a/modules/nixos/k8s/controller.nix b/modules/nixos/k8s/controller.nix
@@ -22,7 +22,7 @@
       {
         name = "default";
         cluster = {
-          certificate-authority = "${topConfig.secretsPath}/ca.pem";
+          certificate-authority = "${topConfig.secretsPath}/k8s-ca.pem";
           server = "https://fat-controller.systems.richtman.au:6443";
         };
       }
@@ -63,6 +63,7 @@
     "--node-cidr-mask-size"
     "120"
     # "--cluster-cidr"
+    # "2403:580a:e4b1::/64"
     # "2001:db8:1234:5678:8:2::/104"
     "--authorization-kubeconfig"
     controllerKubeconfigFile
@@ -73,15 +74,15 @@
     # "--config"
     # controllerConfigFile
     "--client-ca-file"
-    "${topConfig.secretsPath}/ca.pem"
+    "${topConfig.secretsPath}/k8s-ca.pem"
     "--cluster-signing-cert-file"
-    "${topConfig.secretsPath}/ca.pem"
+    "${topConfig.secretsPath}/k8s-ca.pem"
     "--cluster-signing-key-file"
-    "${topConfig.secretsPath}/ca-key.pem"
+    "${topConfig.secretsPath}/k8s-ca-key.pem"
     "--kubeconfig"
     controllerKubeconfigFile
     "--root-ca-file"
-    "${topConfig.secretsPath}/ca.pem"
+    "${topConfig.secretsPath}/k8s-ca.pem"
     "--service-account-private-key-file"
     "${topConfig.secretsPath}/service-account-key.pem"
     "--tls-cert-file"

diff --git a/modules/nixos/k8s/kubelet.nix b/modules/nixos/k8s/kubelet.nix
@@ -28,7 +28,7 @@
       rotateCertificates = false;
       authentication = {
         x509 = {
-          clientCAFile = "${kubeletSecretsPath}/ca.pem";
+          clientCAFile = "${kubeletSecretsPath}/k8s-ca.pem";
         };
         webhook = {
           enable = true;
@@ -69,7 +69,7 @@
       {
         name = "default";
         cluster = {
-          certificate-authority = "${kubeletSecretsPath}/ca.pem";
+          certificate-authority = "${kubeletSecretsPath}/k8s-ca.pem";
           server = "https://fat-controller.systems.richtman.au:6443";
         };
       }

diff --git a/modules/nixos/k8s/scheduler.nix b/modules/nixos/k8s/scheduler.nix
@@ -22,7 +22,7 @@
       {
         name = "default";
         cluster = {
-          certificate-authority = "${topConfig.secretsPath}/ca.pem";
+          certificate-authority = "${topConfig.secretsPath}/k8s-ca.pem";
           server = "https://fat-controller.systems.richtman.au:6443";
         };
       }
@@ -58,7 +58,7 @@
     "--config"
     schedulerConfigFile
     "--client-ca-file"
-    "${topConfig.secretsPath}/ca.pem"
+    "${topConfig.secretsPath}/k8s-ca.pem"
     "--tls-cert-file"
     "${topConfig.secretsPath}/scheduler-tls-cert-file.pem"
     "--tls-private-key-file"

diff --git a/modules/nixos/lab-node/default.nix b/modules/nixos/lab-node/default.nix
@@ -173,6 +173,9 @@ in
           "ip saddr { 192.168.1.0/24,192.168.2.0/24 } tcp dport 443 accept"
           # Allow anything in my primary prefix into HTTPS
           "ip6 saddr { 2403:580a:e4b1::/48 } tcp dport 443 accept"
+          # Something for Cilium
+          "ip6 saddr { 2403:580a:e4b1::/48 } tcp dport 4240 accept"
+          # mDNS
           "ip saddr { 192.168.1.0/24 } udp dport 5353 accept"
           "ip6 saddr { 2403:580a:e4b1::/48 } udp dport 5353 accept"
         ];

diff --git a/test.yaml b/test.yaml
@@ -4,6 +4,7 @@ metadata:
   labels:
     run: test-loopback
   name: test-loopback
+  # namespace: default
 spec:
   containers:
   #   args:
@@ -31,7 +32,7 @@ spec:
     #   readOnly: true
   dnsPolicy: Default
   restartPolicy: Always
-  nodeName: mum.local
+  nodeName: mum.systems.richtman.au
   # automountServiceAccountToken: false
   # volumes:
   # - name: manual-token