Configure host to use SSH certs on the host and client side.

host_config/bootstrap_host.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
+host_ca_key="${SCRIPT_DIR}/ssh_certs/host_ca"
+user_ca_pub="${SCRIPT_DIR}/ssh_certs/user_ca.pub"
+
+if [ ! -f "${host_ca_key}" ]
+then
+  echo "Host CA key not found."
+  exit 1
+fi
+
+if [ ! -f "${user_ca_pub}" ]
+then
+  echo "Public User CA key not found."
+  exit 1
+fi
+
+temp=$(mktemp -d)
+
+cleanup() {
+  rm -rf "${temp}"
+}
+# trap cleanup EXIT
+
+host_type=$1
+hostname=$2
+extra_names=${3:-}
+
+principal_names="${hostname}"
+
+if [ ! -z "${extra_names}" ]
+then
+    principal_names="${principal_names},${extra_names}"
+fi
+
+install -d -m755 "${temp}/persisted/etc/ssh"
+ssh-keygen -t ed25519 -f "${temp}/persisted/etc/ssh/ssh_host_ed25519_key" -C '' -N ''
+ssh-keygen -s ${host_ca_key} -I ${hostname} -h -n "${principal_names}" -V +52w "${temp}/persisted/etc/ssh/ssh_host_ed25519_key.pub"
+
+cp "${user_ca_pub}" "${temp}/persisted/etc/ssh/user_cas.pub"
+
+echo "${temp}"
+
+#nix run github:nix-community/nixos-anywhere -- --extra-files "${temp}" --flake .#${host_type} --target-host root@${hostname}

host_config/code_server_disk.nix

@@ -0,0 +1,89 @@
+{ lib, ... }:
+{
+  disko.devices = {
+    disk.disk1 = {
+      device = lib.mkDefault "/dev/sda";
+      type = "disk";
+      content = {
+        type = "gpt";
+        partitions = {
+          boot = {
+            size = "1M";
+            type = "EF02";
+            priority = 1;
+          };
+          esp = {
+            name = "ESP";
+            size = "500M";
+            type = "EF00";
+            content = {
+              type = "filesystem";
+              format = "vfat";
+              mountpoint = "/boot";
+            };
+          };
+          zfs = {
+            name = "zfs";
+            size = "100%";
+            content = {
+              type = "zfs";
+              pool = "zroot";
+            };
+          };
+        };
+      };
+    };
+
+    zpool = {
+      zroot = {
+        type = "zpool";
+        mode = "";
+        # May cause issues if we end up adding any pools that will be imported before boot, but at the time this was added, we only had a "zroot" pool that was always imported at boot via `boot.zfs.extraPools`, so this was fine.
+        options.cachefile = "none";
+        rootFsOptions = {
+          compression = "zstd";
+          "com.sun:auto-snapshot" = "false";
+          canmount = "off";
+        };
+        mountpoint = null;
+        postCreateHook = ''
+          zfs list -t snapshot -H -o name | grep -E '^zroot/root@blank$' || zfs snapshot zroot/root@blank
+        '';
+        postMountHook = ''
+          mkdir -p /mnt/persisted/var/lib/systemd
+          mkdir -p /mnt/persisted/etc/ssh
+          mkdir -p /mnt/persisted/secrets
+          mkdir -p /mnt/persisted/root
+        '';
+
+        datasets = {
+          root = {
+            type = "zfs_fs";
+            mountpoint = "/";
+            options.canmount = "on";
+          };
+          nix = {
+            type = "zfs_fs";
+            mountpoint = "/nix";
+            options.canmount = "on";
+          };
+          replicated = {
+            type = "zfs_fs";
+            mountpoint = null;
+            options.canmount = "off";
+          };
+          "replicated/home" = {
+            type = "zfs_fs";
+            mountpoint = "/home";
+            options.canmount = "on";
+          };
+          "replicated/persisted" = {
+            type = "zfs_fs";
+            mountpoint = "/persisted";
+            options.canmount = "on";
+          };
+        };
+      };
+    };
+  };
+}

host_config/default.nix

@@ -0,0 +1,166 @@
+{ self, moduleWithSystem, ... }: {
+  flake.nixosModules.code-server = moduleWithSystem (
+    { ... }: # Note: only explicit parameters are passed to this.
+    { pkgs, modulesPath, lib, ... }: {
+      imports = [
+        self.inputs.disko.nixosModules.disko
+        (modulesPath + "/installer/scan/not-detected.nix")
+        (modulesPath + "/profiles/headless.nix")
+        (modulesPath + "/profiles/minimal.nix")
+        (modulesPath + "/profiles/qemu-guest.nix")
+        ./code_server_disk.nix
+      ];
+
+      system.stateVersion = "24.11";
+
+      boot.kernelParams = [ "zfs.zfs_arc_max=536870912" ];
+      boot.zfs.extraPools = [ "zroot" ];
+      boot.initrd.postMountCommands = lib.mkAfter ''
+        zfs rollback -r zroot/root@blank
+      '';
+
+      services.zfs.autoScrub.enable = true;
+      boot.loader.grub = {
+        enable = true;
+        # No need to set devices, disko will add all devices that have an EF02 partition to the list already.
+        # devices = [];
+        efiSupport = true;
+        efiInstallAsRemovable = true;
+      };
+
+      fileSystems = {
+        "/var/lib/systemd" = {
+          device = "/persisted/var/lib/systemd";
+          options = [ "bind" ];
+        };
+        "/var/lib/forgejo" = {
+          device = "/persisted/var/lib/forgejo";
+          options = [ "bind" ];
+        };
+      };
+
+      networking.hostId = "9f1dfd86"; # Required by ZFS.
+      networking.useNetworkd = true;
+      networking.firewall.logRefusedConnections = false;
+
+      nix.gc.automatic = true;
+      nix.gc.dates = "02:15";
+
+      services.cloud-init = {
+        enable = true;
+        network.enable = true;
+        settings = {
+          datasource_list = [ "Hetzner" ];
+
+          # The NixOS cloud-init settings declares the entire `system_info` with `lib.mkDefault`, so we need to copy the defaults from it here and make the changes we want to make.
+          system_info = {
+            paths = {
+              cloud_dir = "/persisted/var/lib/cloud";
+            };
+            distro = "nixos";
+            network = {
+              renderers = [ "networkd" ];
+              activators = [ "networkd" ];
+            };
+          };
+
+          cloud_init_modules = [
+            "migrator"
+            "seed_random"
+            "bootcmd"
+          ];
+
+          cloud_config_modules = [
+            "ssh-import-id"
+            "timezone"
+            "runcmd"
+            "ssh"
+          ];
+
+          cloud_final_modules = [
+            "keys-to-console"
+            "final-message"
+            "power-state-change"
+          ];
+        };
+      };
+
+      services.openssh = {
+        enable = true;
+        hostKeys = [
+          {
+            path = "/persisted/etc/ssh/ssh_host_ed25519_key";
+            type = "ed25519";
+          }
+        ];
+        settings = {
+          PasswordAuthentication = false;
+        };
+        extraConfig = ''
+          HostCertificate /persisted/etc/ssh/ssh_host_ed25519_key-cert.pub
+          TrustedUserCAKeys /persisted/etc/ssh/user_cas.pub
+        '';
+      };
+
+      users.users.root = {
+        home = lib.mkForce "/persisted/root";
+      };
+
+      networking.firewall.allowedTCPPorts = [ 80 443 ];
+      # UDP allowed for HTTP/3.
+      networking.firewall.allowedUDPPorts = [ 80 443 ];
+
+      services.caddy = {
+        enable = true;
+
+        globalConfig = ''
+          # Comment this if building the prod image. The following is only useful for testing.
+          # local_certs
+          skip_install_trust
+        '';
+
+        virtualHosts."code.akols.com".extraConfig = ''
+          encode zstd gzip
+          reverse_proxy http://127.0.0.1:3000
+        '';
+      };
+
+      services.forgejo = {
+        enable = true;
+
+        package = pkgs.forgejo;
+        lfs.enable = true;
+
+        settings = {
+          service = {
+            DISABLE_REGISTRATION = true;
+          };
+
+          database = {
+            SQLITE_JOURNAL_MODE = "WAL";
+          };
+
+          cache = {
+            ADAPTER = "twoqueue";
+            HOST = "{\"size\":100,\"recent_ratio\":0.25,\"ghost_ratio\":0.5}";
+          };
+
+          server = {
+            HTTP_ADDR = "127.0.0.1";
+            HTTP_PORT = 3000;
+            DOMAIN = "code.akols.com";
+            ROOT_URL = "https://code.akols.com";
+          };
+
+          session = {
+            COOKIE_SECURE = true;
+          };
+
+          security = {
+            LOGIN_REMEMBER_DAYS = 365;
+          };
+        };
+      };
+    }
+  );
+}

host_config/sign_user_pub.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
+user_ca_key="${SCRIPT_DIR}/ssh_certs/user_ca"
+
+if [ ! -f "${user_ca_key}" ]
+then
+  echo "User CA key not found."
+  exit 1
+fi
+
+username=$1
+principals=$2
+user_pub=$3
+
+if [ ! -f "${user_pub}" ]
+then
+    echo "User public key not found."
+    exit 1
+fi
+
+ssh-keygen -s "${user_ca_key}" -I "${username}" -n "${principals}" -V +52w "${user_pub}"
+echo "Done!"

host_config/ssh_certs/.gitignore

@@ -0,0 +1,2 @@
+host_ca
+user_ca

host_config/ssh_certs/host_ca.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJygYxMUdGgApUE3KirRQVgG2X5zWurIBPbwEc10FxDi epesooj host ca

host_config/ssh_certs/user_ca.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINdiqYdA3pm9yKgR5hFlL7ZeSV3xeKH9HwyNwaxY6yZZ epesooj user ca