Merge branch 'kubernetes'

This commit is contained in:
Pim Kunis 2023-12-16 14:07:45 +01:00
commit 06aa435612
13 changed files with 271 additions and 29 deletions

View file

@ -25,3 +25,43 @@ Additionally, it deploys an age identity, which is later used for decrypting sec
## Deployment ## Deployment
Deployment can simply be done as follows: `deploy` Deployment can simply be done as follows: `deploy`
## Creating an admin certificate for k3s
Create the admin's private key:
```
openssl genpkey -algorithm ed25519 -out <username>-key.pem
```
Create a CSR for the admin:
```
openssl req -new -key <username>-key.pem -out <username>.csr -subj "/CN=<username>"
```
Create a Kubernetes CSR object on the cluster:
```
k3s kubectl create -f - <<EOF
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
name: <username>-csr
spec:
request: $(cat <username>.csr | base64 | tr -d '\n')
expirationSeconds: 307584000 # 10 years
signerName: kubernetes.io/kube-apiserver-client
usages:
- digital signature
- key encipherment
- client auth
EOF
```
Approve and sign the admin's CSR:
```
k3s kubectl certificate approve <username>-csr
```
Extract the resulting signed certificate from the CSR object:
```
k3s kubectl get csr <username>-csr -o jsonpath='{.status.certificate}' | base64 --decode > <username>.crt
```

3
cluster/README.md Normal file
View file

@ -0,0 +1,3 @@
```
nix --extra-experimental-features nix-command --extra-experimental-features flakes run .#kubenix
```

99
cluster/flake.lock Normal file
View file

@ -0,0 +1,99 @@
{
"nodes": {
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"kubenix": {
"inputs": {
"flake-compat": "flake-compat",
"nixpkgs": "nixpkgs",
"systems": "systems",
"treefmt": "treefmt"
},
"locked": {
"lastModified": 1700116223,
"narHash": "sha256-Pld/UXlBcIDnQMY0JkDzChJkbof/zEcRkaiXtzvArEE=",
"owner": "hall",
"repo": "kubenix",
"rev": "e4d036576436b9983216584a89388af3da995043",
"type": "github"
},
"original": {
"owner": "hall",
"repo": "kubenix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1686488075,
"narHash": "sha256-2otSBt2hbeD+5yY25NF3RhWx7l5SDt1aeU3cJ/9My4M=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "9401a0c780b49faf6c28adf55764f230301d0dce",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"kubenix": "kubenix"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"id": "systems",
"type": "indirect"
}
},
"treefmt": {
"inputs": {
"nixpkgs": [
"kubenix",
"nixpkgs"
]
},
"locked": {
"lastModified": 1688026376,
"narHash": "sha256-qJmkr9BWDpqblk4E9/rCsAEl39y2n4Ycw6KRopvpUcY=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "df3f32b0cc253dfc7009b7317e8f0e7ccd70b1cf",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "treefmt-nix",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

22
cluster/flake.nix Normal file
View file

@ -0,0 +1,22 @@
{
inputs.kubenix.url = "github:hall/kubenix";
outputs = { self, kubenix, ... }:
let
system = "x86_64-linux";
in {
kubenix = kubenix.packages.${system}.default.override {
module = {kubenix, ...}: {
imports = [kubenix.modules.k8s];
kubernetes = {
kubeconfig = "/etc/rancher/k3s/k3s.yaml";
version = "1.24";
};
kubenix.project = "yeet";
kubernetes.resources.pods.web1.spec.containers.nginx.image = "nginx";
kubernetes.resources.pods.web2.spec.containers.nginx.image = "nginx";
};
};
};
}

View file

@ -112,6 +112,7 @@
]; ];
networking = { networking = {
domain = "hyp";
firewall.enable = false; firewall.enable = false;
useDHCP = false; useDHCP = false;
@ -169,10 +170,10 @@
}; };
}; };
virtualisation.libvirtd.enable = true;
hardware.cpu.intel.updateMicrocode = hardware.cpu.intel.updateMicrocode =
lib.mkDefault config.hardware.enableRedistributableFirmware; lib.mkDefault config.hardware.enableRedistributableFirmware;
age.identityPaths = [ "/root/age_ed25519" ]; age.identityPaths = [ "/root/age_ed25519" ];
virtualisation.libvirtd.enable = true;
} }

View file

@ -5,32 +5,41 @@
nixpkgs.url = "github:nixos/nixpkgs/nixos-23.05"; nixpkgs.url = "github:nixos/nixpkgs/nixos-23.05";
nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
deploy-rs.url = "github:serokell/deploy-rs"; deploy-rs.url = "github:serokell/deploy-rs";
disko = { disko = {
url = "github:nix-community/disko"; url = "github:nix-community/disko";
inputs.nixpkgs.follows = "nixpkgs"; inputs.nixpkgs.follows = "nixpkgs";
}; };
agenix = { agenix = {
url = "github:ryantm/agenix"; url = "github:ryantm/agenix";
inputs.nixpkgs.follows = "nixpkgs"; inputs.nixpkgs.follows = "nixpkgs";
}; };
}; };
outputs = { self, nixpkgs, deploy-rs, disko, agenix, nixpkgs-unstable, ... }: outputs =
{ self, nixpkgs, deploy-rs, disko, agenix, nixpkgs-unstable, ... }:
let let
system = "x86_64-linux"; system = "x86_64-linux";
pkgs = nixpkgs.legacyPackages.${system}; pkgs = nixpkgs.legacyPackages.${system};
pkgs-unstable = nixpkgs-unstable.legacyPackages.${system}; pkgs-unstable = nixpkgs-unstable.legacyPackages.${system};
machines = import ./machines; machines = import ./machines;
# TODO: Maybe use mergeAttrLists
mkNixosSystems = systemDef: mkNixosSystems = systemDef:
nixpkgs.lib.foldlAttrs (acc: name: machine: nixpkgs.lib.foldlAttrs
(acc: name: machine:
acc // { acc // {
"${name}" = nixpkgs.lib.nixosSystem (systemDef machine); "${name}" = nixpkgs.lib.nixosSystem (systemDef machine);
}) { } machines; })
{ }
machines;
mkDeployNodes = nodeDef: mkDeployNodes = nodeDef:
nixpkgs.lib.foldlAttrs nixpkgs.lib.foldlAttrs
(acc: name: machine: acc // { "${name}" = nodeDef machine; }) { } (acc: name: machine: acc // { "${name}" = nodeDef machine; })
{ }
machines; machines;
in { in
{
devShells.${system}.default = pkgs.mkShell { devShells.${system}.default = pkgs.mkShell {
packages = [ packages = [
pkgs.libsecret pkgs.libsecret
@ -43,6 +52,7 @@
pkgs.postgresql_15 pkgs.postgresql_15
pkgs-unstable.opentofu pkgs-unstable.opentofu
pkgs.cdrtools pkgs.cdrtools
pkgs.kubectl
]; ];
}; };
@ -73,6 +83,7 @@
}; };
checks = builtins.mapAttrs checks = builtins.mapAttrs
(system: deployLib: deployLib.deployChecks self.deploy) deploy-rs.lib; (system: deployLib: deployLib.deployChecks self.deploy)
deploy-rs.lib;
}; };
} }

View file

@ -14,6 +14,8 @@
}; };
terraformDatabase.enable = true; terraformDatabase.enable = true;
k3s.enable = true;
}; };
disko.devices = { disko.devices = {

View file

@ -3,17 +3,35 @@ let cfg = config.custom.dataDisk;
in { in {
options = { options = {
custom = { custom = {
dataDisk.enable = lib.mkOption { dataDisk = {
enable = lib.mkOption {
default = false; default = false;
type = lib.types.bool; type = lib.types.bool;
description = '' description = ''
Whether to automatically mount /dev/sda1 on /mnt/data Whether to automatically mount a disk to be used as a data disk.
''; '';
}; };
mountPoint = lib.mkOption {
default = "/mnt/data";
type = lib.types.str;
description = ''
Mount point of the data disk (if enabled).
'';
};
devicePath = lib.mkOption {
default = "/dev/sda1";
type = lib.types.str;
description = ''
Path of the device to be used as a data disk.
'';
};
};
}; };
}; };
config = lib.mkIf cfg.enable { config = lib.mkIf cfg.enable {
fileSystems."/mnt/data" = { device = "/dev/sda1"; }; fileSystems.${cfg.mountPoint} = { device = cfg.devicePath; };
}; };
} }

View file

@ -1,3 +1,3 @@
{ {
imports = [ ./terraform-database.nix ./data-disk.nix ./ssh-certificates.nix ]; imports = [ ./terraform-database.nix ./data-disk.nix ./ssh-certificates.nix ./k3s.nix ];
} }

View file

@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: pim-cluster-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: pim

31
modules/custom/k3s.nix Normal file
View file

@ -0,0 +1,31 @@
{ pkgs, lib, config, ... }:
let cfg = config.custom.k3s;
in {
options = {
custom = {
k3s.enable = lib.mkOption {
default = false;
type = lib.types.bool;
description = ''
Whether to start k3s with custom configuration.
'';
};
};
};
config = lib.mkIf cfg.enable {
environment.systemPackages = [ pkgs.k3s ];
services.k3s.enable = true;
services.k3s.role = "server";
services.k3s.extraFlags = "--tls-san ${config.networking.fqdn} --data-dir ${config.custom.dataDisk.mountPoint}/k3s";
# TODO: use kubenix for this.
system.activationScripts.k3s-bootstrap.text =
let
k3sBootstrapFile = pkgs.writeText "k3s-bootstrap" (builtins.readFile ./k3s-bootstrap.yaml);
in
''
ln -sf ${k3sBootstrapFile} ${config.custom.dataDisk.mountPoint}/k3s/server/manifests/k3s-bootstrap.yaml
'';
};
}

View file

@ -20,14 +20,16 @@ in {
package = pkgs.postgresql_15; package = pkgs.postgresql_15;
enableTCPIP = true; enableTCPIP = true;
dataDir = lib.mkIf config.custom.dataDisk.enable dataDir = lib.mkIf config.custom.dataDisk.enable
"/mnt/data/postgresql/${config.services.postgresql.package.psqlSchema}"; "${config.custom.dataDisk.mountPoint}/postgresql/${config.services.postgresql.package.psqlSchema}";
authentication = '' authentication = ''
hostssl terraformstates terraform all cert hostssl terraformstates terraform all cert
''; '';
settings = let settings =
let
serverCert = builtins.toFile "postgresql_server.crt" serverCert = builtins.toFile "postgresql_server.crt"
(builtins.readFile ../../postgresql_server.crt); (builtins.readFile ../../postgresql_server.crt);
in { in
{
ssl = true; ssl = true;
ssl_cert_file = serverCert; ssl_cert_file = serverCert;
ssl_key_file = config.age.secrets."postgresql_server.key".path; ssl_key_file = config.age.secrets."postgresql_server.key".path;

View file

@ -15,6 +15,7 @@ table inet nixos-fw {
chain input-allow { chain input-allow {
tcp dport 22 accept tcp dport 22 accept
tcp dport 5432 accept comment "PostgreSQL server" tcp dport 5432 accept comment "PostgreSQL server"
tcp dport 6443 accept comment "k3s"
icmp type echo-request accept comment "allow ping" icmp type echo-request accept comment "allow ping"
icmpv6 type != { nd-redirect, 139 } accept comment "Accept all ICMPv6 messages except redirects and node information queries (type 139). See RFC 4890, section 4.4." icmpv6 type != { nd-redirect, 139 } accept comment "Accept all ICMPv6 messages except redirects and node information queries (type 139). See RFC 4890, section 4.4."
ip6 daddr fe80::/64 udp dport 546 accept comment "DHCPv6 client" ip6 daddr fe80::/64 udp dport 546 accept comment "DHCPv6 client"