home/nixos-servers
home/nixos-servers
2
0
Fork 0
Code Issues 8 Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'kubernetes'

Browse source
This commit is contained in:
Pim Kunis 2023-12-16 14:07:45 +01:00
commit 06aa435612
13 changed files with 271 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
README.md
View file

@ -25,3 +25,43 @@ Additionally, it deploys an age identity, which is later used for decrypting sec
## Deployment
Deployment can simply be done as follows: `deploy`
## Creating an admin certificate for k3s
Create the admin's private key:
```
openssl genpkey -algorithm ed25519 -out <username>-key.pem
```
Create a CSR for the admin:
```
openssl req -new -key <username>-key.pem -out <username>.csr -subj "/CN=<username>"
```
Create a Kubernetes CSR object on the cluster:
```
k3s kubectl create -f - <<EOF
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
name: <username>-csr
spec:
request: $(cat <username>.csr | base64 | tr -d '\n')
expirationSeconds: 307584000 # 10 years
signerName: kubernetes.io/kube-apiserver-client
usages:
- digital signature
- key encipherment
- client auth
EOF
```
Approve and sign the admin's CSR:
```
k3s kubectl certificate approve <username>-csr
```
Extract the resulting signed certificate from the CSR object:
```
k3s kubectl get csr <username>-csr -o jsonpath='{.status.certificate}' | base64 --decode > <username>.crt
```

3
cluster/README.md Normal file
View file

@ -0,0 +1,3 @@
```
nix --extra-experimental-features nix-command --extra-experimental-features flakes run .#kubenix
```

99
cluster/flake.lock Normal file
View file

@ -0,0 +1,99 @@
{
"nodes": {
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"kubenix": {
"inputs": {
"flake-compat": "flake-compat",
"nixpkgs": "nixpkgs",
"systems": "systems",
"treefmt": "treefmt"
},
"locked": {
"lastModified": 1700116223,
"narHash": "sha256-Pld/UXlBcIDnQMY0JkDzChJkbof/zEcRkaiXtzvArEE=",
"owner": "hall",
"repo": "kubenix",
"rev": "e4d036576436b9983216584a89388af3da995043",
"type": "github"
},
"original": {
"owner": "hall",
"repo": "kubenix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1686488075,
"narHash": "sha256-2otSBt2hbeD+5yY25NF3RhWx7l5SDt1aeU3cJ/9My4M=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "9401a0c780b49faf6c28adf55764f230301d0dce",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"kubenix": "kubenix"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"id": "systems",
"type": "indirect"
}
},
"treefmt": {
"inputs": {
"nixpkgs": [
"kubenix",
"nixpkgs"
]
},
"locked": {
"lastModified": 1688026376,
"narHash": "sha256-qJmkr9BWDpqblk4E9/rCsAEl39y2n4Ycw6KRopvpUcY=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "df3f32b0cc253dfc7009b7317e8f0e7ccd70b1cf",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "treefmt-nix",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

22
cluster/flake.nix Normal file
View file

@ -0,0 +1,22 @@
{
inputs.kubenix.url = "github:hall/kubenix";
outputs = { self, kubenix, ... }:
let
system = "x86_64-linux";
in {
kubenix = kubenix.packages.${system}.default.override {
module = {kubenix, ...}: {
imports = [kubenix.modules.k8s];
kubernetes = {
kubeconfig = "/etc/rancher/k3s/k3s.yaml";
version = "1.24";
};
kubenix.project = "yeet";
kubernetes.resources.pods.web1.spec.containers.nginx.image = "nginx";
kubernetes.resources.pods.web2.spec.containers.nginx.image = "nginx";
};
};
};
}

5
configuration.nix
View file

@ -112,6 +112,7 @@
];
networking = {
domain = "hyp";
firewall.enable = false;
useDHCP = false;
@ -169,10 +170,10 @@
};
};
virtualisation.libvirtd.enable = true;
hardware.cpu.intel.updateMicrocode =
lib.mkDefault config.hardware.enableRedistributableFirmware;
age.identityPaths = [ "/root/age_ed25519" ];
virtualisation.libvirtd.enable = true;
}

29
flake.nix
View file

@ -5,32 +5,41 @@
nixpkgs.url = "github:nixos/nixpkgs/nixos-23.05";
nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
deploy-rs.url = "github:serokell/deploy-rs";
disko = {
url = "github:nix-community/disko";
inputs.nixpkgs.follows = "nixpkgs";
};
agenix = {
url = "github:ryantm/agenix";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs = { self, nixpkgs, deploy-rs, disko, agenix, nixpkgs-unstable, ... }:
outputs =
{ self, nixpkgs, deploy-rs, disko, agenix, nixpkgs-unstable, ... }:
let
system = "x86_64-linux";
pkgs = nixpkgs.legacyPackages.${system};
pkgs-unstable = nixpkgs-unstable.legacyPackages.${system};
machines = import ./machines;
# TODO: Maybe use mergeAttrLists
mkNixosSystems = systemDef:
nixpkgs.lib.foldlAttrs (acc: name: machine:
acc // {
"${name}" = nixpkgs.lib.nixosSystem (systemDef machine);
}) { } machines;
nixpkgs.lib.foldlAttrs
(acc: name: machine:
acc // {
"${name}" = nixpkgs.lib.nixosSystem (systemDef machine);
})
{ }
machines;
mkDeployNodes = nodeDef:
nixpkgs.lib.foldlAttrs
(acc: name: machine: acc // { "${name}" = nodeDef machine; }) { }
machines;
in {
(acc: name: machine: acc // { "${name}" = nodeDef machine; })
{ }
machines;
in
{
devShells.${system}.default = pkgs.mkShell {
packages = [
pkgs.libsecret
@ -43,6 +52,7 @@
pkgs.postgresql_15
pkgs-unstable.opentofu
pkgs.cdrtools
pkgs.kubectl
];
};
@ -73,6 +83,7 @@
};
checks = builtins.mapAttrs
(system: deployLib: deployLib.deployChecks self.deploy) deploy-rs.lib;
(system: deployLib: deployLib.deployChecks self.deploy)
deploy-rs.lib;
};
}

2
machines/default.nix
View file

@ -14,6 +14,8 @@
};
terraformDatabase.enable = true;
k3s.enable = true;
};
disko.devices = {

32
modules/custom/data-disk.nix
View file

@ -3,17 +3,35 @@ let cfg = config.custom.dataDisk;
in {
options = {
custom = {
dataDisk.enable = lib.mkOption {
default = false;
type = lib.types.bool;
description = ''
Whether to automatically mount /dev/sda1 on /mnt/data
'';
dataDisk = {
enable = lib.mkOption {
default = false;
type = lib.types.bool;
description = ''
Whether to automatically mount a disk to be used as a data disk.
'';
};
mountPoint = lib.mkOption {
default = "/mnt/data";
type = lib.types.str;
description = ''
Mount point of the data disk (if enabled).
'';
};
devicePath = lib.mkOption {
default = "/dev/sda1";
type = lib.types.str;
description = ''
Path of the device to be used as a data disk.
'';
};
};
};
};
config = lib.mkIf cfg.enable {
fileSystems."/mnt/data" = { device = "/dev/sda1"; };
fileSystems.${cfg.mountPoint} = { device = cfg.devicePath; };
};
}

2
modules/custom/default.nix
View file

@ -1,3 +1,3 @@
{
imports = [ ./terraform-database.nix ./data-disk.nix ./ssh-certificates.nix ];
imports = [ ./terraform-database.nix ./data-disk.nix ./ssh-certificates.nix ./k3s.nix ];
}

12
modules/custom/k3s-bootstrap.yaml Normal file
View file

@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: pim-cluster-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: pim

31
modules/custom/k3s.nix Normal file
View file

@ -0,0 +1,31 @@
{ pkgs, lib, config, ... }:
let cfg = config.custom.k3s;
in {
options = {
custom = {
k3s.enable = lib.mkOption {
default = false;
type = lib.types.bool;
description = ''
Whether to start k3s with custom configuration.
'';
};
};
};
config = lib.mkIf cfg.enable {
environment.systemPackages = [ pkgs.k3s ];
services.k3s.enable = true;
services.k3s.role = "server";
services.k3s.extraFlags = "--tls-san ${config.networking.fqdn} --data-dir ${config.custom.dataDisk.mountPoint}/k3s";
# TODO: use kubenix for this.
system.activationScripts.k3s-bootstrap.text =
let
k3sBootstrapFile = pkgs.writeText "k3s-bootstrap" (builtins.readFile ./k3s-bootstrap.yaml);
in
''
ln -sf ${k3sBootstrapFile} ${config.custom.dataDisk.mountPoint}/k3s/server/manifests/k3s-bootstrap.yaml
'';
};
}

22
modules/custom/terraform-database.nix
View file

@ -20,19 +20,21 @@ in {
package = pkgs.postgresql_15;
enableTCPIP = true;
dataDir = lib.mkIf config.custom.dataDisk.enable
"/mnt/data/postgresql/${config.services.postgresql.package.psqlSchema}";
"${config.custom.dataDisk.mountPoint}/postgresql/${config.services.postgresql.package.psqlSchema}";
authentication = ''
hostssl terraformstates terraform all cert
'';
settings = let
serverCert = builtins.toFile "postgresql_server.crt"
(builtins.readFile ../../postgresql_server.crt);
in {
ssl = true;
ssl_cert_file = serverCert;
ssl_key_file = config.age.secrets."postgresql_server.key".path;
ssl_ca_file = serverCert;
};
settings =
let
serverCert = builtins.toFile "postgresql_server.crt"
(builtins.readFile ../../postgresql_server.crt);
in
{
ssl = true;
ssl_cert_file = serverCert;
ssl_key_file = config.age.secrets."postgresql_server.key".path;
ssl_ca_file = serverCert;
};
ensureUsers = [{
name = "terraform";
ensurePermissions = { "DATABASE terraformstates" = "ALL PRIVILEGES"; };

1
nftables.conf
View file

@ -15,6 +15,7 @@ table inet nixos-fw {
chain input-allow {
tcp dport 22 accept
tcp dport 5432 accept comment "PostgreSQL server"
tcp dport 6443 accept comment "k3s"
icmp type echo-request accept comment "allow ping"
icmpv6 type != { nd-redirect, 139 } accept comment "Accept all ICMPv6 messages except redirects and node information queries (type 139). See RFC 4890, section 4.4."
ip6 daddr fe80::/64 udp dport 546 accept comment "DHCPv6 client"