re-add kubernetes

This commit is contained in:
Pim Kunis 2024-03-19 22:56:52 +01:00
parent f032ebf0c7
commit 4cf1d13384
11 changed files with 194 additions and 8 deletions

41
docs/k3s.md Normal file
View file

@ -0,0 +1,41 @@
# Kubernetes
## Creating an admin certificate for k3s
Create the admin's private key:
```
openssl genpkey -algorithm ed25519 -out <username>-key.pem
```
Create a CSR for the admin:
```
openssl req -new -key <username>-key.pem -out <username>.csr -subj "/CN=<username>"
```
Create a Kubernetes CSR object on the cluster:
```
k3s kubectl create -f - <<EOF
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
name: <username>-csr
spec:
request: $(cat <username>.csr | base64 | tr -d '\n')
expirationSeconds: 307584000 # 10 years
signerName: kubernetes.io/kube-apiserver-client
usages:
- digital signature
- key encipherment
- client auth
EOF
```
Approve and sign the admin's CSR:
```
k3s kubectl certificate approve <username>-csr
```
Extract the resulting signed certificate from the CSR object:
```
k3s kubectl get csr <username>-csr -o jsonpath='{.status.certificate}' | base64 --decode > <username>.crt
```

View file

@ -122,6 +122,22 @@
"type": "github"
}
},
"flake-compat_2": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-utils": {
"locked": {
"lastModified": 1614513358,
@ -157,7 +173,7 @@
},
"flake-utils_3": {
"inputs": {
"systems": "systems_4"
"systems": "systems_5"
},
"locked": {
"lastModified": 1701680307,
@ -194,6 +210,29 @@
"type": "github"
}
},
"kubenix": {
"inputs": {
"flake-compat": "flake-compat_2",
"nixpkgs": [
"nixpkgs"
],
"systems": "systems_4",
"treefmt": "treefmt"
},
"locked": {
"lastModified": 1705801181,
"narHash": "sha256-vH+n5qMnwFCx3LMON2hQMi9PjMpmTraGYXe1czJTfAg=",
"owner": "hall",
"repo": "kubenix",
"rev": "76b8053b27b062b11f0c9b495050cc55606ac9dc",
"type": "github"
},
"original": {
"owner": "hall",
"repo": "kubenix",
"type": "github"
}
},
"microvm": {
"inputs": {
"flake-utils": "flake-utils_3",
@ -250,11 +289,11 @@
},
"nixpkgs-unstable": {
"locked": {
"lastModified": 1709499969,
"narHash": "sha256-PXi7pnvg+uYY5oUm8Vgw4pXZaGUGjGIOZ4gQ/yGVoPo=",
"lastModified": 1710789445,
"narHash": "sha256-MDljQvOU5jX4S5MUGJ3izSfzaeIWQ1phoc3MarCyUes=",
"owner": "pizzapim",
"repo": "nixpkgs",
"rev": "c74dae81760b4e48633133e689358ff4a120eabe",
"rev": "9235a7f0f9f6f9970cf46345bfb23ad84fa2a5a7",
"type": "github"
},
"original": {
@ -287,6 +326,7 @@
"disko": "disko",
"dns": "dns",
"flake-utils": "flake-utils_2",
"kubenix": "kubenix",
"microvm": "microvm",
"nixos-hardware": "nixos-hardware",
"nixpkgs": "nixpkgs_2",
@ -364,8 +404,43 @@
"type": "github"
},
"original": {
"id": "systems",
"type": "indirect"
}
},
"systems_5": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"treefmt": {
"inputs": {
"nixpkgs": [
"kubenix",
"nixpkgs"
]
},
"locked": {
"lastModified": 1688026376,
"narHash": "sha256-qJmkr9BWDpqblk4E9/rCsAEl39y2n4Ycw6KRopvpUcY=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "df3f32b0cc253dfc7009b7317e8f0e7ccd70b1cf",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "treefmt-nix",
"type": "github"
}
},

View file

@ -27,6 +27,11 @@
url = "github:astro/microvm.nix";
inputs.nixpkgs.follows = "nixpkgs";
};
kubenix = {
url = "github:hall/kubenix";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs =

View file

@ -95,6 +95,7 @@
socat
pciutils
dmidecode
openssl
];
nixpkgs.overlays = [

View file

@ -1,4 +1,4 @@
{ nixpkgs, nixpkgs-unstable, machines, physicalMachines, dns, microvm, disko, agenix, nixos-hardware, ... }:
{ nixpkgs, nixpkgs-unstable, machines, physicalMachines, dns, microvm, disko, agenix, nixos-hardware, kubenix, ... }:
let
mkNixosSystems = systemDef:
builtins.mapAttrs
@ -11,7 +11,7 @@ in
nixosConfigurations = mkNixosSystems (name: machine: {
system = machine.arch;
specialArgs = { inherit nixpkgs-unstable machines machine dns microvm disko agenix nixos-hardware; };
specialArgs = { inherit nixpkgs-unstable machines machine dns microvm disko agenix nixos-hardware kubenix; };
modules = [
../.
{ networking.hostName = name; }

View file

@ -15,6 +15,8 @@
hostCert = builtins.readFile ./certificates/jefke/host_ed25519.crt;
userCert = builtins.readFile ./certificates/jefke/user_ed25519.crt;
};
k3s.enable = true;
};
};
}

View file

@ -7,5 +7,6 @@
./data-sharing.nix
./globals.nix
./monitoring
./k3s
];
}

View file

@ -0,0 +1,16 @@
{ kubenix, ... }: {
imports = [ kubenix.modules.k8s ];
kubernetes.resources.clusterRoleBindings.pim-cluster-admin = {
roleRef = {
apiGroup = "rbac.authorization.k8s.io";
kind = "ClusterRole";
name = "cluster-admin";
};
subjects = [
{
kind = "User";
name = "pim";
}
];
};
}

View file

@ -0,0 +1,36 @@
{ pkgs, lib, config, kubenix, ... }:
let cfg = config.lab.k3s;
in {
options.lab.k3s.enable = lib.mkOption {
default = false;
type = lib.types.bool;
description = ''
Whether to start k3s with custom configuration.
'';
};
config = lib.mkIf cfg.enable {
environment.systemPackages = with pkgs; [ k3s ];
networking = {
nftables.enable = lib.mkForce false;
firewall.enable = lib.mkForce false;
};
services.k3s = {
enable = true;
role = "server";
extraFlags = "--tls-san ${config.networking.fqdn} --data-dir ${config.lab.storage.dataMountPoint}/k3s";
};
system.activationScripts.k3s-bootstrap.text =
let
k3sBootstrapFile = (kubenix.evalModules.x86_64-linux {
module = import ./bootstrap.nix;
}).config.kubernetes.result;
in
''
ln -sf ${k3sBootstrapFile} ${config.lab.storage.dataMountPoint}/k3s/server/manifests/k3s-bootstrap.json
'';
};
}

View file

@ -80,6 +80,15 @@ in
};
};
users = {
users.gatus = {
isSystemUser = true;
group = "gatus";
};
groups.gatus = { };
};
system.activationScripts = lib.mkIf cfg.server.enable {
gatus = ''
mkdir -p /srv/gatus

View file

@ -1,4 +1,4 @@
{ pkgs, nixpkgs-unstable, config, lib, microvm, disko, agenix, machine, machines, dns, nixos-hardware, ... }: {
{ pkgs, nixpkgs-unstable, config, lib, microvm, disko, agenix, machine, machines, dns, nixos-hardware, kubenix, ... }: {
imports = [
microvm.nixosModules.host
]
@ -69,7 +69,7 @@
restartIfChanged = false;
specialArgs = {
inherit agenix disko pkgs lib microvm dns nixpkgs-unstable;
inherit agenix disko pkgs lib microvm dns nixpkgs-unstable kubenix;
machine = vm;
hypervisorConfig = config;
};