home/nixos-servers
home/nixos-servers
Archived
2
0
Fork 0
Code Issues 8 Pull requests Projects Releases Packages Wiki Activity Actions

Format repo

Browse source
This commit is contained in:
Pim Kunis 2024-10-28 14:12:06 +01:00
parent eed797fb13
commit 5341235bc3
21 changed files with 1200 additions and 1076 deletions
Download patch file Download diff file Expand all files Collapse all files

4
modules/k3s/bootstrap.nix
View file

@ -1,5 +1,5 @@
{ kubenix, ... }: {
imports = [ kubenix.modules.k8s ];
{kubenix, ...}: {
imports = [kubenix.modules.k8s];
kubernetes.resources.clusterRoleBindings.cluster-admins = {
roleRef = {
apiGroup = "rbac.authorization.k8s.io";

148
modules/k3s/default.nix
View file

@ -1,8 +1,13 @@
{ self, inputs, pkgs, lib, config, ... }:
let
cfg = config.lab.k3s;
in
{
self,
inputs,
pkgs,
lib,
config,
...
}: let
cfg = config.lab.k3s;
in {
options.lab.k3s = {
enable = lib.mkOption {
default = false;
@ -62,68 +67,66 @@ in
address = "/run/nix-snapshotter/nix-snapshotter.sock";
};
plugins =
let
k3s-cni-plugins = pkgs.buildEnv {
name = "k3s-cni-plugins";
paths = with pkgs; [
cni-plugins
cni-plugin-flannel
];
};
in
{
"io.containerd.grpc.v1.cri" = {
stream_server_address = "127.0.0.1";
stream_server_port = "10010";
enable_selinux = false;
enable_unprivileged_ports = true;
enable_unprivileged_icmp = true;
disable_apparmor = true;
disable_cgroup = true;
restrict_oom_score_adj = true;
sandbox_image = "rancher/mirrored-pause:3.6";
containerd.snapshotter = "nix";
plugins = let
k3s-cni-plugins = pkgs.buildEnv {
name = "k3s-cni-plugins";
paths = with pkgs; [
cni-plugins
cni-plugin-flannel
];
};
in {
"io.containerd.grpc.v1.cri" = {
stream_server_address = "127.0.0.1";
stream_server_port = "10010";
enable_selinux = false;
enable_unprivileged_ports = true;
enable_unprivileged_icmp = true;
disable_apparmor = true;
disable_cgroup = true;
restrict_oom_score_adj = true;
sandbox_image = "rancher/mirrored-pause:3.6";
containerd.snapshotter = "nix";
cni = {
conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/";
bin_dir = "${k3s-cni-plugins}/bin";
};
cni = {
conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/";
bin_dir = "${k3s-cni-plugins}/bin";
};
};
"io.containerd.transfer.v1.local".unpack_config = [{
"io.containerd.transfer.v1.local".unpack_config = [
{
platform = "linux/amd64";
snapshotter = "nix";
}];
};
}
];
};
};
};
services = {
nix-snapshotter.enable = true;
k3s =
let
serverFlagList = [
"--image-service-endpoint=unix:///run/nix-snapshotter/nix-snapshotter.sock"
"--snapshotter=overlayfs"
"--container-runtime-endpoint=unix:///run/containerd/containerd.sock"
"--tls-san=${config.networking.fqdn}"
"--disable=servicelb"
"--cluster-cidr=10.42.0.0/16,2001:cafe:42::/56"
"--service-cidr=10.43.0.0/16,2001:cafe:43::/112"
];
k3s = let
serverFlagList = [
"--image-service-endpoint=unix:///run/nix-snapshotter/nix-snapshotter.sock"
"--snapshotter=overlayfs"
"--container-runtime-endpoint=unix:///run/containerd/containerd.sock"
"--tls-san=${config.networking.fqdn}"
"--disable=servicelb"
"--cluster-cidr=10.42.0.0/16,2001:cafe:42::/56"
"--service-cidr=10.43.0.0/16,2001:cafe:43::/112"
];
serverFlags = builtins.concatStringsSep " " serverFlagList;
in
{
enable = true;
role = cfg.role;
tokenFile = config.sops.secrets."k3s/serverToken".path;
extraFlags = lib.mkIf (cfg.role == "server") (lib.mkForce serverFlags);
clusterInit = cfg.clusterInit;
serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr;
};
serverFlags = builtins.concatStringsSep " " serverFlagList;
in {
enable = true;
role = cfg.role;
tokenFile = config.sops.secrets."k3s/serverToken".path;
extraFlags = lib.mkIf (cfg.role == "server") (lib.mkForce serverFlags);
clusterInit = cfg.clusterInit;
serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr;
};
# Required for Longhorn
openiscsi = {
@ -143,11 +146,14 @@ in
k3s-bootstrap = lib.mkIf (cfg.role == "server") {
text = (
let
k3sBootstrapFile = (inputs.kubenix.evalModules.x86_64-linux {
module = import ./bootstrap.nix;
}).config.kubernetes.result;
in
''
k3sBootstrapFile =
(inputs.kubenix.evalModules.x86_64-linux {
module = import ./bootstrap.nix;
})
.config
.kubernetes
.result;
in ''
mkdir -p /var/lib/rancher/k3s/server/manifests
ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json
''
@ -166,18 +172,16 @@ in
};
};
sops.secrets =
let
keyPathBase = "/var/lib/rancher/k3s/server/tls";
in
{
"k3s/serverToken" = { };
"k3s/keys/clientCAKey".path = "${keyPathBase}/client-ca.key";
"k3s/keys/requestHeaderCAKey".path = "${keyPathBase}/request-header-ca.key";
"k3s/keys/serverCAKey".path = "${keyPathBase}/server-ca.key";
"k3s/keys/serviceKey".path = "${keyPathBase}/service.key";
"k3s/keys/etcd/peerCAKey".path = "${keyPathBase}/etcd/peer-ca.key";
"k3s/keys/etcd/serverCAKey".path = "${keyPathBase}/etcd/server-ca.key";
};
sops.secrets = let
keyPathBase = "/var/lib/rancher/k3s/server/tls";
in {
"k3s/serverToken" = {};
"k3s/keys/clientCAKey".path = "${keyPathBase}/client-ca.key";
"k3s/keys/requestHeaderCAKey".path = "${keyPathBase}/request-header-ca.key";
"k3s/keys/serverCAKey".path = "${keyPathBase}/server-ca.key";
"k3s/keys/serviceKey".path = "${keyPathBase}/service.key";
"k3s/keys/etcd/peerCAKey".path = "${keyPathBase}/etcd/peer-ca.key";
"k3s/keys/etcd/serverCAKey".path = "${keyPathBase}/etcd/server-ca.key";
};
};
}