home/nixos-servers
home/nixos-servers
Archived
2
0
Fork 0
Code Issues 8 Pull requests Projects Releases Packages Wiki Activity Actions

Use nix-snapshotter as k3s' snapshotter and image service

Browse source 
Replace k3s' embedded containerd with Nix-managed one
Deploy test k8s with image from Nix store
This commit is contained in:
Pim Kunis 2024-06-24 23:31:06 +02:00
parent a62d854a0e
commit e7d75fbb21
5 changed files with 162 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

80
nixos-modules/k3s/default.nix
View file

@ -1,6 +1,24 @@
{ inputs, pkgs, lib, config, ... }:
let cfg = config.lab.k3s;
in {
let
cfg = config.lab.k3s;
k3s-cni-plugins = pkgs.buildEnv {
name = "k3s-cni-plugins";
paths = with pkgs; [
cni-plugins
cni-plugin-flannel
];
};
image = pkgs.nix-snapshotter.buildImage {
name = "redis";
resolvedByNix = true;
config = {
entrypoint = [ "${pkgs.redis}/bin/redis-server" ];
};
};
in
{
options.lab.k3s = {
enable = lib.mkOption {
default = false;
@ -48,16 +66,66 @@ in {
firewall.enable = lib.mkForce false;
};
virtualisation.containerd = {
enable = true;
settings = {
version = 2;
proxy_plugins.nix = {
type = "snapshot";
address = "/run/nix-snapshotter/nix-snapshotter.sock";
};
plugins = {
"io.containerd.grpc.v1.cri" = {
stream_server_address = "127.0.0.1";
stream_server_port = "10010";
enable_selinux = false;
enable_unprivileged_ports = true;
enable_unprivileged_icmp = true;
disable_apparmor = true;
disable_cgroup = true;
restrict_oom_score_adj = true;
sandbox_image = "rancher/mirrored-pause:3.6";
containerd.snapshotter = "nix";
cni = {
conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/";
bin_dir = "${k3s-cni-plugins}/bin";
};
};
"io.containerd.transfer.v1.local".unpack_config = [{
platform = "linux/amd64";
snapshotter = "nix";
}];
};
};
};
services = {
nix-snapshotter.enable = true;
k3s =
let
serverFlags = "--tls-san ${config.networking.fqdn} --disable servicelb --cluster-cidr=10.42.0.0/16,2001:cafe:42::/56 --service-cidr=10.43.0.0/16,2001:cafe:43::/112";
serverFlagList = [
"--image-service-endpoint=unix:///run/nix-snapshotter/nix-snapshotter.sock"
"--snapshotter=overlayfs"
"--container-runtime-endpoint=unix:///run/containerd/containerd.sock"
"--tls-san=${config.networking.fqdn}"
"--disable=servicelb"
"--cluster-cidr=10.42.0.0/16,2001:cafe:42::/56"
"--service-cidr=10.43.0.0/16,2001:cafe:43::/112"
];
serverFlags = builtins.concatStringsSep " " serverFlagList;
in
{
enable = true;
role = cfg.role;
tokenFile = config.sops.secrets."k3s/serverToken".path;
extraFlags = lib.mkIf (cfg.role == "server") serverFlags;
extraFlags = lib.mkIf (cfg.role == "server") (lib.mkForce serverFlags);
clusterInit = cfg.clusterInit;
serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr;
};
@ -98,6 +166,10 @@ in {
cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt
cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt
'';
nix-snapshotter-image = ''
ln -sf ${image} /root/image.tar
'';
};
};