Use nix-snapshotter as k3s' snapshotter and image service
Replace k3s' embedded containerd with Nix-managed one Deploy test k8s with image from Nix store
This commit is contained in:
parent
a62d854a0e
commit
e7d75fbb21
5 changed files with 162 additions and 4 deletions
|
|
@ -4,6 +4,7 @@
|
|||
machine.nixosModule
|
||||
inputs.disko.nixosModules.disko
|
||||
inputs.sops-nix.nixosModules.sops
|
||||
inputs.nix-snapshotter.nixosModules.nix-snapshotter
|
||||
] ++ lib.lists.optional (machine.isRaspberryPi) inputs.nixos-hardware.nixosModules.raspberry-pi-4;
|
||||
|
||||
config = {
|
||||
|
|
|
|||
60
flake.lock
60
flake.lock
|
|
@ -115,6 +115,43 @@
|
|||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-compat_3": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1696426674,
|
||||
"narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-parts": {
|
||||
"inputs": {
|
||||
"nixpkgs-lib": [
|
||||
"nix-snapshotter",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1704152458,
|
||||
"narHash": "sha256-DS+dGw7SKygIWf9w4eNBUZsK+4Ug27NwEWmn2tnbycg=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"rev": "88a2cd8166694ba0b6cb374700799cec53aef527",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils": {
|
||||
"locked": {
|
||||
"lastModified": 1614513358,
|
||||
|
|
@ -277,6 +314,28 @@
|
|||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nix-snapshotter": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat_3",
|
||||
"flake-parts": "flake-parts",
|
||||
"nixpkgs": [
|
||||
"nixpkgs-unstable"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1715438114,
|
||||
"narHash": "sha256-btb702TXuhDg0D6tW0dCOy4+II9Wl6BJ0LvpT+O9wrs=",
|
||||
"owner": "pdtpartners",
|
||||
"repo": "nix-snapshotter",
|
||||
"rev": "7b251c9356bc7bb383ebeedcd0045b3ae431bff7",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "pdtpartners",
|
||||
"repo": "nix-snapshotter",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixhelm": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils_3",
|
||||
|
|
@ -413,6 +472,7 @@
|
|||
"dns": "dns",
|
||||
"flake-utils": "flake-utils_2",
|
||||
"kubenix": "kubenix",
|
||||
"nix-snapshotter": "nix-snapshotter",
|
||||
"nixhelm": "nixhelm",
|
||||
"nixos-hardware": "nixos-hardware",
|
||||
"nixpkgs": "nixpkgs_2",
|
||||
|
|
|
|||
|
|
@ -37,6 +37,11 @@
|
|||
url = "github:Mic92/sops-nix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
|
||||
nix-snapshotter = {
|
||||
url = "github:pdtpartners/nix-snapshotter";
|
||||
inputs.nixpkgs.follows = "nixpkgs-unstable";
|
||||
};
|
||||
};
|
||||
|
||||
outputs =
|
||||
|
|
|
|||
|
|
@ -24,6 +24,26 @@ let
|
|||
];
|
||||
in
|
||||
{
|
||||
kubernetes.resources.pods.testje.spec = {
|
||||
containers.redis = {
|
||||
image = "nix:0/nix/store/5nmh9qawhbwinzxidafjlfw68wfkh0pj-nix-image-redis.tar";
|
||||
args = [ "--protected-mode" "no" ];
|
||||
|
||||
ports = [{
|
||||
name = "redis";
|
||||
containerPort = 6379;
|
||||
}];
|
||||
};
|
||||
|
||||
affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{
|
||||
matchExpressions = [{
|
||||
key = "kubernetes.io/hostname";
|
||||
operator = "In";
|
||||
values = [ "atlas" ];
|
||||
}];
|
||||
}];
|
||||
};
|
||||
|
||||
imports = [
|
||||
./base.nix
|
||||
./longhorn.nix
|
||||
|
|
|
|||
|
|
@ -1,6 +1,24 @@
|
|||
{ inputs, pkgs, lib, config, ... }:
|
||||
let cfg = config.lab.k3s;
|
||||
in {
|
||||
let
|
||||
cfg = config.lab.k3s;
|
||||
|
||||
k3s-cni-plugins = pkgs.buildEnv {
|
||||
name = "k3s-cni-plugins";
|
||||
paths = with pkgs; [
|
||||
cni-plugins
|
||||
cni-plugin-flannel
|
||||
];
|
||||
};
|
||||
|
||||
image = pkgs.nix-snapshotter.buildImage {
|
||||
name = "redis";
|
||||
resolvedByNix = true;
|
||||
config = {
|
||||
entrypoint = [ "${pkgs.redis}/bin/redis-server" ];
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options.lab.k3s = {
|
||||
enable = lib.mkOption {
|
||||
default = false;
|
||||
|
|
@ -48,16 +66,66 @@ in {
|
|||
firewall.enable = lib.mkForce false;
|
||||
};
|
||||
|
||||
virtualisation.containerd = {
|
||||
enable = true;
|
||||
|
||||
settings = {
|
||||
version = 2;
|
||||
|
||||
proxy_plugins.nix = {
|
||||
type = "snapshot";
|
||||
address = "/run/nix-snapshotter/nix-snapshotter.sock";
|
||||
};
|
||||
|
||||
plugins = {
|
||||
"io.containerd.grpc.v1.cri" = {
|
||||
stream_server_address = "127.0.0.1";
|
||||
stream_server_port = "10010";
|
||||
enable_selinux = false;
|
||||
enable_unprivileged_ports = true;
|
||||
enable_unprivileged_icmp = true;
|
||||
disable_apparmor = true;
|
||||
disable_cgroup = true;
|
||||
restrict_oom_score_adj = true;
|
||||
sandbox_image = "rancher/mirrored-pause:3.6";
|
||||
containerd.snapshotter = "nix";
|
||||
|
||||
cni = {
|
||||
conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/";
|
||||
bin_dir = "${k3s-cni-plugins}/bin";
|
||||
};
|
||||
};
|
||||
|
||||
"io.containerd.transfer.v1.local".unpack_config = [{
|
||||
platform = "linux/amd64";
|
||||
snapshotter = "nix";
|
||||
}];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
services = {
|
||||
nix-snapshotter.enable = true;
|
||||
|
||||
k3s =
|
||||
let
|
||||
serverFlags = "--tls-san ${config.networking.fqdn} --disable servicelb --cluster-cidr=10.42.0.0/16,2001:cafe:42::/56 --service-cidr=10.43.0.0/16,2001:cafe:43::/112";
|
||||
serverFlagList = [
|
||||
"--image-service-endpoint=unix:///run/nix-snapshotter/nix-snapshotter.sock"
|
||||
"--snapshotter=overlayfs"
|
||||
"--container-runtime-endpoint=unix:///run/containerd/containerd.sock"
|
||||
"--tls-san=${config.networking.fqdn}"
|
||||
"--disable=servicelb"
|
||||
"--cluster-cidr=10.42.0.0/16,2001:cafe:42::/56"
|
||||
"--service-cidr=10.43.0.0/16,2001:cafe:43::/112"
|
||||
];
|
||||
|
||||
serverFlags = builtins.concatStringsSep " " serverFlagList;
|
||||
in
|
||||
{
|
||||
enable = true;
|
||||
role = cfg.role;
|
||||
tokenFile = config.sops.secrets."k3s/serverToken".path;
|
||||
extraFlags = lib.mkIf (cfg.role == "server") serverFlags;
|
||||
extraFlags = lib.mkIf (cfg.role == "server") (lib.mkForce serverFlags);
|
||||
clusterInit = cfg.clusterInit;
|
||||
serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr;
|
||||
};
|
||||
|
|
@ -98,6 +166,10 @@ in {
|
|||
cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt
|
||||
cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt
|
||||
'';
|
||||
|
||||
nix-snapshotter-image = ''
|
||||
ln -sf ${image} /root/image.tar
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue