Resolve cyberchef and radicale images using nix-snapshotter

Increase inotify max user instances to 256
Disable tailscale by default
This commit is contained in:
Pim Kunis 2024-08-25 17:04:31 +02:00
parent 52efd614fe
commit e9899c0d0f
11 changed files with 129 additions and 81 deletions

View file

@ -91,6 +91,7 @@
boot = lib.mkIf (! machine.isRaspberryPi) { boot = lib.mkIf (! machine.isRaspberryPi) {
kernelModules = [ "kvm-intel" ]; kernelModules = [ "kvm-intel" ];
extraModulePackages = [ ]; extraModulePackages = [ ];
kernel.sysctl."fs.inotify.max_user_instances" = 256;
initrd = { initrd = {
kernelModules = [ ]; kernelModules = [ ];

View file

@ -1,2 +1,2 @@
{ cyberchef = { cyberchef = { finalImageName = "mpepping/cyberchef"; finalImageTag = "latest"; imageDigest = "sha256:5044c72dd8070fb6e0595e720fc4440bf6168493b2433db06a1c966406398ba2"; imageName = "mpepping/cyberchef"; sha256 = "177yjfbz0ijc8lfqfr50fhqqmjk72373c0igyrxv3wwg0pyrgpv4"; }; }; } { cyberchef = { finalImageName = "mpepping/cyberchef"; finalImageTag = "latest"; imageDigest = "sha256:2c89d08580395b932c92d708041c2a702dc8fa899fcc1677901c2dc881bed789"; imageName = "mpepping/cyberchef"; sha256 = "1frlvv6lyghf99pa37l48r7j2wvh7mcb9x99fvf0ba2zhq2xfsy4"; }; radicale = { finalImageName = "tomsquest/docker-radicale"; finalImageTag = "3.2.2.0"; imageDigest = "sha256:af050e02c4a3f7385a09595dd2a1424db6831aa9f24404095c6d2244d1c94138"; imageName = "tomsquest/docker-radicale"; sha256 = "07mxn6iqm0fhb06dwmxdhqnw3c8yi2dm2jpb2n80dxzmvavrv0lk"; }; }

View file

@ -30,12 +30,16 @@ in
let let
images = { images = {
cyberchef = { cyberchef = {
cyberchef = { name = "mpepping/cyberchef";
image-name = "mpepping/cyberchef"; tag = "latest";
image-tag = "latest"; };
};
radicale = {
name = "tomsquest/docker-radicale";
tag = "3.2.2.0";
}; };
}; };
imagesJSON = builtins.toFile "images.json" (builtins.toJSON images); imagesJSON = builtins.toFile "images.json" (builtins.toJSON images);
in in
pkgs.writers.writePython3Bin "prefetch-container-images" pkgs.writers.writePython3Bin "prefetch-container-images"
@ -55,31 +59,28 @@ in
with open(images_file_name, 'r') as file: with open(images_file_name, 'r') as file:
data = json.load(file) data = json.load(file)
for project_name, images in data.items(): for image_name, image in data.items():
print(f"Prefetching images for project {project_name}", file=sys.stderr) name = image["name"]
tag = image["tag"]
for image_name, image in images.items(): print(f"Prefetching image {name}:{tag}", file=sys.stderr)
name = image["image-name"]
tag = image["image-tag"]
print(f"Prefetching image {name}:{tag}", file=sys.stderr) prefetch_args = [
prefetch_docker_cmd,
"--os", "linux",
"--arch", "amd64",
"--image-name", name,
"--image-tag", tag,
"--json",
"--quiet"
]
result = subprocess.run(prefetch_args,
check=True,
capture_output=True,
text=True)
prefetch_args = [ prefetch_data = json.loads(result.stdout)
prefetch_docker_cmd, results[image_name] = prefetch_data
"--os", "linux",
"--arch", "amd64",
"--image-name", name,
"--image-tag", tag,
"--json",
"--quiet"
]
result = subprocess.run(prefetch_args,
check=True,
capture_output=True,
text=True)
prefetch_data = json.loads(result.stdout)
results[project_name][image_name] = prefetch_data
with tempfile.NamedTemporaryFile(mode='w+', suffix='.json') as temp_file: with tempfile.NamedTemporaryFile(mode='w+', suffix='.json') as temp_file:
json.dump(results, temp_file, indent=4) json.dump(results, temp_file, indent=4)

View file

@ -9,6 +9,7 @@
spec.containers.cyberchef = { spec.containers.cyberchef = {
image = myLib.globals.images.cyberchef; image = myLib.globals.images.cyberchef;
imagePullPolicy = "Always";
ports.web.containerPort = 8000; ports.web.containerPort = 8000;
}; };
}; };

View file

@ -5,6 +5,7 @@
nixosModule.lab = { nixosModule.lab = {
storage.profile = "kubernetes"; storage.profile = "kubernetes";
tailscale.enable = true;
k3s = { k3s = {
enable = true; enable = true;

View file

@ -5,6 +5,7 @@
nixosModule.lab = { nixosModule.lab = {
storage.profile = "kubernetes"; storage.profile = "kubernetes";
tailscale.enable = true;
k3s = { k3s = {
enable = true; enable = true;

View file

@ -11,6 +11,7 @@
storage.profile = "kubernetes"; storage.profile = "kubernetes";
backups.enable = true; backups.enable = true;
data-sharing.enable = true; data-sharing.enable = true;
tailscale.enable = true;
k3s = { k3s = {
enable = true; enable = true;

View file

@ -7,7 +7,11 @@
lab = { lab = {
storage.profile = "pi"; storage.profile = "pi";
monitoring.server.enable = true; monitoring.server.enable = true;
tailscale.advertiseExitNode = true;
tailscale = {
advertiseExitNode = true;
enable = true;
};
}; };
}; };
}; };

View file

@ -44,7 +44,7 @@
nextcloudPostgres = "postgres:15"; nextcloudPostgres = "postgres:15";
inbucket = "inbucket/inbucket:edge"; inbucket = "inbucket/inbucket:edge";
syncthing = "lscr.io/linuxserver/syncthing:1.27.10"; syncthing = "lscr.io/linuxserver/syncthing:1.27.10";
radicale = "tomsquest/docker-radicale:3.2.2.0"; radicale = "nix:0/var/docker_images/radicale.tar";
ntfy = "binwiederhier/ntfy:v2.11.0"; ntfy = "binwiederhier/ntfy:v2.11.0";
forgejo = "codeberg.org/forgejo/forgejo:8.0.1"; forgejo = "codeberg.org/forgejo/forgejo:8.0.1";
pihole = "pihole/pihole:2024.07.0"; pihole = "pihole/pihole:2024.07.0";
@ -53,7 +53,7 @@
immichRedis = "docker.io/redis:6.2-alpine@sha256:e3b17ba9479deec4b7d1eeec1548a253acc5374d68d3b27937fcfe4df8d18c7e"; immichRedis = "docker.io/redis:6.2-alpine@sha256:e3b17ba9479deec4b7d1eeec1548a253acc5374d68d3b27937fcfe4df8d18c7e";
immichPostgres = "docker.io/tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:90724186f0a3517cf6914295b5ab410db9ce23190a2d9d0b9dd6463e3fa298f0"; immichPostgres = "docker.io/tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:90724186f0a3517cf6914295b5ab410db9ce23190a2d9d0b9dd6463e3fa298f0";
kitchenowl = "tombursch/kitchenowl:v0.5.2"; kitchenowl = "tombursch/kitchenowl:v0.5.2";
cyberchef = "mpepping/cyberchef"; cyberchef = "nix:0/var/docker_images/cyberchef.tar";
freshrss = "freshrss/freshrss:1.24.2"; freshrss = "freshrss/freshrss:1.24.2";
bind9 = "ubuntu/bind9:9.18-22.04_beta"; bind9 = "ubuntu/bind9:9.18-22.04_beta";
dnsmasq = "dockurr/dnsmasq:2.90"; dnsmasq = "dockurr/dnsmasq:2.90";

View file

@ -1,22 +1,6 @@
{ inputs, pkgs, lib, config, ... }: { self, inputs, pkgs, lib, config, ... }:
let let
cfg = config.lab.k3s; cfg = config.lab.k3s;
k3s-cni-plugins = pkgs.buildEnv {
name = "k3s-cni-plugins";
paths = with pkgs; [
cni-plugins
cni-plugin-flannel
];
};
image = pkgs.nix-snapshotter.buildImage {
name = "redis";
resolvedByNix = true;
config = {
entrypoint = [ "${pkgs.redis}/bin/redis-server" ];
};
};
in in
{ {
options.lab.k3s = { options.lab.k3s = {
@ -78,30 +62,40 @@ in
address = "/run/nix-snapshotter/nix-snapshotter.sock"; address = "/run/nix-snapshotter/nix-snapshotter.sock";
}; };
plugins = { plugins =
"io.containerd.grpc.v1.cri" = { let
stream_server_address = "127.0.0.1"; k3s-cni-plugins = pkgs.buildEnv {
stream_server_port = "10010"; name = "k3s-cni-plugins";
enable_selinux = false; paths = with pkgs; [
enable_unprivileged_ports = true; cni-plugins
enable_unprivileged_icmp = true; cni-plugin-flannel
disable_apparmor = true; ];
disable_cgroup = true;
restrict_oom_score_adj = true;
sandbox_image = "rancher/mirrored-pause:3.6";
containerd.snapshotter = "nix";
cni = {
conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/";
bin_dir = "${k3s-cni-plugins}/bin";
}; };
}; in
{
"io.containerd.grpc.v1.cri" = {
stream_server_address = "127.0.0.1";
stream_server_port = "10010";
enable_selinux = false;
enable_unprivileged_ports = true;
enable_unprivileged_icmp = true;
disable_apparmor = true;
disable_cgroup = true;
restrict_oom_score_adj = true;
sandbox_image = "rancher/mirrored-pause:3.6";
containerd.snapshotter = "nix";
"io.containerd.transfer.v1.local".unpack_config = [{ cni = {
platform = "linux/amd64"; conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/";
snapshotter = "nix"; bin_dir = "${k3s-cni-plugins}/bin";
}]; };
}; };
"io.containerd.transfer.v1.local".unpack_config = [{
platform = "linux/amd64";
snapshotter = "nix";
}];
};
}; };
}; };
@ -145,9 +139,9 @@ in
"L+ /usr/local/bin - - - - /run/current-system/sw/bin/" "L+ /usr/local/bin - - - - /run/current-system/sw/bin/"
]; ];
system = lib.mkIf (cfg.role == "server") { system.activationScripts = {
activationScripts = { k3s-bootstrap = lib.mkIf (cfg.role == "server") {
k3s-bootstrap.text = ( text = (
let let
k3sBootstrapFile = (inputs.kubenix.evalModules.x86_64-linux { k3sBootstrapFile = (inputs.kubenix.evalModules.x86_64-linux {
module = import ./bootstrap.nix; module = import ./bootstrap.nix;
@ -158,8 +152,10 @@ in
ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json
'' ''
); );
};
k3s-certs.text = '' k3s-certs = lib.mkIf (cfg.role == "server") {
text = ''
mkdir -p /var/lib/rancher/k3s/server/tls/etcd mkdir -p /var/lib/rancher/k3s/server/tls/etcd
cp -f ${./k3s-ca/server-ca.crt} /var/lib/rancher/k3s/server/tls/server-ca.crt cp -f ${./k3s-ca/server-ca.crt} /var/lib/rancher/k3s/server/tls/server-ca.crt
cp -f ${./k3s-ca/client-ca.crt} /var/lib/rancher/k3s/server/tls/client-ca.crt cp -f ${./k3s-ca/client-ca.crt} /var/lib/rancher/k3s/server/tls/client-ca.crt
@ -167,11 +163,47 @@ in
cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt
cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt
''; '';
nix-snapshotter-image = ''
ln -sf ${image} /root/image.tar
'';
}; };
docker-images.text =
let
imageLinkDir = "/var/docker_images";
imageDefs = import "${self}/container-images.nix";
setupCommands = [
"rm -rf ${imageLinkDir}"
"mkdir -p ${imageLinkDir}"
];
getDockerImageConfig = dockerImage:
let
configJson = pkgs.runCommand "config.json"
{
nativeBuildInputs = [ pkgs.skopeo pkgs.jq ];
}
''
skopeo --tmpdir $TMPDIR --insecure-policy inspect docker-archive:${dockerImage} --config | jq '.config' > $out
'';
in
builtins.fromJSON (builtins.readFile configJson);
imageDefToLinkCommand = name: imageDef:
let
dockerImage = pkgs.dockerTools.pullImage imageDef;
nixSnapshotterImage = pkgs.nix-snapshotter.buildImage {
inherit name;
resolvedByNix = true;
fromImage = dockerImage;
config = getDockerImageConfig dockerImage;
};
imageLinkPath = "${imageLinkDir}/${name}.tar";
in
"ln -sf ${nixSnapshotterImage} ${imageLinkPath}";
linkCommandList = lib.attrsets.mapAttrsToList imageDefToLinkCommand imageDefs;
commandList = setupCommands ++ linkCommandList;
in
builtins.concatStringsSep "\n" commandList;
}; };
sops.secrets = sops.secrets =

View file

@ -4,13 +4,17 @@ let
in in
{ {
options = { options = {
lab.tailscale.advertiseExitNode = lib.mkOption { lab.tailscale = {
type = lib.types.bool; enable = lib.mkEnableOption "tailscale";
default = false;
advertiseExitNode = lib.mkOption {
type = lib.types.bool;
default = false;
};
}; };
}; };
config = { config = lib.mkIf cfg.enable {
services.tailscale = { services.tailscale = {
enable = true; enable = true;
authKeyFile = config.sops.secrets."tailscale/authKey".path; authKeyFile = config.sops.secrets."tailscale/authKey".path;
@ -25,5 +29,7 @@ in
}; };
sops.secrets."tailscale/authKey" = { }; sops.secrets."tailscale/authKey" = { };
systemd.network.wait-online.ignoredInterfaces = [ "tailscale0" ];
}; };
} }