From 472175c5a30477795a20a88b9e3567cf96ba1182 Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Sun, 28 Jan 2024 11:48:13 +0100 Subject: [PATCH] integrate VM definitions --- flake.nix | 72 +------- nixos/default.nix | 250 +++++++++++++++------------ nixos/machines/default.nix | 54 +++--- nixos/modules/networking/default.nix | 13 +- nixos/modules/storage.nix | 4 +- nixos/vm.nix | 119 +++++++++++++ 6 files changed, 296 insertions(+), 216 deletions(-) create mode 100644 nixos/vm.nix diff --git a/flake.nix b/flake.nix index f7ef537..61bb6b3 100644 --- a/flake.nix +++ b/flake.nix @@ -39,6 +39,7 @@ lib = pkgs.lib; pkgs-unstable = nixpkgs-unstable.legacyPackages.${system}; machines = import ./nixos/machines; + physicalMachines = lib.filterAttrs (n: v: v.type == "physical") machines; # TODO: Maybe use mergeAttrLists mkNixosSystems = systemDef: nixpkgs.lib.foldlAttrs @@ -47,12 +48,12 @@ "${name}" = nixpkgs.lib.nixosSystem (systemDef machine); }) { } - machines; + physicalMachines; mkDeployNodes = nodeDef: nixpkgs.lib.foldlAttrs (acc: name: machine: acc // { "${name}" = nodeDef machine; }) { } - machines; + physicalMachines; in { devShells.${system}.default = pkgs.mkShell { @@ -76,15 +77,8 @@ nixosConfigurations = mkNixosSystems (machine: { inherit system; - specialArgs = { inherit kubenix dns; }; - modules = [ - microvm.nixosModules.host - machine.nixosModule - disko.nixosModules.disko - agenix.nixosModules.default - ./nixos - { networking.hostName = machine.name; } - ]; + specialArgs = { inherit machines machine kubenix dns microvm disko agenix; }; + modules = [ ./nixos ]; }); deploy = { @@ -92,10 +86,11 @@ user = "root"; nodes = mkDeployNodes (machine: { - hostname = machine.hostName; + # TODO: simply get this from nixos configuration? + hostname = "${machine.hostName}.${machine.domain}"; profiles.system = { path = deploy-rs.lib.${system}.activate.nixos - self.nixosConfigurations.${machine.name}; + self.nixosConfigurations.${machine.hostName}; }; }); }; @@ -103,56 +98,5 @@ checks = builtins.mapAttrs (system: deployLib: deployLib.deployChecks self.deploy) deploy-rs.lib; - - apps.${system} = - let - hostNames = builtins.concatStringsSep " " (builtins.map (host: "\"${host.config.networking.fqdn}\"") (builtins.attrValues self.nixosConfigurations)); - in - { - reboot-all-vms = - let - reboot-all-vms = pkgs.writeScriptBin "reboot-all-vms" '' - hostNames=(${hostNames}) - for hostName in ''${hostNames[@]}; do - units=$(${pkgs.openssh}/bin/ssh root@$hostName systemctl list-units --all) - microvmUnits=$(${pkgs.coreutils}/bin/echo "$units" | ${pkgs.gnugrep}/bin/grep 'microvm@.*\.service' | ${pkgs.gawkInteractive}/bin/awk '{print $1}') - if [ -n "$microvmUnits" ]; then - for microvmUnit in "$microvmUnits"; do - ${pkgs.coreutils}/bin/echo "Restarting $microvmUnit on $hostName" - ${pkgs.openssh}/bin/ssh root@$hostName systemctl restart "$microvmUnit" - done - fi - done - ''; - in - { - type = "app"; - program = "${reboot-all-vms}/bin/reboot-all-vms"; - }; - - reboot-vm = - let - reboot-vm = pkgs.writeScriptBin "reboot-vm" '' - if [ -z "$1" ]; then - ${pkgs.coreutils}/bin/echo "Please provide a VM name!" - exit 1 - fi - - hostNames=(${hostNames}) - unitName="microvm@$1.service" - for hostName in ''${hostNames[@]}; do - units=$(${pkgs.openssh}/bin/ssh root@$hostName systemctl list-units --all) - if [[ "''${units[@]}" =~ "$unitName" ]]; then - ${pkgs.coreutils}/bin/echo "Restarting $unitName on $hostName" - ${pkgs.openssh}/bin/ssh root@$hostName systemctl restart "$unitName" - fi - done - ''; - in - { - type = "app"; - program = "${reboot-vm}/bin/reboot-vm"; - }; - }; }; } diff --git a/nixos/default.nix b/nixos/default.nix index 15a8f26..1cef138 100644 --- a/nixos/default.nix +++ b/nixos/default.nix @@ -1,142 +1,168 @@ -{ pkgs, config, lib, modulesPath, ... }: { +{ pkgs, config, lib, modulesPath, machine, microvm, disko, agenix, machines, ... }: { imports = [ (modulesPath + "/installer/scan/not-detected.nix") ./modules ./lab.nix + machine.nixosModule + disko.nixosModules.disko + agenix.nixosModules.default + microvm.nixosModules.host ]; - boot = { - kernelModules = [ "kvm-intel" ]; - extraModulePackages = [ ]; + config = { + boot = { + kernelModules = [ "kvm-intel" ]; + extraModulePackages = [ ]; - initrd = { - availableKernelModules = [ - "ahci" - "xhci_pci" - "nvme" - "usbhid" - "usb_storage" - "sd_mod" - "sdhci_pci" - ]; - kernelModules = [ ]; - }; + initrd = { + availableKernelModules = [ + "ahci" + "xhci_pci" + "nvme" + "usbhid" + "usb_storage" + "sd_mod" + "sdhci_pci" + ]; + kernelModules = [ ]; + }; - loader = { - systemd-boot.enable = true; - efi.canTouchEfiVariables = true; - }; - }; - - time.timeZone = "Europe/Amsterdam"; - - i18n = { - defaultLocale = "en_US.UTF-8"; - - extraLocaleSettings = { - LC_ADDRESS = "nl_NL.UTF-8"; - LC_IDENTIFICATION = "nl_NL.UTF-8"; - LC_MEASUREMENT = "nl_NL.UTF-8"; - LC_MONETARY = "nl_NL.UTF-8"; - LC_NAME = "nl_NL.UTF-8"; - LC_NUMERIC = "nl_NL.UTF-8"; - LC_PAPER = "nl_NL.UTF-8"; - LC_TELEPHONE = "nl_NL.UTF-8"; - LC_TIME = "nl_NL.UTF-8"; - }; - }; - - services = { - openssh = { - enable = true; - openFirewall = true; - - settings = { - PasswordAuthentication = false; - KbdInteractiveAuthentication = false; + loader = { + systemd-boot.enable = true; + efi.canTouchEfiVariables = true; }; }; - xserver = { - layout = "us"; - xkbVariant = ""; + time.timeZone = "Europe/Amsterdam"; + + i18n = { + defaultLocale = "en_US.UTF-8"; + + extraLocaleSettings = { + LC_ADDRESS = "nl_NL.UTF-8"; + LC_IDENTIFICATION = "nl_NL.UTF-8"; + LC_MEASUREMENT = "nl_NL.UTF-8"; + LC_MONETARY = "nl_NL.UTF-8"; + LC_NAME = "nl_NL.UTF-8"; + LC_NUMERIC = "nl_NL.UTF-8"; + LC_PAPER = "nl_NL.UTF-8"; + LC_TELEPHONE = "nl_NL.UTF-8"; + LC_TIME = "nl_NL.UTF-8"; + }; }; - }; - users.users.root.openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOodpLr+FDRyKyHjucHizNLVFHZ5AQmE9GmxMnOsSoaw pimkunis@thinkpadpim" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINUZp4BCxf7uLa1QWonx/Crf8tYZ5MKIZ+EuaBa82LrV user@user-laptop" - ]; + services = { + openssh = { + enable = true; + openFirewall = true; - programs = { - ssh = { - knownHosts = { - dmz = { - hostNames = [ "*.dmz" ]; - publicKey = - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAX2IhgHNxC6JTvLu9cej+iWuG+uJFMXn4AiRro9533x"; - certAuthority = true; - }; - - hypervisors = { - hostNames = [ "*.hyp" ]; - publicKey = - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzRkH3d/KVJQouswY/DMpenWbDFVOnI3Vut0xR0e1tb"; - certAuthority = true; + settings = { + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; }; }; + xserver = { + layout = "us"; + xkbVariant = ""; + }; }; - neovim = { - enable = true; - vimAlias = true; - viAlias = true; + users.users.root.openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOodpLr+FDRyKyHjucHizNLVFHZ5AQmE9GmxMnOsSoaw pimkunis@thinkpadpim" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINUZp4BCxf7uLa1QWonx/Crf8tYZ5MKIZ+EuaBa82LrV user@user-laptop" + ]; + + programs = { + ssh = { + knownHosts = { + dmz = { + hostNames = [ "*.dmz" ]; + publicKey = + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAX2IhgHNxC6JTvLu9cej+iWuG+uJFMXn4AiRro9533x"; + certAuthority = true; + }; + + hypervisors = { + hostNames = [ "*.hyp" ]; + publicKey = + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzRkH3d/KVJQouswY/DMpenWbDFVOnI3Vut0xR0e1tb"; + certAuthority = true; + }; + }; + + }; + + neovim = { + enable = true; + vimAlias = true; + viAlias = true; + }; }; - }; - nixpkgs = { - config.allowUnfree = true; - hostPlatform = "x86_64-linux"; - }; + nixpkgs = { + config.allowUnfree = true; + hostPlatform = "x86_64-linux"; + }; - environment.systemPackages = with pkgs; [ - neofetch - wget - git - btop - htop - ripgrep - dig - tree - file - tcpdump - lsof - parted - radvd - ]; + environment.systemPackages = with pkgs; [ + neofetch + wget + git + btop + htop + ripgrep + dig + tree + file + tcpdump + lsof + parted + radvd + ]; - hardware.cpu.intel.updateMicrocode = - lib.mkDefault config.hardware.enableRedistributableFirmware; + hardware.cpu.intel.updateMicrocode = config.hardware.enableRedistributableFirmware; - age.identityPaths = [ "/etc/age_ed25519" ]; + age.identityPaths = [ "/etc/age_ed25519" ]; - virtualisation.libvirtd.enable = true; + virtualisation.libvirtd.enable = true; - nix = { - package = pkgs.nixFlakes; - extraOptions = '' - experimental-features = nix-command flakes - ''; - }; + nix = { + package = pkgs.nixFlakes; + extraOptions = '' + experimental-features = nix-command flakes + ''; + }; - system = { - stateVersion = "23.05"; + system = { + stateVersion = "23.05"; - activationScripts.diff = '' - if [[ -e /run/current-system ]]; then - ${pkgs.nix}/bin/nix store diff-closures /run/current-system "$systemConfig" - fi - ''; + activationScripts.diff = '' + if [[ -e /run/current-system ]]; then + ${pkgs.nix}/bin/nix store diff-closures /run/current-system "$systemConfig" + fi + ''; + }; + + microvm.vms = + let + vmsForHypervisor = lib.attrValues (lib.filterAttrs (n: v: v.type == "virtual" && v.hypervisorName == machine.hostName) machines); + in + lib.attrsets.mergeAttrsList (map + (vm: + { + "${vm.hostName}" = { + # TODO Simplify? + specialArgs = { inherit agenix disko pkgs lib microvm; machine = vm; hypervisorConfig = config; }; + config = { + imports = [ + ./vm.nix + ]; + }; + }; + } + ) + vmsForHypervisor + ); }; } diff --git a/nixos/machines/default.nix b/nixos/machines/default.nix index 558d110..40c30e4 100644 --- a/nixos/machines/default.nix +++ b/nixos/machines/default.nix @@ -1,7 +1,9 @@ { jefke = { - name = "jefke"; - hostName = "jefke.hyp"; + # TODO: directly set and read this from nixos config. + hostName = "jefke"; + domain = "hyp"; + type = "physical"; nixosModule.lab = { terraformDatabase.enable = true; @@ -20,8 +22,9 @@ }; atlas = { - name = "atlas"; - hostName = "atlas.hyp"; + hostName = "atlas"; + domain = "hyp"; + type = "physical"; nixosModule = { config, ... }: let inherit (config.lab.networking) dmzServicesIPv4 dmzServicesIPv6; in @@ -49,8 +52,9 @@ }; lewis = { - name = "lewis"; - hostName = "lewis.hyp"; + hostName = "lewis"; + domain = "hyp"; + type = "physical"; nixosModule = { pkgs, ... }: { lab = { @@ -67,35 +71,21 @@ userCert = builtins.readFile ./lewis_user_ed25519-cert.pub; }; }; + }; + }; - microvm.vms.my-microvm.config = { - services.openssh.enable = true; - networking.firewall.enable = false; + my-microvm = { + hostName = "my-microvm"; + domain = "dmz"; + type = "virtual"; + hypervisorName = "lewis"; - users.users.root.openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOodpLr+FDRyKyHjucHizNLVFHZ5AQmE9GmxMnOsSoaw pimkunis@thinkpadpim" - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINUZp4BCxf7uLa1QWonx/Crf8tYZ5MKIZ+EuaBa82LrV user@user-laptop" - ]; + nixosModule = { pkgs, config, ... }: { + programs.bash.interactiveShellInit = '' + echo "Hello world from inside a virtual machine!" | ${pkgs.lolcat}/bin/lolcat + ''; - programs.bash.interactiveShellInit = '' - echo "Hello world from inside a virtual machine!" | ${pkgs.lolcat}/bin/lolcat - ''; - - microvm = { - shares = [{ - source = "/nix/store"; - mountPoint = "/nix/.ro-store"; - tag = "ro-store"; - proto = "virtiofs"; - }]; - - interfaces = [{ - type = "tap"; - id = "vm-my-microvm"; - mac = "48:2D:63:E1:C5:39"; - }]; - }; - }; + lab.vmMacAddress = "BA:DB:EE:F0:00:00"; }; }; } diff --git a/nixos/modules/networking/default.nix b/nixos/modules/networking/default.nix index de60475..0a0e640 100644 --- a/nixos/modules/networking/default.nix +++ b/nixos/modules/networking/default.nix @@ -1,4 +1,4 @@ -{ lib, config, ... }: +{ lib, config, machine, ... }: let cfg = config.lab.networking; in { imports = [ ./dmz ]; @@ -89,17 +89,18 @@ in { config = { networking = { - domain = "hyp"; + hostName = machine.hostName; + domain = machine.domain; + nftables.enable = true; + useDHCP = machine.type == "virtual"; + firewall = { enable = true; checkReversePath = false; }; - - nftables.enable = true; - useDHCP = false; }; - systemd.network = { + systemd.network = lib.mkIf (machine.type == "physical") { enable = true; netdevs = { diff --git a/nixos/modules/storage.nix b/nixos/modules/storage.nix index f01b665..3ad1b78 100644 --- a/nixos/modules/storage.nix +++ b/nixos/modules/storage.nix @@ -1,4 +1,4 @@ -{ lib, config, ... }: +{ lib, config, machine, ... }: let cfg = config.lab.storage; in { options.lab.storage = { @@ -25,7 +25,7 @@ in { }; }; - config = { + config = lib.mkIf (machine.type == "physical") { fileSystems.${cfg.dataMountPoint}.device = cfg.dataPartition; # TODO: Rename this to 'osDisk'. Unfortunately, we would need to run nixos-anywhere again then. diff --git a/nixos/vm.nix b/nixos/vm.nix new file mode 100644 index 0000000..7887b26 --- /dev/null +++ b/nixos/vm.nix @@ -0,0 +1,119 @@ +{ pkgs, lib, config, agenix, disko, machine, hypervisorConfig, ... }: { + imports = [ + ./modules + ./lab.nix + machine.nixosModule + disko.nixosModules.disko + agenix.nixosModules.default + ]; + + options.lab.vmMacAddress = lib.mkOption { + type = lib.types.str; + description = '' + The MAC address of the VM's main NIC. + ''; + }; + + # TODO: remove overlap with physical nixos module + # Perhaps a sane defaults module? + config = { + system.stateVersion = hypervisorConfig.system.stateVersion; + time.timeZone = "Europe/Amsterdam"; + + i18n = { + defaultLocale = "en_US.UTF-8"; + + extraLocaleSettings = { + LC_ADDRESS = "nl_NL.UTF-8"; + LC_IDENTIFICATION = "nl_NL.UTF-8"; + LC_MEASUREMENT = "nl_NL.UTF-8"; + LC_MONETARY = "nl_NL.UTF-8"; + LC_NAME = "nl_NL.UTF-8"; + LC_NUMERIC = "nl_NL.UTF-8"; + LC_PAPER = "nl_NL.UTF-8"; + LC_TELEPHONE = "nl_NL.UTF-8"; + LC_TIME = "nl_NL.UTF-8"; + }; + }; + + services = { + openssh = { + enable = true; + openFirewall = true; + + settings = { + PasswordAuthentication = false; + KbdInteractiveAuthentication = false; + }; + }; + + xserver = { + layout = "us"; + xkbVariant = ""; + }; + }; + + users.users.root.openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOodpLr+FDRyKyHjucHizNLVFHZ5AQmE9GmxMnOsSoaw pimkunis@thinkpadpim" + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINUZp4BCxf7uLa1QWonx/Crf8tYZ5MKIZ+EuaBa82LrV user@user-laptop" + ]; + + programs = { + ssh = { + knownHosts = { + dmz = { + hostNames = [ "*.dmz" ]; + publicKey = + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAX2IhgHNxC6JTvLu9cej+iWuG+uJFMXn4AiRro9533x"; + certAuthority = true; + }; + + hypervisors = { + hostNames = [ "*.hyp" ]; + publicKey = + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzRkH3d/KVJQouswY/DMpenWbDFVOnI3Vut0xR0e1tb"; + certAuthority = true; + }; + }; + + }; + + neovim = { + enable = true; + vimAlias = true; + viAlias = true; + }; + }; + + environment.systemPackages = with pkgs; [ + neofetch + wget + git + btop + htop + ripgrep + dig + tree + file + tcpdump + lsof + parted + radvd + ]; + + microvm = { + shares = [{ + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + tag = "ro-store"; + proto = "virtiofs"; + }]; + + interfaces = [{ + type = "tap"; + id = "vm-${machine.hostName}"; + mac = config.lab.vmMacAddress; + }]; + }; + }; +}