Merge branch 'microvm2'

This commit is contained in:
Pim Kunis 2024-01-28 13:59:25 +01:00
commit 0fd0713e08
8 changed files with 336 additions and 182 deletions

View file

@ -152,6 +152,24 @@
"type": "github"
}
},
"flake-utils_2": {
"inputs": {
"systems": "systems_3"
},
"locked": {
"lastModified": 1701680307,
"narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"home-manager": {
"inputs": {
"nixpkgs": [
@ -196,6 +214,28 @@
"type": "github"
}
},
"microvm": {
"inputs": {
"flake-utils": "flake-utils_2",
"nixpkgs": [
"nixpkgs"
],
"spectrum": "spectrum"
},
"locked": {
"lastModified": 1705263072,
"narHash": "sha256-DCqqaNWn9G81U+0Myyr36JrOKitcmS34oBWxqiHjabk=",
"owner": "astro",
"repo": "microvm.nix",
"rev": "088ba565537eaef1041a87be5a44ca0daa4e1908",
"type": "github"
},
"original": {
"owner": "astro",
"repo": "microvm.nix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1702272962,
@ -251,10 +291,27 @@
"disko": "disko",
"dns": "dns",
"kubenix": "kubenix",
"microvm": "microvm",
"nixpkgs": "nixpkgs_2",
"nixpkgs-unstable": "nixpkgs-unstable"
}
},
"spectrum": {
"flake": false,
"locked": {
"lastModified": 1703273931,
"narHash": "sha256-CJ1Crdi5fXHkCiemovsp20/RC4vpDaZl1R6V273FecI=",
"ref": "refs/heads/main",
"rev": "97e2f3429ee61dc37664b4d096b2fec48a57b691",
"revCount": 597,
"type": "git",
"url": "https://spectrum-os.org/git/spectrum"
},
"original": {
"type": "git",
"url": "https://spectrum-os.org/git/spectrum"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
@ -284,6 +341,21 @@
"type": "indirect"
}
},
"systems_3": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"treefmt": {
"inputs": {
"nixpkgs": [

View file

@ -24,29 +24,32 @@
url = "github:kirelagin/dns.nix";
inputs.nixpkgs.follows = "nixpkgs";
};
microvm = {
url = "github:astro/microvm.nix";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs =
{ self, nixpkgs, deploy-rs, disko, agenix, kubenix, nixpkgs-unstable, dns, ... }:
{ self, nixpkgs, deploy-rs, disko, agenix, kubenix, nixpkgs-unstable, dns, microvm, ... }:
let
system = "x86_64-linux";
pkgs = nixpkgs.legacyPackages.${system};
lib = pkgs.lib;
pkgs-unstable = nixpkgs-unstable.legacyPackages.${system};
machines = import ./nixos/machines;
# TODO: Maybe use mergeAttrLists
physicalMachines = lib.filterAttrs (n: v: v.type == "physical") machines;
mkNixosSystems = systemDef:
nixpkgs.lib.foldlAttrs
(acc: name: machine:
acc // {
"${name}" = nixpkgs.lib.nixosSystem (systemDef machine);
})
{ }
machines;
builtins.mapAttrs
(name: machine:
nixpkgs.lib.nixosSystem (systemDef name machine)
)
physicalMachines;
mkDeployNodes = nodeDef:
nixpkgs.lib.foldlAttrs
(acc: name: machine: acc // { "${name}" = nodeDef machine; })
{ }
machines;
builtins.mapAttrs
(name: machine: nodeDef name machine)
physicalMachines;
in
{
devShells.${system}.default = pkgs.mkShell {
@ -68,27 +71,21 @@
formatter.${system} = pkgs.nixfmt;
nixosConfigurations = mkNixosSystems (machine: {
nixosConfigurations = mkNixosSystems (name: machine: {
inherit system;
specialArgs = { inherit kubenix dns; };
modules = [
machine.nixosModule
disko.nixosModules.disko
agenix.nixosModules.default
./nixos
{ networking.hostName = machine.name; }
];
specialArgs = { inherit machines machine kubenix dns microvm disko agenix; };
modules = [ ./nixos ];
});
deploy = {
sshUser = "root";
user = "root";
nodes = mkDeployNodes (machine: {
hostname = machine.hostName;
nodes = mkDeployNodes (name: machine: {
hostname = self.nixosConfigurations.${name}.config.networking.fqdn;
profiles.system = {
path = deploy-rs.lib.${system}.activate.nixos
self.nixosConfigurations.${machine.name};
self.nixosConfigurations.${name};
};
});
};

View file

@ -1,142 +1,96 @@
{ pkgs, config, lib, modulesPath, ... }: {
{ pkgs, lib, machine, disko, agenix, ... }: {
imports = [
(modulesPath + "/installer/scan/not-detected.nix")
./modules
./lab.nix
];
machine.nixosModule
disko.nixosModules.disko
agenix.nixosModules.default
]
++ lib.lists.optional (machine.type == "physical") ./physical.nix
++ lib.lists.optional (machine.type == "virtual") ./virtual.nix;
boot = {
kernelModules = [ "kvm-intel" ];
extraModulePackages = [ ];
config = {
time.timeZone = "Europe/Amsterdam";
initrd = {
availableKernelModules = [
"ahci"
"xhci_pci"
"nvme"
"usbhid"
"usb_storage"
"sd_mod"
"sdhci_pci"
];
kernelModules = [ ];
};
i18n = {
defaultLocale = "en_US.UTF-8";
loader = {
systemd-boot.enable = true;
efi.canTouchEfiVariables = true;
};
};
time.timeZone = "Europe/Amsterdam";
i18n = {
defaultLocale = "en_US.UTF-8";
extraLocaleSettings = {
LC_ADDRESS = "nl_NL.UTF-8";
LC_IDENTIFICATION = "nl_NL.UTF-8";
LC_MEASUREMENT = "nl_NL.UTF-8";
LC_MONETARY = "nl_NL.UTF-8";
LC_NAME = "nl_NL.UTF-8";
LC_NUMERIC = "nl_NL.UTF-8";
LC_PAPER = "nl_NL.UTF-8";
LC_TELEPHONE = "nl_NL.UTF-8";
LC_TIME = "nl_NL.UTF-8";
};
};
services = {
openssh = {
enable = true;
openFirewall = true;
settings = {
PasswordAuthentication = false;
KbdInteractiveAuthentication = false;
extraLocaleSettings = {
LC_ADDRESS = "nl_NL.UTF-8";
LC_IDENTIFICATION = "nl_NL.UTF-8";
LC_MEASUREMENT = "nl_NL.UTF-8";
LC_MONETARY = "nl_NL.UTF-8";
LC_NAME = "nl_NL.UTF-8";
LC_NUMERIC = "nl_NL.UTF-8";
LC_PAPER = "nl_NL.UTF-8";
LC_TELEPHONE = "nl_NL.UTF-8";
LC_TIME = "nl_NL.UTF-8";
};
};
xserver = {
layout = "us";
xkbVariant = "";
};
};
services = {
openssh = {
enable = true;
openFirewall = true;
users.users.root.openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOodpLr+FDRyKyHjucHizNLVFHZ5AQmE9GmxMnOsSoaw pimkunis@thinkpadpim"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINUZp4BCxf7uLa1QWonx/Crf8tYZ5MKIZ+EuaBa82LrV user@user-laptop"
];
programs = {
ssh = {
knownHosts = {
dmz = {
hostNames = [ "*.dmz" ];
publicKey =
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAX2IhgHNxC6JTvLu9cej+iWuG+uJFMXn4AiRro9533x";
certAuthority = true;
};
hypervisors = {
hostNames = [ "*.hyp" ];
publicKey =
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzRkH3d/KVJQouswY/DMpenWbDFVOnI3Vut0xR0e1tb";
certAuthority = true;
settings = {
PasswordAuthentication = false;
KbdInteractiveAuthentication = false;
};
};
xserver = {
layout = "us";
xkbVariant = "";
};
};
neovim = {
enable = true;
vimAlias = true;
viAlias = true;
users.users.root.openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOodpLr+FDRyKyHjucHizNLVFHZ5AQmE9GmxMnOsSoaw pimkunis@thinkpadpim"
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINUZp4BCxf7uLa1QWonx/Crf8tYZ5MKIZ+EuaBa82LrV user@user-laptop"
];
programs = {
ssh = {
knownHosts = {
dmz = {
hostNames = [ "*.dmz" ];
publicKey =
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAX2IhgHNxC6JTvLu9cej+iWuG+uJFMXn4AiRro9533x";
certAuthority = true;
};
hypervisors = {
hostNames = [ "*.hyp" ];
publicKey =
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzRkH3d/KVJQouswY/DMpenWbDFVOnI3Vut0xR0e1tb";
certAuthority = true;
};
};
};
neovim = {
enable = true;
vimAlias = true;
viAlias = true;
};
};
};
nixpkgs = {
config.allowUnfree = true;
hostPlatform = "x86_64-linux";
};
environment.systemPackages = with pkgs; [
neofetch
wget
git
btop
htop
ripgrep
dig
tree
file
tcpdump
lsof
parted
radvd
];
hardware.cpu.intel.updateMicrocode =
lib.mkDefault config.hardware.enableRedistributableFirmware;
age.identityPaths = [ "/etc/age_ed25519" ];
virtualisation.libvirtd.enable = true;
nix = {
package = pkgs.nixFlakes;
extraOptions = ''
experimental-features = nix-command flakes
'';
};
system = {
stateVersion = "23.05";
activationScripts.diff = ''
if [[ -e /run/current-system ]]; then
${pkgs.nix}/bin/nix store diff-closures /run/current-system "$systemConfig"
fi
'';
environment.systemPackages = with pkgs; [
neofetch
wget
git
btop
htop
ripgrep
dig
tree
file
tcpdump
lsof
parted
radvd
];
};
}

View file

@ -1,31 +1,36 @@
{
jefke = {
name = "jefke";
hostName = "jefke.hyp";
type = "physical";
nixosModule.lab = {
terraformDatabase.enable = true;
nixosModule = {
networking.hostName = "jefke";
storage = {
osDisk = "/dev/sda";
dataPartition = "/dev/nvme0n1p1";
};
lab = {
terraformDatabase.enable = true;
ssh = {
useCertificates = true;
hostCert = builtins.readFile ./jefke_host_ed25519-cert.pub;
userCert = builtins.readFile ./jefke_user_ed25519-cert.pub;
storage = {
osDisk = "/dev/sda";
dataPartition = "/dev/nvme0n1p1";
};
ssh = {
useCertificates = true;
# TODO: automatically set this?
hostCert = builtins.readFile ./jefke_host_ed25519-cert.pub;
userCert = builtins.readFile ./jefke_user_ed25519-cert.pub;
};
};
};
};
atlas = {
name = "atlas";
hostName = "atlas.hyp";
type = "physical";
nixosModule = { config, ... }:
let inherit (config.lab.networking) dmzServicesIPv4 dmzServicesIPv6; in
{
networking.hostName = "atlas";
lab = {
networking = {
# TODO: Ideally, we don't have to set this here.
@ -49,22 +54,39 @@
};
lewis = {
name = "lewis";
hostName = "lewis.hyp";
type = "physical";
nixosModule.lab = {
dataHost.enable = true;
nixosModule = {
networking.hostName = "lewis";
storage = {
osDisk = "/dev/sda";
dataPartition = "/dev/nvme0n1p1";
};
lab = {
dataHost.enable = true;
ssh = {
useCertificates = true;
hostCert = builtins.readFile ./lewis_host_ed25519-cert.pub;
userCert = builtins.readFile ./lewis_user_ed25519-cert.pub;
storage = {
osDisk = "/dev/sda";
dataPartition = "/dev/nvme0n1p1";
};
ssh = {
useCertificates = true;
hostCert = builtins.readFile ./lewis_host_ed25519-cert.pub;
userCert = builtins.readFile ./lewis_user_ed25519-cert.pub;
};
};
};
};
my-microvm = {
type = "virtual";
hypervisorName = "lewis";
nixosModule = { pkgs, ... }: {
networking.hostName = "my-microvm";
lab.vmMacAddress = "BA:DB:EE:F0:00:00";
programs.bash.interactiveShellInit = ''
echo "Hello world from inside a virtual machine!!" | ${pkgs.lolcat}/bin/lolcat
'';
};
};
}

View file

@ -1,4 +1,4 @@
{ lib, config, ... }:
{ lib, config, machine, ... }:
let cfg = config.lab.networking;
in {
imports = [ ./dmz ];
@ -89,17 +89,17 @@ in {
config = {
networking = {
domain = "hyp";
domain = if machine.type == "physical" then "hyp" else "dmz";
nftables.enable = true;
useDHCP = machine.type == "virtual";
firewall = {
enable = true;
checkReversePath = false;
};
nftables.enable = true;
useDHCP = false;
};
systemd.network = {
systemd.network = lib.mkIf (machine.type == "physical") {
enable = true;
netdevs = {
@ -153,6 +153,11 @@ in {
++ lib.lists.optional (cfg.staticDMZIPv6Address != "") cfg.staticDMZIPv6Address;
};
};
"40-vms" = {
matchConfig.Name = "vm-*";
networkConfig.Bridge = cfg.dmzBridgeName;
};
};
};
};

View file

@ -1,4 +1,4 @@
{ lib, config, ... }:
{ lib, config, machine, ... }:
let cfg = config.lab.storage;
in {
options.lab.storage = {
@ -25,7 +25,7 @@ in {
};
};
config = {
config = lib.mkIf (machine.type == "physical") {
fileSystems.${cfg.dataMountPoint}.device = cfg.dataPartition;
# TODO: Rename this to 'osDisk'. Unfortunately, we would need to run nixos-anywhere again then.

77
nixos/physical.nix Normal file
View file

@ -0,0 +1,77 @@
{ pkgs, config, lib, modulesPath, microvm, disko, agenix, machines, ... }: {
imports = [
(modulesPath + "/installer/scan/not-detected.nix")
microvm.nixosModules.host
];
config = {
boot = {
kernelModules = [ "kvm-intel" ];
extraModulePackages = [ ];
initrd = {
availableKernelModules = [
"ahci"
"xhci_pci"
"nvme"
"usbhid"
"usb_storage"
"sd_mod"
"sdhci_pci"
];
kernelModules = [ ];
};
loader = {
systemd-boot.enable = true;
efi.canTouchEfiVariables = true;
};
};
nixpkgs = {
config.allowUnfree = true;
hostPlatform = "x86_64-linux";
};
hardware.cpu.intel.updateMicrocode = config.hardware.enableRedistributableFirmware;
age.identityPaths = [ "/etc/age_ed25519" ];
virtualisation.libvirtd.enable = true;
nix = {
package = pkgs.nixFlakes;
extraOptions = ''
experimental-features = nix-command flakes
'';
};
system = {
stateVersion = "23.05";
activationScripts.diff = ''
if [[ -e /run/current-system ]]; then
${pkgs.nix}/bin/nix store diff-closures /run/current-system "$systemConfig"
fi
'';
};
microvm.vms =
let
vmsForHypervisor = lib.filterAttrs (n: v: v.type == "virtual" && v.hypervisorName == config.networking.hostName) machines;
in
builtins.mapAttrs
(name: vm:
{
# TODO Simplify?
specialArgs = { inherit agenix disko pkgs lib microvm; machine = vm; hypervisorConfig = config; };
config = {
imports = [
./.
];
};
}
)
vmsForHypervisor;
};
}

27
nixos/virtual.nix Normal file
View file

@ -0,0 +1,27 @@
{ lib, config, hypervisorConfig, ... }: {
options.lab.vmMacAddress = lib.mkOption {
type = lib.types.str;
description = ''
The MAC address of the VM's main NIC.
'';
};
config = {
system.stateVersion = hypervisorConfig.system.stateVersion;
microvm = {
shares = [{
source = "/nix/store";
mountPoint = "/nix/.ro-store";
tag = "ro-store";
proto = "virtiofs";
}];
interfaces = [{
type = "tap";
id = "vm-${config.networking.hostName}";
mac = config.lab.vmMacAddress;
}];
};
};
}