nixos-servers/nixos/modules/backups.nix

{ pkgs, lib, config, ... }:
let
  cfg = config.lab.backups;
  beforeEverything = pkgs.writeShellScriptBin "beforeEverything" ''
    if [ -d "${cfg.snapshotLocation}" ]; then
      ${pkgs.btrfs-progs}/bin/btrfs subvolume delete ${cfg.snapshotLocation}
    fi

    ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r ${cfg.subvolumeLocation} ${cfg.snapshotLocation}
  '';

  borgmaticConfig = pkgs.writeTextFile {
    name = "borgmatic-config";
    text = ''
      source_directories:
      - ${cfg.snapshotLocation}
      repositories:
      - path: ${cfg.repoLocation}
        label: nfs
      - path: ssh://admin@ec2-3-254-121-39.eu-west-1.compute.amazonaws.com/mnt/data/nfs.borg
        label: ec2
      ssh_command: "${pkgs.openssh}/bin/ssh -i ${config.age.secrets."ec2_borg_server.pem".path} -o StrictHostKeychecking=no -o ConnectTimeout=10 -o ConnectionAttempts=3"
      keep_daily: 7
      keep_weekly: 4
      keep_monthly: 6
      encryption_passcommand: "${pkgs.coreutils}/bin/cat ''${BORG_PASSPHRASE_FILE}"
      before_everything:
      - ${beforeEverything}/bin/beforeEverything
      postgresql_databases:
      - name: nextcloud
        hostname: lewis.dmz
        username: nextcloud
        password: ''${NEXTCLOUD_DATABASE_PASSWORD}
        format: tar
      - name: hedgedoc
        hostname: lewis.dmz
        username: hedgedoc
        password: ''${HEDGEDOC_DATABASE_PASSWORD}
        format: tar
      - name: paperless
        hostname: lewis.dmz
        username: paperless
        password: ''${PAPERLESS_DATABASE_PASSWORD}
        format: tar
    '';
  };
in
{
  options.lab.backups = {
    enable = lib.mkOption {
      default = false;
      type = lib.types.bool;
      description = ''
        Whether to enable backups of persistent data on this machine.
      '';
    };

    repoLocation = lib.mkOption {
      default = "${config.lab.storage.dataMountPoint}/backups/nfs.borg";
      type = lib.types.str;
      description = ''
        Location of the Borg repository to back up to.
      '';
    };

    subvolumeLocation = lib.mkOption {
      default = "${config.lab.storage.dataMountPoint}/nfs";
      type = lib.types.str;
      description = ''
        Location of the btrfs subvolume holding the data.
      '';
    };

    snapshotLocation = lib.mkOption {
      default = "${config.lab.storage.dataMountPoint}/snapshot-nfs";
      type = lib.types.str;
      description = ''
        Location to (temporary) create a snapshot of the subvolume.
      '';
    };
  };

  config = lib.mkIf cfg.enable {
    environment.systemPackages = with pkgs; [ borgbackup postgresql ];
    # Converted from:
    # https://github.com/borgmatic-collective/borgmatic/tree/84823dfb912db650936e3492f6ead7e0e0d32a0f/sample/systemd
    systemd.services.borgmatic = {
      description = "borgmatic backup";
      wants = [ "network-online.target" ];
      after = [ "network-online.target" ];
      unitConfig.ConditionACPower = true;
      preStart = "${pkgs.coreutils}/bin/sleep 10s";
      path = with pkgs; [ postgresql ];

      serviceConfig = {
        Type = "oneshot";
        Nice = 19;
        CPUSchedulingPolicy = "batch";
        IOSchedulingClass = "best-effort";
        IOSchedulingPriority = 7;
        IOWeight = 100;
        Restart = "no";
        LogRateLimitIntervalSec = 0;
        EnvironmentFile = config.age.secrets."database_passwords.env".path;
        Environment = "BORG_PASSPHRASE_FILE=${config.age.secrets."borg_passphrase".path}";
      };

      script = "${pkgs.systemd}/bin/systemd-inhibit --who=\"borgmatic\" --what=\"sleep:shutdown\" --why=\"Prevent interrupting scheduled backup\" ${pkgs.borgmatic}/bin/borgmatic --verbosity -2 --syslog-verbosity 1 -c ${borgmaticConfig}";
    };

    systemd.timers.borgmatic = {
      description = "Run borgmatic backup";
      wantedBy = [ "timers.target" ];
      timerConfig = {
        OnCalendar = "*-*-* 3:00:00";
        Persistent = true;
        RandomizedDelaySec = "3h";
      };
    };

    age.secrets = {
      "database_passwords.env".file = ../secrets/database_passwords.env.age;
      "borg_passphrase".file = ../secrets/borg_passphrase.age;
      "ec2_borg_server.pem".file = ../secrets/ec2_borg_server.pem.age;
    };
  };
}