move to push backups

ansible/atlas.yml

@@ -32,9 +32,16 @@
         dest: /etc/network/interfaces.d/dmz.conf
       notify: enable interfaces
 
+    - name: Create data directory
+      file:
+        path: /data
+        state: directory
+        mode: og=rw
+
   roles:
     - {role: setup-apt, tags: setup-apt}
     - {role: postgresql, tags: postgresql}
     - {role: githubixx.ansible_role_wireguard, tags: wireguard}
-    - {role: ssh-ca, tags: ssh-ca}
+    - {role: ssh_ca, tags: ssh_ca}
-    - {role: backup, tags: backup}
+      # - {role: backup, tags: backup}
+    - {role: backupng, tags: backupng}

ansible/inventory/host_vars/atlas.yml

@@ -4,7 +4,7 @@ user_ca: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGKOClnK6/Hj8INjEgULY/lD2FM/nbiJHq
 storage_pools: [iso, disk, init]
 wireguard_addresses:
   - "10.42.0.1/32"
-wireguard_endpoint: "atlas.lan"
+wireguard_endpoint: "atlas.hyp"
 wireguard_private_key: !vault |
           $ANSIBLE_VAULT;1.1;AES256
           65666463346536363662353234666662376330396365656361636530663032366436653336383134
@@ -30,9 +30,10 @@ apt_install_packages:
   - bridge-utils
 
 ssh_ca_dir: /root/ssh_ca
-ssh_ca_user_ca_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGKOClnK6/Hj8INjEgULY/lD2FM/nbiJHqaSXtEw4+Fj User Certificate Authority for DMZ"
+ssh_ca_key_pairs:
-ssh_ca_host_ca_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAX2IhgHNxC6JTvLu9cej+iWuG+uJFMXn4AiRro9533x Host Certficate Authority for DMZ"
+  - name: dmz_user
-ssh_ca_user_ca_private_key: !vault |
+    public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGKOClnK6/Hj8INjEgULY/lD2FM/nbiJHqaSXtEw4+Fj User Certificate Authority for DMZ"
+    private_key: !vault |
       $ANSIBLE_VAULT;1.1;AES256
       64656264643864643263383739313232363933313662363831396262636263356435666130323063
       3032336337663363376135643730666133623864656430390a653736313736633834623037376238
@@ -60,7 +61,9 @@ ssh_ca_user_ca_private_key: !vault |
       32663431383763366635336339663164653938613334336230383966363936363262656165353661
       66316534333735646666316364396636363738383263613864326261383061326135346638623833
       3536633732663537383931363031386633623861396433303934
-ssh_ca_host_ca_private_key: !vault |
+  - name: dmz_host
+    public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAX2IhgHNxC6JTvLu9cej+iWuG+uJFMXn4AiRro9533x Host Certficate Authority for DMZ"
+    private_key: !vault |
       $ANSIBLE_VAULT;1.1;AES256
       61303063616332323937323661636631316231663331333364353665393961346366356532313438
       3636353765393564383062363666353063643936653130350a343863333339353633663664613337
@@ -88,3 +91,63 @@ ssh_ca_host_ca_private_key: !vault |
       34636530353761623632656333643463306432343163343533393130313739313239333131656561
       38356164353362393332333436363138346530663864343062393165343531303163376330353364
       3163643637316230666666653736366432386535326334383063
+  - name: hyp_user
+    public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHZreEhS/rMHfJB7IenEEfk38zCjmyce+X2AWxzU/N81 User Certificate Authority for *.hyp"
+    private_key: !vault |
+      $ANSIBLE_VAULT;1.1;AES256
+      36346139316436343632363836316538366265626564363438386266303763386663383132623134
+      3137646239613636613063323430386162323332356538650a366265316336323432656139346661
+      35383762623563313530646663633839386235396633623163396666653361663439636636316231
+      3962653536373934390a643563373836356566343938323833376164333435636139313164306338
+      65373038383462626262373965393136636439613938383130393265373831333433353238373034
+      34313565653839613831306364643231663739623236633065336131386638323431323138396631
+      33353930326562323238336564393163643338396537383665396164653531646533656130386538
+      34393332373639663037643165656566336562623732643135623164323266353030323437373130
+      61636364613439623966373939656634353737336233333836386334306339386163643263616461
+      30393966643163306462373338333432393930386464373930313932383061616532656635346466
+      61363736333130396466626330343862316163633935323062666166386362663331366337616233
+      33653636323233303263363337616465356130313835663838303038653831356432323065356461
+      66353236323434613934663233373965623433393832623235336639323734383265373439636639
+      32633139623030313866653730626666376231626561306238323437396566353831366230353535
+      63303362313666313161383566346231383061343561656338636266633763346434636436303630
+      62333238666534306130356266633134623836383234356537376134626434653830613037623835
+      34353839343032633264346331383236626230333066383734383865353234363135356562626438
+      36333530383034643864653964643333616331336661633936316161373063613237643432333130
+      38363739373738663263383133313937306237616532356166623037386236613935626332333763
+      66313630373166666336333461313437316461653930336165653238366466656164366633303438
+      39666537326561393862393562653631626133303064613363393665363633653632366264303631
+      35366162336535616137336631616334656136646433363737323430353534303535616262373965
+      37343964346435383832333630363033393536323966393466303435663234666530646661366663
+      39326631346361646632633633623236333131363062376363366339306533303136346432626338
+      35643933303334303162333163353466366634353464366635643032633762356236303564643535
+      3937313435653232336362306565346138326261393162646263
+  - name: hyp_host
+    public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzRkH3d/KVJQouswY/DMpenWbDFVOnI3Vut0xR0e1tb Host Certficate Authority for HYP"
+    private_key: !vault |
+      $ANSIBLE_VAULT;1.1;AES256
+      35316665363462323366326532363466636665353361396137313730383461306632363533396461
+      3538306465613737313466306435373162313931386263320a656136623566356330343634633764
+      61613031353536623832636466386131353932646333336530663839343138346563626534653338
+      3465336562383932350a666539383438346663613531323932383731336233333435333236343061
+      33633537623564376561646165316439376139396265666635313132353630343032356633393563
+      34333638663032643138363536373037363230636264323939643766613262363262366334653962
+      65383339373530663731363134343931353638396161396534346564366663373630316330376231
+      64646533633834356435343438613338343266653733653566646633666165353037653564636663
+      31613966313636643239373435393131303334623033303833386565616536336262646465313130
+      61373431383230313863343964386431333931643533333862313662333666333631363366346362
+      33633736363036646637646538396535323231353031323334323262643333323339663637386162
+      61333337353331346563306236636134333939356434623965303138336430636637383033363936
+      34326163353266366336343761386630396363383938333265323966316438313566663336666137
+      34393438653961323732333965623763383336646431343535613230636335613066356362623564
+      36666566363561383838343862663961343461643432303561313064613436363661356333386430
+      39393636326539373434636434396631346661346333326363623635666431393035323433633937
+      36363261306332346664663437663136363065326464373630336461326135313863636566643363
+      65323136613963643663616163396464393131653738333363393932323032623363383738356233
+      63323937386536396364333762303464376633343664306339623861633235376330616663393234
+      34393763623263373137313136613439393637633835393134626533653030616234343333643163
+      33346233306131663332333031623066396333393863376561616134373462326365393239653566
+      65336338626436636164373337383163643634396336616161373431643530373031333333613863
+      30323965613635316465616566656462636664653564346266323965633132383661663835366463
+      37383235363931346164326566323639303733313736363637666632376430383130323030373431
+      62326166306434353230363630333530633330636130323334626563353033383362623033333465
+      3236663032346630396131623633633131333632356530623230

ansible/inventory/hosts.yml

@@ -1,5 +1,5 @@
 all:
   hosts:
     atlas:
-      ansible_host: atlas.lan
+      ansible_host: atlas.hyp
       ansible_user: root

ansible/requirements.yml

@@ -1,6 +1,6 @@
 - name: setup-apt
   src: https://github.com/sunscrapers/ansible-role-apt.git
   scm: git
-- name: ssh-ca
+- name: ssh_ca
   src: https://git.pim.kunis.nl/pim/ansible-role-ssh-ca
   scm: git

ansible/roles/backup/backup_control.sh

@@ -0,0 +1,15 @@
+VIRSH="virsh --connect qemu:///system"
+read -p "" option
+case "$option" in
+	up)
+		for i in $($VIRSH list --all --name --autostart); do $VIRSH start "$i"; done
+		;;
+	down)
+		for i in $($VIRSH list --state-running --name --autostart); do
+			$VIRSH shutdown "$i"
+			until $VIRSH domstate "$i" | grep shut; do
+				sleep 0.5
+			done
+		done
+		;;
+esac

ansible/roles/backup/sshd.conf.j2

@@ -0,0 +1,14 @@
+TrustedUserCAKeys /etc/ssh/user_ca_key.pub
+
+Match User {{ backup_share_user }}
+	AuthorizedPrincipalsFile /etc/ssh/backup_principals
+	ChrootDirectory /kvm/data
+	ForceCommand internal-sftp
+	AllowTcpForwarding no
+	X11Forwarding no
+
+Match User {{ backup_control_user }}
+	AuthorizedPrincipalsFile /etc/ssh/backup_principals
+	ForceCommand /home/{{ backup_control_user }}/control.sh
+	AllowTcpForwarding no
+	X11Forwarding no

ansible/roles/backupng/files/backup.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Backup data using Borgmatic
+
+[Service]
+ExecStart=/usr/bin/borgmatic --config /root/backup.yml
+Type=oneshot

ansible/roles/backupng/files/backup.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Backup data daily
+
+[Timer]
+OnCalendar=*-*-* 3:00:00
+Persistent=true
+RandomizedDelaySec=1h
+
+[Install]
+WantedBy=timers.target

ansible/roles/backupng/files/backup.yml

@@ -0,0 +1,16 @@
+location:
+  source_directories:
+    - /data
+  repositories:
+    - 'ssh://root@lewis.hyp/mnt/kingston1TB/hosts/atlas'
+retention:
+  keep_daily: 7
+  keep_weekly: 4
+  keep_monthly: 6
+storage:
+  unknown_unencrypted_repo_access_is_ok: true
+hooks:
+  before_everything:
+    - /root/stop_vms.sh
+  after_everything:
+    - /root/start_vms.sh

ansible/roles/backupng/files/ssh_user_certificate.conf

@@ -0,0 +1,2 @@
+CertificateFile /etc/ssh/ssh_user_ed25519_key-cert.pub
+IdentityFile /etc/ssh/ssh_user_ed25519_key

ansible/roles/backupng/files/start_vms.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+for i in $(virsh list --all --name --autostart); do virsh start "$i"; done

ansible/roles/backupng/files/stop_vms.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+for i in $(virsh list --state-running --name --autostart); do
+	virsh shutdown "$i"
+	echo Stopping domain "$i"
+	until virsh domstate "$i" | grep shut; do
+		sleep 0.5
+	done
+done

ansible/roles/backupng/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: systemd daemon reload
+  systemd:
+    name: backup.timer
+    daemon_reload: true

ansible/roles/backupng/tasks/main.yml

@@ -0,0 +1,114 @@
+- name: Generate user key pair
+  openssh_keypair:
+    path: /etc/ssh/ssh_user_ed25519_key
+    type: ed25519
+    comment: "{{ ansible_fqdn }}"
+  register: user_key
+
+- name: Check whether user certificate exists
+  stat:
+    path: /etc/ssh/ssh_user_ed25519_key-cert.pub
+  register: cert_stat
+
+- name: Generate SSH user certificate
+  command:
+    cmd: "ssh -o ConnectTimeout=3 -o ConnectionAttempts=1 root@atlas.hyp '/root/ssh_ca/ssh_ca.sh user hyp_user \"{{ user_key.public_key }}\" {{ ansible_fqdn }} \"hypervisor\"'"
+  register: user_certificate
+  delegate_to: localhost
+  when: not cert_stat.stat.exists
+
+- name: Place user certificate
+  copy:
+    dest: /etc/ssh/ssh_user_ed25519_key-cert.pub
+    content: "{{ user_certificate.stdout }}"
+    mode: 0644
+  when: not cert_stat.stat.exists
+
+- name: Enable user certificate
+  copy:
+    src: "{{ role_path }}/files/ssh_user_certificate.conf"
+    dest: /etc/ssh/ssh_config.d/user_certificate.conf
+
+- name: Install Borg
+  apt:
+    pkg:
+      - borgbackup
+      - borgmatic
+
+- name: Copy Borgmatic script
+  template:
+    src: "{{ role_path }}/files/backup.yml"
+    dest: /root/backup.yml
+
+- name: Copy start_vms.sh
+  copy:
+    src: "{{ role_path }}/files/start_vms.sh"
+    dest: /root/start_vms.sh
+    mode: preserve
+
+- name: Copy stop_vms.sh
+  copy:
+    src: "{{ role_path }}/files/stop_vms.sh"
+    dest: /root/stop_vms.sh
+    mode: preserve
+
+- name: Copy systemd backup unit
+  copy:
+    src: "{{ role_path }}/files/backup.service"
+    dest: /etc/systemd/system/backup.service
+  notify: systemd daemon reload
+
+- name: Copy systemd backup timer
+  copy:
+    src: "{{ role_path }}/files/backup.timer"
+    dest: /etc/systemd/system/backup.timer
+  notify: systemd daemon reload
+
+- name: Enable backup timer
+  systemd:
+    name: backup.timer
+    enabled: true
+    state: started
+
+- name: Add SSH host CA known host
+  known_hosts:
+    key: '@cert-authority *.dmz ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAX2IhgHNxC6JTvLu9cej+iWuG+uJFMXn4AiRro9533x Host Certficate Authority for DMZ'
+    name: '@cert-authority *.dmz'
+    path: /etc/ssh/ssh_known_hosts
+
+- name: Add SSH host CA known host
+  known_hosts:
+    key: '@cert-authority *.hyp ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzRkH3d/KVJQouswY/DMpenWbDFVOnI3Vut0xR0e1tb Host Certficate Authority for HYP'
+    name: '@cert-authority *.hyp'
+    path: /etc/ssh/ssh_known_hosts
+
+- name: Generate host key pair
+  openssh_keypair:
+    path: /etc/ssh/ssh_host_ed25519_key
+    type: ed25519
+    comment: "{{ ansible_host }}"
+  register: host_key
+
+- name: Check whether host certificate exists
+  stat:
+    path: /etc/ssh/ssh_host_ed25519_key-cert.pub
+  register: cert_stat
+
+- name: Generate SSH host certificate
+  command:
+    cmd: "ssh -o ConnectTimeout=3 -o ConnectionAttempts=1 root@atlas.hyp '/root/ssh_ca/ssh_ca.sh host hyp_host \"{{ host_key.public_key }}\" {{ ansible_host }}'"
+  register: host_certificate
+  delegate_to: localhost
+  when: not cert_stat.stat.exists
+
+- name: Place host certificate
+  copy:
+    dest: /etc/ssh/ssh_host_ed25519_key-cert.pub
+    content: "{{ host_certificate.stdout }}"
+    mode: 0644
+  when: not cert_stat.stat.exists
+
+- name: Enable host certificate
+  copy:
+    dest: /etc/ssh/sshd_config.d/sshd_host_certificate.conf
+    content: "HostCertificate /etc/ssh/ssh_host_ed25519_key-cert.pub"