move to push backups

ansible/atlas.yml

@@ -32,9 +32,16 @@
         dest: /etc/network/interfaces.d/dmz.conf
       notify: enable interfaces
 
+    - name: Create data directory
+      file:
+        path: /data
+        state: directory
+        mode: og=rw
+
   roles:
     - {role: setup-apt, tags: setup-apt}
     - {role: postgresql, tags: postgresql}
     - {role: githubixx.ansible_role_wireguard, tags: wireguard}
-    - {role: ssh-ca, tags: ssh-ca}
+    - {role: ssh_ca, tags: ssh_ca}
-    - {role: backup, tags: backup}
+      # - {role: backup, tags: backup}
+    - {role: backupng, tags: backupng}

ansible/inventory/host_vars/atlas.yml

@@ -4,7 +4,7 @@ user_ca: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGKOClnK6/Hj8INjEgULY/lD2FM/nbiJHq
 storage_pools: [iso, disk, init]
 wireguard_addresses:
   - "10.42.0.1/32"
-wireguard_endpoint: "atlas.lan"
+wireguard_endpoint: "atlas.hyp"
 wireguard_private_key: !vault |
           $ANSIBLE_VAULT;1.1;AES256
           65666463346536363662353234666662376330396365656361636530663032366436653336383134
@@ -30,61 +30,124 @@ apt_install_packages:
   - bridge-utils
 
 ssh_ca_dir: /root/ssh_ca
-ssh_ca_user_ca_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGKOClnK6/Hj8INjEgULY/lD2FM/nbiJHqaSXtEw4+Fj User Certificate Authority for DMZ"
+ssh_ca_key_pairs:
-ssh_ca_host_ca_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAX2IhgHNxC6JTvLu9cej+iWuG+uJFMXn4AiRro9533x Host Certficate Authority for DMZ"
+  - name: dmz_user
-ssh_ca_user_ca_private_key: !vault |
+    public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGKOClnK6/Hj8INjEgULY/lD2FM/nbiJHqaSXtEw4+Fj User Certificate Authority for DMZ"
-          $ANSIBLE_VAULT;1.1;AES256
+    private_key: !vault |
-          64656264643864643263383739313232363933313662363831396262636263356435666130323063
+      $ANSIBLE_VAULT;1.1;AES256
-          3032336337663363376135643730666133623864656430390a653736313736633834623037376238
+      64656264643864643263383739313232363933313662363831396262636263356435666130323063
-          31383933626638643134613361363939633161373937656437343064346531323435633435326134
+      3032336337663363376135643730666133623864656430390a653736313736633834623037376238
-          6262626330366134360a363730373233626436343535346130613766616431383639353133356433
+      31383933626638643134613361363939633161373937656437343064346531323435633435326134
-          66383764383565343833313839646236356463636333383633633630663632356335373862663837
+      6262626330366134360a363730373233626436343535346130613766616431383639353133356433
-          65376662346662636430633634663735316336636437626263353937623630393331636633396436
+      66383764383565343833313839646236356463636333383633633630663632356335373862663837
-          34626166323836356633616333373533633634643464333837363634373337323463383332343232
+      65376662346662636430633634663735316336636437626263353937623630393331636633396436
-          65313732336639613366616632323134306162613839663962346638616333306661363631646564
+      34626166323836356633616333373533633634643464333837363634373337323463383332343232
-          66346464396465646166313862333834616664343332363065313832343762323934626366636335
+      65313732336639613366616632323134306162613839663962346638616333306661363631646564
-          31353033633130333036323534363532633063343666336630643162303932313835663430633431
+      66346464396465646166313862333834616664343332363065313832343762323934626366636335
-          31393763363730666539636538653361333531373566343331373730333137386439653830646262
+      31353033633130333036323534363532633063343666336630643162303932313835663430633431
-          66313761346162396633653564643833313930353231366661316161383330306365346538373230
+      31393763363730666539636538653361333531373566343331373730333137386439653830646262
-          33623134303762336338333064663433303963396439353834396364356465653764643335663066
+      66313761346162396633653564643833313930353231366661316161383330306365346538373230
-          39343863323539636236643933343635396639363236646337666638643333623366653030356234
+      33623134303762336338333064663433303963396439353834396364356465653764643335663066
-          32383636353364663635353133316464313664663830643936323833613765313739663938643662
+      39343863323539636236643933343635396639363236646337666638643333623366653030356234
-          36633338353830396536623230346565346163393134336230633262373133623430333962396538
+      32383636353364663635353133316464313664663830643936323833613765313739663938643662
-          33363963333138653837613130363137343366376561323733363561376530353930383431626435
+      36633338353830396536623230346565346163393134336230633262373133623430333962396538
-          62396666323562323535393564333030636462663463393364653964303162386233646634306337
+      33363963333138653837613130363137343366376561323733363561376530353930383431626435
-          33393130633537616139386561646163376531383362386538396430653761373138356363373261
+      62396666323562323535393564333030636462663463393364653964303162386233646634306337
-          37616632646135646234666633393136646664313139383566383638363635633137303039643437
+      33393130633537616139386561646163376531383362386538396430653761373138356363373261
-          61313430333064623534306539313361353033316432366266616231643234653638376466643338
+      37616632646135646234666633393136646664313139383566383638363635633137303039643437
-          30353364393939636365383861366465393031303937323234366161393938653135333731346462
+      61313430333064623534306539313361353033316432366266616231643234653638376466643338
-          33353364346464336161643232306236343338373830653462313936303264663364613032363937
+      30353364393939636365383861366465393031303937323234366161393938653135333731346462
-          32663431383763366635336339663164653938613334336230383966363936363262656165353661
+      33353364346464336161643232306236343338373830653462313936303264663364613032363937
-          66316534333735646666316364396636363738383263613864326261383061326135346638623833
+      32663431383763366635336339663164653938613334336230383966363936363262656165353661
-          3536633732663537383931363031386633623861396433303934
+      66316534333735646666316364396636363738383263613864326261383061326135346638623833
-ssh_ca_host_ca_private_key: !vault |
+      3536633732663537383931363031386633623861396433303934
-          $ANSIBLE_VAULT;1.1;AES256
+  - name: dmz_host
-          61303063616332323937323661636631316231663331333364353665393961346366356532313438
+    public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAX2IhgHNxC6JTvLu9cej+iWuG+uJFMXn4AiRro9533x Host Certficate Authority for DMZ"
-          3636353765393564383062363666353063643936653130350a343863333339353633663664613337
+    private_key: !vault |
-          36623631633036356265336165303733376634356465663534386664393533306564306438633938
+      $ANSIBLE_VAULT;1.1;AES256
-          3631366332396362360a643831336561323266666338633865313062653037373839393535346431
+      61303063616332323937323661636631316231663331333364353665393961346366356532313438
-          64363531613861306532353139336333343066343637613432643532396661313636383365626338
+      3636353765393564383062363666353063643936653130350a343863333339353633663664613337
-          30613431653162633331313439663231623363626564376534373566663932633636313666333865
+      36623631633036356265336165303733376634356465663534386664393533306564306438633938
-          33653737613966323338616431383532323666393033383034353531616533653331303533646261
+      3631366332396362360a643831336561323266666338633865313062653037373839393535346431
-          30666465386636663361373137626563643964636264613761353462633662393538653939623563
+      64363531613861306532353139336333343066343637613432643532396661313636383365626338
-          63353832623431303266376466663263316430343836396434386134633739386432323833373033
+      30613431653162633331313439663231623363626564376534373566663932633636313666333865
-          35333361343136623034313835666563376262373830623062343136313164343466396632633332
+      33653737613966323338616431383532323666393033383034353531616533653331303533646261
-          61623135333063383737643661326566613262666637373230336635306235623439623464643833
+      30666465386636663361373137626563643964636264613761353462633662393538653939623563
-          65613131303832636133343962333439313662343061336239373862346233653139616135313266
+      63353832623431303266376466663263316430343836396434386134633739386432323833373033
-          66366438363132653131306432376530343564323062343539376535373036326430613164356630
+      35333361343136623034313835666563376262373830623062343136313164343466396632633332
-          37376231383063633039393865393964396531363466383330636635323635653362633862356335
+      61623135333063383737643661326566613262666637373230336635306235623439623464643833
-          31396462383364303037626130653133363630633933306636306238373538333532656537346164
+      65613131303832636133343962333439313662343061336239373862346233653139616135313266
-          33306534666464613430356461363536623265353737653664623062643538323461633564366131
+      66366438363132653131306432376530343564323062343539376535373036326430613164356630
-          39323964656235616666383763633135653730366531646134333731653335366131313637623234
+      37376231383063633039393865393964396531363466383330636635323635653362633862356335
-          39393161313932316463316435666162396439383065643630363530376435643966326233393035
+      31396462383364303037626130653133363630633933306636306238373538333532656537346164
-          63613135623965636532346337656530316166346236386662613362653635633631623763653930
+      33306534666464613430356461363536623265353737653664623062643538323461633564366131
-          65313239333461303564366634336533376464363738323766653335633663336136373435633135
+      39323964656235616666383763633135653730366531646134333731653335366131313637623234
-          66616434363335396634323038323335326639633165613634313361343666333838363936343064
+      39393161313932316463316435666162396439383065643630363530376435643966326233393035
-          31616135333266396363393635353061333966326565386633326663663666303439373439393331
+      63613135623965636532346337656530316166346236386662613362653635633631623763653930
-          37393338306461383534343065396332633439626466303636643630656530306534623766373065
+      65313239333461303564366634336533376464363738323766653335633663336136373435633135
-          34636530353761623632656333643463306432343163343533393130313739313239333131656561
+      66616434363335396634323038323335326639633165613634313361343666333838363936343064
-          38356164353362393332333436363138346530663864343062393165343531303163376330353364
+      31616135333266396363393635353061333966326565386633326663663666303439373439393331
-          3163643637316230666666653736366432386535326334383063
+      37393338306461383534343065396332633439626466303636643630656530306534623766373065
+      34636530353761623632656333643463306432343163343533393130313739313239333131656561
+      38356164353362393332333436363138346530663864343062393165343531303163376330353364
+      3163643637316230666666653736366432386535326334383063
+  - name: hyp_user
+    public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHZreEhS/rMHfJB7IenEEfk38zCjmyce+X2AWxzU/N81 User Certificate Authority for *.hyp"
+    private_key: !vault |
+      $ANSIBLE_VAULT;1.1;AES256
+      36346139316436343632363836316538366265626564363438386266303763386663383132623134
+      3137646239613636613063323430386162323332356538650a366265316336323432656139346661
+      35383762623563313530646663633839386235396633623163396666653361663439636636316231
+      3962653536373934390a643563373836356566343938323833376164333435636139313164306338
+      65373038383462626262373965393136636439613938383130393265373831333433353238373034
+      34313565653839613831306364643231663739623236633065336131386638323431323138396631
+      33353930326562323238336564393163643338396537383665396164653531646533656130386538
+      34393332373639663037643165656566336562623732643135623164323266353030323437373130
+      61636364613439623966373939656634353737336233333836386334306339386163643263616461
+      30393966643163306462373338333432393930386464373930313932383061616532656635346466
+      61363736333130396466626330343862316163633935323062666166386362663331366337616233
+      33653636323233303263363337616465356130313835663838303038653831356432323065356461
+      66353236323434613934663233373965623433393832623235336639323734383265373439636639
+      32633139623030313866653730626666376231626561306238323437396566353831366230353535
+      63303362313666313161383566346231383061343561656338636266633763346434636436303630
+      62333238666534306130356266633134623836383234356537376134626434653830613037623835
+      34353839343032633264346331383236626230333066383734383865353234363135356562626438
+      36333530383034643864653964643333616331336661633936316161373063613237643432333130
+      38363739373738663263383133313937306237616532356166623037386236613935626332333763
+      66313630373166666336333461313437316461653930336165653238366466656164366633303438
+      39666537326561393862393562653631626133303064613363393665363633653632366264303631
+      35366162336535616137336631616334656136646433363737323430353534303535616262373965
+      37343964346435383832333630363033393536323966393466303435663234666530646661366663
+      39326631346361646632633633623236333131363062376363366339306533303136346432626338
+      35643933303334303162333163353466366634353464366635643032633762356236303564643535
+      3937313435653232336362306565346138326261393162646263
+  - name: hyp_host
+    public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzRkH3d/KVJQouswY/DMpenWbDFVOnI3Vut0xR0e1tb Host Certficate Authority for HYP"
+    private_key: !vault |
+      $ANSIBLE_VAULT;1.1;AES256
+      35316665363462323366326532363466636665353361396137313730383461306632363533396461
+      3538306465613737313466306435373162313931386263320a656136623566356330343634633764
+      61613031353536623832636466386131353932646333336530663839343138346563626534653338
+      3465336562383932350a666539383438346663613531323932383731336233333435333236343061
+      33633537623564376561646165316439376139396265666635313132353630343032356633393563
+      34333638663032643138363536373037363230636264323939643766613262363262366334653962
+      65383339373530663731363134343931353638396161396534346564366663373630316330376231
+      64646533633834356435343438613338343266653733653566646633666165353037653564636663
+      31613966313636643239373435393131303334623033303833386565616536336262646465313130
+      61373431383230313863343964386431333931643533333862313662333666333631363366346362
+      33633736363036646637646538396535323231353031323334323262643333323339663637386162
+      61333337353331346563306236636134333939356434623965303138336430636637383033363936
+      34326163353266366336343761386630396363383938333265323966316438313566663336666137
+      34393438653961323732333965623763383336646431343535613230636335613066356362623564
+      36666566363561383838343862663961343461643432303561313064613436363661356333386430
+      39393636326539373434636434396631346661346333326363623635666431393035323433633937
+      36363261306332346664663437663136363065326464373630336461326135313863636566643363
+      65323136613963643663616163396464393131653738333363393932323032623363383738356233
+      63323937386536396364333762303464376633343664306339623861633235376330616663393234
+      34393763623263373137313136613439393637633835393134626533653030616234343333643163
+      33346233306131663332333031623066396333393863376561616134373462326365393239653566
+      65336338626436636164373337383163643634396336616161373431643530373031333333613863
+      30323965613635316465616566656462636664653564346266323965633132383661663835366463
+      37383235363931346164326566323639303733313736363637666632376430383130323030373431
+      62326166306434353230363630333530633330636130323334626563353033383362623033333465
+      3236663032346630396131623633633131333632356530623230

ansible/inventory/hosts.yml

@@ -1,5 +1,5 @@
 all:
   hosts:
     atlas:
-      ansible_host: atlas.lan
+      ansible_host: atlas.hyp
       ansible_user: root

ansible/requirements.yml

@@ -1,6 +1,6 @@
 - name: setup-apt
   src: https://github.com/sunscrapers/ansible-role-apt.git
   scm: git
-- name: ssh-ca
+- name: ssh_ca
   src: https://git.pim.kunis.nl/pim/ansible-role-ssh-ca
   scm: git

ansible/roles/backup/backup_control.sh

@@ -0,0 +1,15 @@
+VIRSH="virsh --connect qemu:///system"
+read -p "" option
+case "$option" in
+	up)
+		for i in $($VIRSH list --all --name --autostart); do $VIRSH start "$i"; done
+		;;
+	down)
+		for i in $($VIRSH list --state-running --name --autostart); do
+			$VIRSH shutdown "$i"
+			until $VIRSH domstate "$i" | grep shut; do
+				sleep 0.5
+			done
+		done
+		;;
+esac

ansible/roles/backup/sshd.conf.j2

@@ -0,0 +1,14 @@
+TrustedUserCAKeys /etc/ssh/user_ca_key.pub
+
+Match User {{ backup_share_user }}
+	AuthorizedPrincipalsFile /etc/ssh/backup_principals
+	ChrootDirectory /kvm/data
+	ForceCommand internal-sftp
+	AllowTcpForwarding no
+	X11Forwarding no
+
+Match User {{ backup_control_user }}
+	AuthorizedPrincipalsFile /etc/ssh/backup_principals
+	ForceCommand /home/{{ backup_control_user }}/control.sh
+	AllowTcpForwarding no
+	X11Forwarding no

ansible/roles/backupng/files/backup.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Backup data using Borgmatic
+
+[Service]
+ExecStart=/usr/bin/borgmatic --config /root/backup.yml
+Type=oneshot

ansible/roles/backupng/files/backup.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Backup data daily
+
+[Timer]
+OnCalendar=*-*-* 3:00:00
+Persistent=true
+RandomizedDelaySec=1h
+
+[Install]
+WantedBy=timers.target

ansible/roles/backupng/files/backup.yml

@@ -0,0 +1,16 @@
+location:
+  source_directories:
+    - /data
+  repositories:
+    - 'ssh://root@lewis.hyp/mnt/kingston1TB/hosts/atlas'
+retention:
+  keep_daily: 7
+  keep_weekly: 4
+  keep_monthly: 6
+storage:
+  unknown_unencrypted_repo_access_is_ok: true
+hooks:
+  before_everything:
+    - /root/stop_vms.sh
+  after_everything:
+    - /root/start_vms.sh

ansible/roles/backupng/files/ssh_user_certificate.conf

@@ -0,0 +1,2 @@
+CertificateFile /etc/ssh/ssh_user_ed25519_key-cert.pub
+IdentityFile /etc/ssh/ssh_user_ed25519_key

ansible/roles/backupng/files/start_vms.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+for i in $(virsh list --all --name --autostart); do virsh start "$i"; done

ansible/roles/backupng/files/stop_vms.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+for i in $(virsh list --state-running --name --autostart); do
+	virsh shutdown "$i"
+	echo Stopping domain "$i"
+	until virsh domstate "$i" | grep shut; do
+		sleep 0.5
+	done
+done

ansible/roles/backupng/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: systemd daemon reload
+  systemd:
+    name: backup.timer
+    daemon_reload: true

ansible/roles/backupng/tasks/main.yml

@@ -0,0 +1,114 @@
+- name: Generate user key pair
+  openssh_keypair:
+    path: /etc/ssh/ssh_user_ed25519_key
+    type: ed25519
+    comment: "{{ ansible_fqdn }}"
+  register: user_key
+
+- name: Check whether user certificate exists
+  stat:
+    path: /etc/ssh/ssh_user_ed25519_key-cert.pub
+  register: cert_stat
+
+- name: Generate SSH user certificate
+  command:
+    cmd: "ssh -o ConnectTimeout=3 -o ConnectionAttempts=1 root@atlas.hyp '/root/ssh_ca/ssh_ca.sh user hyp_user \"{{ user_key.public_key }}\" {{ ansible_fqdn }} \"hypervisor\"'"
+  register: user_certificate
+  delegate_to: localhost
+  when: not cert_stat.stat.exists
+
+- name: Place user certificate
+  copy:
+    dest: /etc/ssh/ssh_user_ed25519_key-cert.pub
+    content: "{{ user_certificate.stdout }}"
+    mode: 0644
+  when: not cert_stat.stat.exists
+
+- name: Enable user certificate
+  copy:
+    src: "{{ role_path }}/files/ssh_user_certificate.conf"
+    dest: /etc/ssh/ssh_config.d/user_certificate.conf
+
+- name: Install Borg
+  apt:
+    pkg:
+      - borgbackup
+      - borgmatic
+
+- name: Copy Borgmatic script
+  template:
+    src: "{{ role_path }}/files/backup.yml"
+    dest: /root/backup.yml
+
+- name: Copy start_vms.sh
+  copy:
+    src: "{{ role_path }}/files/start_vms.sh"
+    dest: /root/start_vms.sh
+    mode: preserve
+
+- name: Copy stop_vms.sh
+  copy:
+    src: "{{ role_path }}/files/stop_vms.sh"
+    dest: /root/stop_vms.sh
+    mode: preserve
+
+- name: Copy systemd backup unit
+  copy:
+    src: "{{ role_path }}/files/backup.service"
+    dest: /etc/systemd/system/backup.service
+  notify: systemd daemon reload
+
+- name: Copy systemd backup timer
+  copy:
+    src: "{{ role_path }}/files/backup.timer"
+    dest: /etc/systemd/system/backup.timer
+  notify: systemd daemon reload
+
+- name: Enable backup timer
+  systemd:
+    name: backup.timer
+    enabled: true
+    state: started
+
+- name: Add SSH host CA known host
+  known_hosts:
+    key: '@cert-authority *.dmz ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAX2IhgHNxC6JTvLu9cej+iWuG+uJFMXn4AiRro9533x Host Certficate Authority for DMZ'
+    name: '@cert-authority *.dmz'
+    path: /etc/ssh/ssh_known_hosts
+
+- name: Add SSH host CA known host
+  known_hosts:
+    key: '@cert-authority *.hyp ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFzRkH3d/KVJQouswY/DMpenWbDFVOnI3Vut0xR0e1tb Host Certficate Authority for HYP'
+    name: '@cert-authority *.hyp'
+    path: /etc/ssh/ssh_known_hosts
+
+- name: Generate host key pair
+  openssh_keypair:
+    path: /etc/ssh/ssh_host_ed25519_key
+    type: ed25519
+    comment: "{{ ansible_host }}"
+  register: host_key
+
+- name: Check whether host certificate exists
+  stat:
+    path: /etc/ssh/ssh_host_ed25519_key-cert.pub
+  register: cert_stat
+
+- name: Generate SSH host certificate
+  command:
+    cmd: "ssh -o ConnectTimeout=3 -o ConnectionAttempts=1 root@atlas.hyp '/root/ssh_ca/ssh_ca.sh host hyp_host \"{{ host_key.public_key }}\" {{ ansible_host }}'"
+  register: host_certificate
+  delegate_to: localhost
+  when: not cert_stat.stat.exists
+
+- name: Place host certificate
+  copy:
+    dest: /etc/ssh/ssh_host_ed25519_key-cert.pub
+    content: "{{ host_certificate.stdout }}"
+    mode: 0644
+  when: not cert_stat.stat.exists
+
+- name: Enable host certificate
+  copy:
+    dest: /etc/ssh/sshd_config.d/sshd_host_certificate.conf
+    content: "HostCertificate /etc/ssh/ssh_host_ed25519_key-cert.pub"