home/nixos-servers
home/nixos-servers
2
0
Fork 0
Code Issues 8 Pull requests Projects Releases Packages Wiki Activity Actions

remove docker swarm

Browse source
This commit is contained in:
Pim Kunis 2024-04-09 20:18:00 +02:00
parent ef7b1bd189
commit 0c65530d7c
21 changed files with 0 additions and 558 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
docker_swarm/.envrc
View file

@ -1 +0,0 @@
use_flake

32
docker_swarm/README.md
View file

@ -1,32 +0,0 @@
# Docker Swarm
On each of our machines, we deploy a virtual machine that participates in a Docker Swarm.
However, only one VM is a manager (`maestro`) while two are workers (`bancomart` and `vpay`).
This lack of redundancy in the cluster is deliberate: in case all nodes are down (e.g. misconfiguration or power outage) manual action would need to be taken in order to restore the cluster.
In case of only one manager node, the cluster is always able to restore itself automatically.
While the operating system of the VMs is managed by NixOS, cluster creation and the deployment of workloads is done through Ansible.
In my opinion, Ansible is a perfect fit for environments that tend to change a lot (such as a container cluster).
## Stacks
On top of the Docker Swarm, we host several services deployed as Docker Stacks:
- [Nextcloud](https://nextcloud.com/)
- [Paperless-ngx](https://github.com/paperless-ngx/paperless-ngx)
- [Syncthing](https://syncthing.net/)
- [Pi-hole](https://pi-hole.net/)
- [Radicale](https://github.com/Kozea/Radicale)
- [FreshRSS](https://www.freshrss.org/)
- [Traefik](https://traefik.io/traefik/)
- [Forgejo](https://forgejo.org/)
- [KitchenOwl](https://kitchenowl.org/)
- [kms](https://hub.docker.com/r/teddysun/kms/)
- [Inbucket](https://inbucket.org/)
- [CyberChef](https://github.com/gchq/CyberChef)
- [HedgeDoc](https://hedgedoc.org/)
- [Swarm Dashboard](https://github.com/mohsenasm/swarm-dashboard)
## Secret decryption
The Ansible playbooks assume you have the password to Ansible vault present at `~/.config/home/ansible-vault-secret`.

9
docker_swarm/ansible.cfg
View file

@ -1,9 +0,0 @@
[defaults]
roles_path=~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles:roles
inventory=inventory
interpreter_python=/run/current-system/sw/bin/python3.11
remote_user = root
vault_password_file=$HOME/.config/home/ansible-vault-secret
[diff]
always = True

61
docker_swarm/flake.lock
View file

@ -1,61 +0,0 @@
{
"nodes": {
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1709126324,
"narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "d465f4819400de7c8d874d50b982301f28a84605",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1709309926,
"narHash": "sha256-VZFBtXGVD9LWTecGi6eXrE0hJ/mVB3zGUlHImUs2Qak=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "79baff8812a0d68e24a836df0a364c678089e2c7",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-23.11",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

16
docker_swarm/flake.nix
View file

@ -1,16 +0,0 @@
{
description = "Ansible development shell";
inputs = {
nixpkgs.url = "github:nixos/nixpkgs/nixos-23.11";
flake-utils.url = "github:numtide/flake-utils";
};
outputs = { self, nixpkgs, flake-utils }: flake-utils.lib.eachDefaultSystem (system:
let pkgs = import nixpkgs { inherit system; }; in
{
devShells.default = pkgs.mkShell {
packages = [ pkgs.ansible ];
};
});
}

19
docker_swarm/inventory/group_vars/all.yml
View file

@ -1,19 +0,0 @@
git_ssh_port: 56287
database_passwords:
nextcloud: !vault |
$ANSIBLE_VAULT;1.1;AES256
66326230303135303930363761316534313439383365376231623661316635393839336431313262
3832626365376533646561653863316364313135343366330a356136343938666133356532613263
39663037623232363266376335643834353735363431636535386566643763386463353962663930
3466343563353162320a376437353933656166323364323166376663323531373338656563653463
33346263626430616164613937363836343430383233393061643231346661656539623938333631
3632373964346139316637663364646132636636373461613534
hedgedoc: !vault |
$ANSIBLE_VAULT;1.1;AES256
63363464666633663762393135333362613966636338623533393132376338343339653431396465
6634643863623163366235393434343662313735363438610a373065363361326565633766633835
38383637343230363031636634623930666365333739323162313937656239646166613738393965
3533666462303563360a313233306335396234393932396331313238376464363964363839396164
66366662356135343035363935616664613831626131376330643133313530636431613266636165
6265613666616164373637356235396165383662333561393939

9
docker_swarm/inventory/hosts.yml
View file

@ -1,9 +0,0 @@
all:
hosts:
manager:
ansible_host: maestro.dmz
children:
workers:
hosts:
vpay:
ansible_host: vpay.dmz

9
docker_swarm/playbooks/remove_stack.yml
View file

@ -1,9 +0,0 @@
---
- name: Remove a Docker swarm stack
hosts: manager
tasks:
- name: Remove the stack
docker_stack:
name: "{{ stack }}"
state: absent

23
docker_swarm/playbooks/setup.yml
View file

@ -1,23 +0,0 @@
---
- name: Setup Docker Swarm manager
hosts: manager
tasks:
- name: Create Docker Swarm
docker_swarm:
- name: Get Docker Swarm manager info
docker_swarm_info:
nodes: yes
nodes_filters:
name: manager
register: swarm_info
- hosts: workers
tasks:
- name: Join Docker Swarm
docker_swarm:
state: join
join_token: "{{ hostvars.manager.swarm_info.swarm_facts.JoinTokens.Worker }}"
remote_addrs:
- "{{ hostvars.manager.ansible_default_ipv4.address }}"

6
docker_swarm/playbooks/stacks.yml
View file

@ -1,6 +0,0 @@
---
- name: Start Docker stacks
hosts: manager
roles:
- {role: traefik, tags: traefik}
- {role: swarm_dashboard, tags: swarm_dashboard}

44
docker_swarm/roles/hedgedoc/docker-stack.yml.j2
View file

@ -1,44 +0,0 @@
# vi: ft=yaml
version: '3'
networks:
traefik:
external: true
volumes:
uploads:
driver_opts:
type: "nfs"
o: "addr=lewis.dmz,nolock,soft,rw"
device: ":/mnt/data/nfs/hedgedoc/uploads"
services:
hedgedoc:
image: quay.io/hedgedoc/hedgedoc:1.9.7
environment:
- CMD_DB_URL=postgres://hedgedoc:{{ database_passwords.hedgedoc }}@lewis.dmz:5432/hedgedoc
- CMD_DOMAIN=md.kun.is
- CMD_PORT=3000
- CMD_URL_ADDPORT=false
- CMD_ALLOW_ANONYMOUS=true
- CMD_ALLOW_EMAIL_REGISTER=false
- CMD_PROTOCOL_USESSL=true
- CMD_SESSION_SECRET={{ session_secret }}
volumes:
- type: volume
source: uploads
target: /hedgedoc/public/uploads
volume:
nocopy: true
networks:
- traefik
deploy:
labels:
- traefik.enable=true
- traefik.http.routers.hedgedoc.entrypoints=websecure
- traefik.http.routers.hedgedoc.rule=Host(`md.kun.is`)
- traefik.http.routers.hedgedoc.tls=true
- traefik.http.routers.hedgedoc.tls.certresolver=letsencrypt
- traefik.http.routers.hedgedoc.service=hedgedoc
- traefik.http.services.hedgedoc.loadbalancer.server.port=3000
- traefik.docker.network=traefik

5
docker_swarm/roles/hedgedoc/tasks/main.yml
View file

@ -1,5 +0,0 @@
- name: Deploy Docker stack
docker_stack:
name: hedgedoc
compose:
- "{{ lookup('template', '{{ role_path }}/docker-stack.yml.j2') | from_yaml }}"

10
docker_swarm/roles/hedgedoc/vars/main.yml
View file

@ -1,10 +0,0 @@
session_secret: !vault |
$ANSIBLE_VAULT;1.1;AES256
30633835386265643561343033326536653166343630396139303137613138383233666565666330
3032613865333836656566626435383165396539323837350a376331306464643766373839386638
65653865343539633636323833343964636332636461386434386432306230343833343431363134
6563373138626637650a633932313862326231666330343662343765666166373961376237396434
33396131353830323063326266623862353731653665626466653335656434303033353333353164
61613535373037646565386131383631366338616565373261396136616433393462313537313861
35313661616365373231373963323865393635626132343138363230313431636333363130346239
32656335333635613736

31
docker_swarm/roles/swarm_dashboard/docker-stack.yml.j2
View file

@ -1,31 +0,0 @@
# vi: ft=yaml
version: "3"
networks:
traefik:
external: true
services:
swarm-dashboard:
image: charypar/swarm-dashboard
volumes:
- type: bind
source: /var/run/docker.sock
target: /var/run/docker.sock
environment:
PORT: 80
networks:
- traefik
deploy:
placement:
constraints:
- node.role == manager
labels:
- traefik.enable=true
- traefik.http.routers.swarm-dashboard.entrypoints=localsecure
- traefik.http.routers.swarm-dashboard.rule=Host(`swarm.kun.is`)
- traefik.http.routers.swarm-dashboard.tls=true
- traefik.http.routers.swarm-dashboard.tls.certresolver=letsencrypt
- traefik.http.routers.swarm-dashboard.service=swarm-dashboard
- traefik.http.services.swarm-dashboard.loadbalancer.server.port=80
- traefik.docker.network=traefik

5
docker_swarm/roles/swarm_dashboard/tasks/main.yml
View file

@ -1,5 +0,0 @@
- name: Deploy Docker stack
docker_stack:
name: swarm_dashboard
compose:
- "{{ lookup('template', '{{ role_path }}/docker-stack.yml.j2') | from_yaml }}"

213
docker_swarm/roles/traefik/docker-stack.yml.j2
View file

@ -1,213 +0,0 @@
# vi: ft=yaml
version: "3.7"
networks:
traefik:
external: true
configs:
services:
external: true
name: "{{ services.config_name }}"
volumes:
acme:
driver_opts:
type: "nfs"
o: "addr=lewis.dmz,nolock,soft,rw"
device: ":/mnt/data/nfs/traefik/acme"
services:
traefik:
image: traefik:3.0.0-beta2
networks:
- traefik
ports:
- mode: host
protocol: tcp
published: 443
target: 443
- mode: host
protocol: tcp
published: 80
target: 80
- mode: host
protocol: tcp
published: 444
target: 444
deploy:
placement:
constraints:
- node.role == manager
labels:
- traefik.enable=true
- traefik.http.routers.dashboard.entrypoints=localsecure
- traefik.http.routers.dashboard.rule=Host(`traefik.kun.is`)
- traefik.http.routers.dashboard.service=api@internal
- traefik.http.services.dashboard.loadbalancer.server.port=8080
- traefik.http.routers.dashboard.tls=true
- traefik.http.routers.dashboard.tls.certresolver=letsencrypt
- traefik.docker.network=traefik
- traefik.http.routers.esrom.entrypoints=websecure
- traefik.http.routers.esrom.service=esrom@file
- traefik.http.routers.esrom.rule=Host(`esrom.kun.is`)
- traefik.http.routers.esrom.tls=true
- traefik.http.routers.esrom.tls.certresolver=letsencrypt
- traefik.http.routers.cyberchef.entrypoints=websecure
- traefik.http.routers.cyberchef.service=k3s@file
- traefik.http.routers.cyberchef.rule=Host(`cyberchef.kun.is`)
- traefik.http.routers.cyberchef.tls=true
- traefik.http.routers.cyberchef.tls.certresolver=letsencrypt
- traefik.http.routers.freshrss.entrypoints=websecure
- traefik.http.routers.freshrss.service=k3s@file
- traefik.http.routers.freshrss.rule=Host(`rss.kun.is`)
- traefik.http.routers.freshrss.tls=true
- traefik.http.routers.freshrss.tls.certresolver=letsencrypt
- traefik.http.routers.inbucket.entrypoints=localsecure
- traefik.http.routers.inbucket.service=k3s@file
- traefik.http.routers.inbucket.rule=Host(`inbucket.kun.is`)
- traefik.http.routers.inbucket.tls=true
- traefik.http.routers.inbucket.tls.certresolver=letsencrypt
- traefik.http.routers.radicale.entrypoints=websecure
- traefik.http.routers.radicale.service=k3s@file
- traefik.http.routers.radicale.rule=Host(`dav.kun.is`)
- traefik.http.routers.radicale.tls=true
- traefik.http.routers.radicale.tls.certresolver=letsencrypt
- traefik.http.routers.syncthing.entrypoints=localsecure
- traefik.http.routers.syncthing.service=k3s@file
- traefik.http.routers.syncthing.rule=Host(`sync.kun.is`)
- traefik.http.routers.syncthing.tls=true
- traefik.http.routers.syncthing.tls.certresolver=letsencrypt
- traefik.http.routers.pihole.entrypoints=localsecure
- traefik.http.routers.pihole.service=k3s@file
- traefik.http.routers.pihole.rule=Host(`pihole.kun.is`)
- traefik.http.routers.pihole.tls=true
- traefik.http.routers.pihole.tls.certresolver=letsencrypt
- traefik.http.routers.hedgedoc.entrypoints=websecure
- traefik.http.routers.hedgedoc.service=k3s@file
- traefik.http.routers.hedgedoc.rule=Host(`md.kun.is`)
- traefik.http.routers.hedgedoc.tls=true
- traefik.http.routers.hedgedoc.tls.certresolver=letsencrypt
- traefik.http.routers.nextcloud.entrypoints=websecure
- traefik.http.routers.nextcloud.service=k3s@file
- traefik.http.routers.nextcloud.rule=Host(`cloud.kun.is`)
- traefik.http.routers.nextcloud.tls=true
- traefik.http.routers.nextcloud.tls.certresolver=letsencrypt
- traefik.http.routers.paperless-ngx.entrypoints=websecure
- traefik.http.routers.paperless-ngx.service=k3s@file
- traefik.http.routers.paperless-ngx.rule=Host(`paperless.kun.is`)
- traefik.http.routers.paperless-ngx.tls=true
- traefik.http.routers.paperless-ngx.tls.certresolver=letsencrypt
- traefik.http.routers.kitchenowl.entrypoints=websecure
- traefik.http.routers.kitchenowl.service=k3s@file
- traefik.http.routers.kitchenowl.rule=Host(`boodschappen.kun.is`)
- traefik.http.routers.kitchenowl.tls=true
- traefik.http.routers.kitchenowl.tls.certresolver=letsencrypt
- traefik.http.routers.forgejo.entrypoints=websecure
- traefik.http.routers.forgejo.service=k3s@file
- traefik.http.routers.forgejo.rule=Host(`git.kun.is`)
- traefik.http.routers.forgejo.tls=true
- traefik.http.routers.forgejo.tls.certresolver=letsencrypt
- traefik.http.routers.jellyfin.entrypoints=websecure
- traefik.http.routers.jellyfin.service=k3s@file
- traefik.http.routers.jellyfin.rule=Host(`media.kun.is`)
- traefik.http.routers.jellyfin.tls=true
- traefik.http.routers.jellyfin.tls.certresolver=letsencrypt
- traefik.http.routers.transmission.entrypoints=localsecure
- traefik.http.routers.transmission.service=k3s@file
- traefik.http.routers.transmission.rule=Host(`transmission.kun.is`)
- traefik.http.routers.transmission.tls=true
- traefik.http.routers.transmission.tls.certresolver=letsencrypt
- traefik.http.routers.jellyseerr.entrypoints=localsecure
- traefik.http.routers.jellyseerr.service=k3s@file
- traefik.http.routers.jellyseerr.rule=Host(`jellyseerr.kun.is`)
- traefik.http.routers.jellyseerr.tls=true
- traefik.http.routers.jellyseerr.tls.certresolver=letsencrypt
- traefik.http.routers.radarr.entrypoints=localsecure
- traefik.http.routers.radarr.service=k3s@file
- traefik.http.routers.radarr.rule=Host(`radarr.kun.is`)
- traefik.http.routers.radarr.tls=true
- traefik.http.routers.radarr.tls.certresolver=letsencrypt
- traefik.http.routers.prowlarr.entrypoints=localsecure
- traefik.http.routers.prowlarr.service=k3s@file
- traefik.http.routers.prowlarr.rule=Host(`prowlarr.kun.is`)
- traefik.http.routers.prowlarr.tls=true
- traefik.http.routers.prowlarr.tls.certresolver=letsencrypt
- traefik.http.routers.sonarr.entrypoints=localsecure
- traefik.http.routers.sonarr.service=k3s@file
- traefik.http.routers.sonarr.rule=Host(`sonarr.kun.is`)
- traefik.http.routers.sonarr.tls=true
- traefik.http.routers.sonarr.tls.certresolver=letsencrypt
- traefik.http.routers.bazarr.entrypoints=localsecure
- traefik.http.routers.bazarr.service=k3s@file
- traefik.http.routers.bazarr.rule=Host(`bazarr.kun.is`)
- traefik.http.routers.bazarr.tls=true
- traefik.http.routers.bazarr.tls.certresolver=letsencrypt
volumes:
- type: bind
source: /var/run/docker.sock
target: /var/run/docker.sock
- type: volume
source: acme
target: /acme
volume:
nocopy: true
configs:
- source: services
target: /etc/traefik/services.yml
command:
- --providers.docker
- --providers.docker.swarmmode
- --providers.docker.watch
- --providers.docker.exposedbydefault=false
- --providers.file.filename=/etc/traefik/services.yml
- --api
- --api.insecure=false
- --api.dashboard=true
- --entrypoints.web.address=:80
- --entrypoints.web.http.redirections.entrypoint=true
- --entrypoints.web.http.redirections.entrypoint.to=websecure
- --entrypoints.web.http.redirections.entrypoint.scheme=https
- --entrypoints.web.http.redirections.entrypoint.permanent=true
- --entrypoints.websecure.address=:443
- --entrypoints.localsecure.address=:444
- --certificatesresolvers.letsencrypt.acme=true
- --certificatesresolvers.letsencrypt.acme.email=pim@kunis.nl
- --certificatesresolvers.letsencrypt.acme.storage=/acme/acme.json
- --certificatesresolvers.letsencrypt.acme.httpchallenge=true
- --certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web
- --serversTransport.insecureSkipVerify=true
- --accesslog=true
- --accesslog.fields.defaultmode=keep
- --accesslog.fields.names.ClientUsername=drop
- --accesslog.fields.headers.defaultmode=keep
- --accesslog.fields.headers.names.User-Agent=keep
- --accesslog.fields.headers.names.Authorization=drop
- --accesslog.fields.headers.names.Content-Type=keep

12
docker_swarm/roles/traefik/services.yml
View file

@ -1,12 +0,0 @@
http:
services:
k3s:
loadBalancer:
servers:
# TODO: This WILL break when the cluster is reprovisioned and another IP addrss is chosen.
# The load balancer service for Traefik is automatically provisioned by k3s, unsure how to statically assign the IP address.
- url: http://192.168.30.128
esrom:
loadBalancer:
servers:
- url: http://esrom.dmz:80/

18
docker_swarm/roles/traefik/tasks/main.yml
View file

@ -1,18 +0,0 @@
- name: Create Traefik network
docker_network:
name: traefik
driver: overlay
- name: Create Docker config
docker_config:
name: traefik_services
data: "{{ lookup('file', '{{ role_path }}/services.yml') }}"
use_ssh_client: true
rolling_versions: true
register: services
- name: Deploy Docker stack
docker_stack:
name: traefik
compose:
- "{{ lookup('template', '{{ role_path }}/docker-stack.yml.j2') | from_yaml }}"

2
nix/machines/default.nix
View file

@ -64,8 +64,6 @@ in
./jefke.nix
./lewis.nix
./hermes.nix
./maestro.nix
./vpay.nix
];
options = {

18
nix/machines/maestro.nix
View file

@ -1,18 +0,0 @@
{
machines.maestro = {
kind = "virtual";
hypervisorName = "atlas";
nixosModule = { config, ... }: {
microvm.balloonMem = 10000;
lab = {
dockerSwarm.enable = true;
vm = {
id = 1;
};
};
};
};
}

15
nix/machines/vpay.nix
View file

@ -1,15 +0,0 @@
{
machines.vpay = {
kind = "virtual";
hypervisorName = "lewis";
nixosModule = {
microvm.balloonMem = 10000;
lab = {
dockerSwarm.enable = true;
vm.id = 3;
};
};
};
}