feat: add Hetzner-hosted todo application stack

Implement the server-rendered Go todo application with PostgreSQL persistence, Tailwind styling, health checks, and test coverage.

Provision production and staging infrastructure with OpenTofu, including private networking, static IPs, Object Storage backups, and load balancing.

Configure Rocky Linux hosts with Ansible, rootless Podman, PostgreSQL, WAL-G backup automation, staging restores, and generated inventory.
This commit is contained in:
cole-maxwell1
2026-08-06 21:34:56 -05:00
commit 1382c42803
68 changed files with 2380 additions and 0 deletions
+11
View File
@@ -0,0 +1,11 @@
[defaults]
roles_path = roles
interpreter_python = auto_silent
host_key_checking = True
retry_files_enabled = False
[ssh_connection]
ssh_args = -o StrictHostKeyChecking=accept-new
[privilege_escalation]
become = True
+22
View File
@@ -0,0 +1,22 @@
app_user: todo
app_image: registry.example.com/todo:latest
app_port: 8080
app_db_name: todo
app_db_user: todo_app
postgres_data_dir: /var/lib/pgsql/data
postgres_service: postgresql
wal_g_version: v3.0.7
wal_g_download_url: "https://github.com/wal-g/wal-g/releases/download/{{ wal_g_version }}/wal-g-pg-ubuntu-20.04-amd64.tar.gz"
wal_g_s3_prefix: s3://replace-with-backup-bucket/prod
wal_g_s3_endpoint: https://nbg1.your-objectstorage.com
wal_g_s3_region: nbg1
wal_g_access_key: "replace-with-object-storage-access-key"
wal_g_secret_key: "replace-with-object-storage-secret-key"
wal_g_retention_count: 7
# Optional private registry credentials. Omit both for public images.
# registry_username: registry-user
# registry_password: registry-password
# registry_server: registry.example.com
@@ -0,0 +1 @@
app_db_password: "replace-with-production-database-password"
@@ -0,0 +1 @@
app_db_password: "replace-with-distinct-staging-database-password"
+16
View File
@@ -0,0 +1,16 @@
all:
vars:
ansible_user: root
children:
prod:
hosts:
todo-prod:
ansible_host: 203.0.113.10
private_ip: 10.0.1.10
deployment_environment: prod
staging:
hosts:
todo-staging:
ansible_host: 203.0.113.20
private_ip: 10.0.1.20
deployment_environment: staging
+43
View File
@@ -0,0 +1,43 @@
---
- name: Deploy todo application
hosts: all
become: true
roles:
- app
tasks:
- name: Pull application image
ansible.builtin.command:
argv:
- podman
- pull
- "{{ app_image }}"
become_user: "{{ app_user }}"
environment:
XDG_RUNTIME_DIR: "/run/user/{{ app_user_uid }}"
register: app_pull
changed_when: "'Copying blob' in app_pull.stdout or 'Writing manifest' in app_pull.stdout"
- name: Reload application user services
ansible.builtin.command:
argv: [systemctl, --user, daemon-reload]
become_user: "{{ app_user }}"
environment:
XDG_RUNTIME_DIR: "/run/user/{{ app_user_uid }}"
changed_when: false
- name: Restart application
ansible.builtin.command:
argv: [systemctl, --user, restart, app-todo.service]
become_user: "{{ app_user }}"
environment:
XDG_RUNTIME_DIR: "/run/user/{{ app_user_uid }}"
changed_when: true
- name: Wait for application health endpoint
ansible.builtin.uri:
url: "http://127.0.0.1:{{ app_port }}/healthz"
status_code: 200
register: app_health
retries: 12
delay: 2
until: app_health.status == 200
+8
View File
@@ -0,0 +1,8 @@
---
- name: Configure todo nodes
hosts: all
become: true
roles:
- common
- postgres
- wal_g
+4
View File
@@ -0,0 +1,4 @@
app_user: todo
app_port: 8080
app_db_name: todo
app_db_user: todo_app
+71
View File
@@ -0,0 +1,71 @@
---
- name: Look up application user
ansible.builtin.getent:
database: passwd
key: "{{ app_user }}"
- name: Store application user ID
ansible.builtin.set_fact:
app_user_uid: "{{ ansible_facts.getent_passwd[app_user][1] }}"
- name: Create Quadlet directory
ansible.builtin.file:
path: "/home/{{ app_user }}/.config/containers/systemd"
state: directory
owner: "{{ app_user }}"
group: "{{ app_user }}"
mode: "0700"
- name: Install protected application environment
ansible.builtin.template:
src: app.env.j2
dest: "/home/{{ app_user }}/.config/todo.env"
owner: "{{ app_user }}"
group: "{{ app_user }}"
mode: "0600"
no_log: true
- name: Authenticate to private registry
ansible.builtin.command:
argv:
- podman
- login
- --username
- "{{ registry_username }}"
- --password-stdin
- "{{ registry_server }}"
stdin: "{{ registry_password }}"
become_user: "{{ app_user }}"
environment:
XDG_RUNTIME_DIR: "/run/user/{{ app_user_uid }}"
changed_when: true
no_log: true
when:
- registry_username is defined
- registry_password is defined
- registry_server is defined
- name: Install application Quadlet
ansible.builtin.template:
src: app-todo.container.j2
dest: "/home/{{ app_user }}/.config/containers/systemd/app-todo.container"
owner: "{{ app_user }}"
group: "{{ app_user }}"
mode: "0644"
- name: Reload application user manager
ansible.builtin.command:
argv: [systemctl, --user, daemon-reload]
become_user: "{{ app_user }}"
environment:
XDG_RUNTIME_DIR: "/run/user/{{ app_user_uid }}"
changed_when: false
- name: Enable application service
ansible.builtin.command:
argv: [systemctl, --user, enable, --now, app-todo.service]
become_user: "{{ app_user }}"
environment:
XDG_RUNTIME_DIR: "/run/user/{{ app_user_uid }}"
register: app_enable
changed_when: "'Created symlink' in app_enable.stderr"
@@ -0,0 +1,22 @@
[Unit]
Description=Todo web application
After=network-online.target
Wants=network-online.target
[Container]
Image={{ app_image }}
ContainerName=app-todo
EnvironmentFile=/home/{{ app_user }}/.config/todo.env
Network=host
Pull=never
NoNewPrivileges=true
ReadOnly=true
DropCapability=all
[Service]
Restart=always
RestartSec=5
TimeoutStopSec=15
[Install]
WantedBy=default.target
+2
View File
@@ -0,0 +1,2 @@
APP_ADDR=0.0.0.0:{{ app_port }}
DATABASE_URL=postgres://{{ app_db_user | urlencode }}:{{ app_db_password | urlencode }}@127.0.0.1:5432/{{ app_db_name | urlencode }}?sslmode=disable
+1
View File
@@ -0,0 +1 @@
app_user: todo
+37
View File
@@ -0,0 +1,37 @@
---
- name: Install operating system packages
ansible.builtin.dnf:
name:
- ca-certificates
- curl
- policycoreutils-python-utils
- podman
- postgresql-server
- postgresql-contrib
- sudo
- tar
state: present
update_cache: true
- name: Create rootless application user
ansible.builtin.user:
name: "{{ app_user }}"
create_home: true
shell: /bin/bash
state: present
register: app_user_result
- name: Store application user ID
ansible.builtin.set_fact:
app_user_uid: "{{ app_user_result.uid }}"
- name: Enable linger for application user
ansible.builtin.command:
argv: [loginctl, enable-linger, "{{ app_user }}"]
args:
creates: "/var/lib/systemd/linger/{{ app_user }}"
- name: Start application user manager
ansible.builtin.systemd_service:
name: "user@{{ app_user_uid }}.service"
state: started
+4
View File
@@ -0,0 +1,4 @@
postgres_data_dir: /var/lib/pgsql/data
postgres_service: postgresql
app_db_name: todo
app_db_user: todo_app
+5
View File
@@ -0,0 +1,5 @@
---
- name: Restart PostgreSQL
ansible.builtin.systemd_service:
name: "{{ postgres_service }}"
state: restarted
+75
View File
@@ -0,0 +1,75 @@
---
- name: Initialize PostgreSQL data directory
ansible.builtin.command:
argv: [postgresql-setup, --initdb]
args:
creates: "{{ postgres_data_dir }}/PG_VERSION"
- name: Create canonical PostgreSQL configuration directory
ansible.builtin.file:
path: /etc/todo-postgresql
state: directory
owner: root
group: postgres
mode: "0750"
- name: Render canonical PostgreSQL server configuration
ansible.builtin.template:
src: postgresql.conf.j2
dest: /etc/todo-postgresql/postgresql.conf
owner: root
group: postgres
mode: "0640"
notify: Restart PostgreSQL
- name: Render canonical PostgreSQL client authentication
ansible.builtin.template:
src: pg_hba.conf.j2
dest: /etc/todo-postgresql/pg_hba.conf
owner: root
group: postgres
mode: "0640"
notify: Restart PostgreSQL
- name: Install PostgreSQL configuration in data directory
ansible.builtin.copy:
src: "/etc/todo-postgresql/{{ item }}"
dest: "{{ postgres_data_dir }}/{{ item }}"
remote_src: true
owner: postgres
group: postgres
mode: "0600"
loop:
- postgresql.conf
- pg_hba.conf
notify: Restart PostgreSQL
- name: Start PostgreSQL
ansible.builtin.systemd_service:
name: "{{ postgres_service }}"
enabled: true
state: started
- name: Apply PostgreSQL configuration before provisioning
ansible.builtin.meta: flush_handlers
- name: Provision application database role
ansible.builtin.command:
argv: [psql, --set, "role={{ app_db_user }}", --set, "password={{ app_db_password }}", --dbname, postgres]
stdin: |
SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'role', :'password')
WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'role') \gexec
SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'role', :'password') \gexec
become_user: postgres
changed_when: false
no_log: true
- name: Provision application database
ansible.builtin.command:
argv: [psql, --set, "database={{ app_db_name }}", --set, "role={{ app_db_user }}", --dbname, postgres]
stdin: |
SELECT format('CREATE DATABASE %I OWNER %I', :'database', :'role')
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = :'database') \gexec
SELECT format('ALTER DATABASE %I OWNER TO %I', :'database', :'role') \gexec
become_user: postgres
changed_when: false
@@ -0,0 +1,4 @@
local all postgres peer
local all all peer
host {{ app_db_name }} {{ app_db_user }} 127.0.0.1/32 scram-sha-256
host all all ::1/128 scram-sha-256
@@ -0,0 +1,20 @@
data_directory = '{{ postgres_data_dir }}'
listen_addresses = '127.0.0.1'
port = 5432
max_connections = 100
shared_buffers = 256MB
password_encryption = 'scram-sha-256'
wal_level = replica
max_wal_senders = 5
wal_compression = on
{% if deployment_environment == 'prod' %}
archive_mode = on
archive_command = '/usr/local/sbin/wal-g-postgres wal-push %p'
archive_timeout = 300
{% else %}
archive_mode = off
archive_command = ''
{% endif %}
logging_collector = on
log_directory = 'log'
log_filename = 'postgresql-%a.log'
+2
View File
@@ -0,0 +1,2 @@
wal_g_version: v3.0.7
wal_g_retention_count: 7
+12
View File
@@ -0,0 +1,12 @@
---
- name: Restart WAL-G timer
ansible.builtin.systemd_service:
name: wal-g-backup.timer
daemon_reload: true
state: restarted
- name: Restart DB sync timer
ansible.builtin.systemd_service:
name: db-sync.timer
daemon_reload: true
state: restarted
+123
View File
@@ -0,0 +1,123 @@
---
- name: Validate WAL-G configuration
ansible.builtin.assert:
that:
- wal_g_s3_prefix is match('^s3://.+/prod/?$')
- wal_g_access_key | length > 0
- wal_g_secret_key | length > 0
fail_msg: WAL-G must use the production backup prefix and external credentials.
no_log: true
- name: Create WAL-G temporary directory
ansible.builtin.file:
path: "/var/tmp/wal-g-install/{{ wal_g_version }}"
state: directory
mode: "0755"
- name: Download WAL-G
ansible.builtin.unarchive:
src: "{{ wal_g_download_url }}"
dest: "/var/tmp/wal-g-install/{{ wal_g_version }}"
remote_src: true
creates: "/var/tmp/wal-g-install/{{ wal_g_version }}/wal-g-pg-ubuntu-20.04-amd64"
- name: Install WAL-G executable
ansible.builtin.copy:
src: "/var/tmp/wal-g-install/{{ wal_g_version }}/wal-g-pg-ubuntu-20.04-amd64"
dest: /usr/local/bin/wal-g
remote_src: true
owner: root
group: root
mode: "0755"
- name: Create WAL-G configuration directory
ansible.builtin.file:
path: /etc/wal-g
state: directory
owner: root
group: postgres
mode: "0750"
- name: Install protected WAL-G environment
ansible.builtin.template:
src: wal-g.env.j2
dest: /etc/wal-g/wal-g.env
owner: postgres
group: postgres
mode: "0600"
no_log: true
- name: Install PostgreSQL WAL-G wrapper
ansible.builtin.template:
src: wal-g-postgres.j2
dest: /usr/local/sbin/wal-g-postgres
owner: root
group: postgres
mode: "0750"
- name: Install production backup service
ansible.builtin.template:
src: wal-g-backup.service.j2
dest: /etc/systemd/system/wal-g-backup.service
mode: "0644"
when: deployment_environment == 'prod'
notify: Restart WAL-G timer
- name: Install production backup timer
ansible.builtin.template:
src: wal-g-backup.timer.j2
dest: /etc/systemd/system/wal-g-backup.timer
mode: "0644"
when: deployment_environment == 'prod'
notify: Restart WAL-G timer
- name: Enable production backup timer
ansible.builtin.systemd_service:
name: wal-g-backup.timer
enabled: true
daemon_reload: true
state: started
when: deployment_environment == 'prod'
- name: Install staging restore configuration
ansible.builtin.template:
src: db-sync.env.j2
dest: /etc/wal-g/db-sync.env
owner: root
group: root
mode: "0600"
when: deployment_environment == 'staging'
no_log: true
- name: Install staging restore script
ansible.builtin.template:
src: db-sync.sh.j2
dest: /usr/local/sbin/db-sync
owner: root
group: root
mode: "0700"
when: deployment_environment == 'staging'
- name: Install staging restore service
ansible.builtin.template:
src: db-sync.service.j2
dest: /etc/systemd/system/db-sync.service
mode: "0644"
when: deployment_environment == 'staging'
notify: Restart DB sync timer
- name: Install staging restore timer
ansible.builtin.template:
src: db-sync.timer.j2
dest: /etc/systemd/system/db-sync.timer
mode: "0644"
when: deployment_environment == 'staging'
notify: Restart DB sync timer
- name: Enable staging restore timer
ansible.builtin.systemd_service:
name: db-sync.timer
enabled: true
daemon_reload: true
state: started
when: deployment_environment == 'staging'
@@ -0,0 +1,12 @@
AWS_ACCESS_KEY_ID={{ wal_g_access_key }}
AWS_SECRET_ACCESS_KEY={{ wal_g_secret_key }}
AWS_ENDPOINT={{ wal_g_s3_endpoint }}
AWS_REGION={{ wal_g_s3_region }}
AWS_S3_FORCE_PATH_STYLE=true
WALG_S3_PREFIX={{ wal_g_s3_prefix | regex_replace('/$', '') }}
WALG_COMPRESSION_METHOD=lz4
PGDATA={{ postgres_data_dir }}
POSTGRES_SERVICE={{ postgres_service }}
APP_USER={{ app_user }}
APP_DB_USER={{ app_db_user }}
APP_DB_PASSWORD={{ app_db_password }}
@@ -0,0 +1,14 @@
[Unit]
Description=Restore production PostgreSQL backup into staging
After=network-online.target
Wants=network-online.target
ConditionPathExists=/etc/wal-g/db-sync.env
[Service]
Type=oneshot
EnvironmentFile=/etc/wal-g/db-sync.env
ExecStart=/usr/local/sbin/db-sync
TimeoutStartSec=45min
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
@@ -0,0 +1,54 @@
#!/bin/bash
set -Eeuo pipefail
source /etc/wal-g/db-sync.env
expected_pgdata="/var/lib/pgsql/data"
if [[ "$PGDATA" != "$expected_pgdata" || ! -d "$PGDATA" || "$(stat -c %U "$PGDATA")" != "postgres" ]]; then
echo "Refusing destructive restore for unexpected PGDATA: $PGDATA" >&2
exit 1
fi
if [[ "$WALG_S3_PREFIX" != s3://*/prod ]]; then
echo "Refusing restore from a non-production prefix" >&2
exit 1
fi
systemctl --user --machine="${APP_USER}@" stop app-todo.service || true
systemctl stop "$POSTGRES_SERVICE"
shopt -s dotglob nullglob
contents=("$PGDATA"/*)
if (( ${#contents[@]} > 0 )); then
rm -rf --one-file-system -- "${contents[@]}"
fi
sudo -u postgres --preserve-env=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_ENDPOINT,AWS_REGION,AWS_S3_FORCE_PATH_STYLE,WALG_S3_PREFIX,WALG_COMPRESSION_METHOD \
/usr/local/bin/wal-g backup-fetch "$PGDATA" LATEST
chown -R postgres:postgres "$PGDATA"
chmod 0700 "$PGDATA"
install -o postgres -g postgres -m 0600 /etc/todo-postgresql/postgresql.conf "$PGDATA/postgresql.conf"
install -o postgres -g postgres -m 0600 /etc/todo-postgresql/pg_hba.conf "$PGDATA/pg_hba.conf"
cat >"$PGDATA/postgresql.auto.conf" <<'RECOVERY'
restore_command = '/usr/local/sbin/wal-g-postgres wal-fetch %f %p'
recovery_target_timeline = 'latest'
RECOVERY
touch "$PGDATA/recovery.signal"
chown postgres:postgres "$PGDATA/postgresql.auto.conf" "$PGDATA/recovery.signal"
systemctl start "$POSTGRES_SERVICE"
for _ in {1..120}; do
if sudo -u postgres psql --dbname=postgres --tuples-only --no-align --command="SELECT NOT pg_is_in_recovery()" 2>/dev/null | grep -qx t; then
break
fi
sleep 2
done
if ! sudo -u postgres psql --dbname=postgres --tuples-only --no-align --command="SELECT NOT pg_is_in_recovery()" | grep -qx t; then
echo "PostgreSQL did not complete recovery" >&2
exit 1
fi
sudo -u postgres psql --dbname=postgres --set="role=$APP_DB_USER" --set="password=$APP_DB_PASSWORD" <<'SQL'
SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'role', :'password') \gexec
SQL
systemctl --user --machine="${APP_USER}@" start app-todo.service
@@ -0,0 +1,9 @@
[Unit]
Description=Run staging database restore at 06:00 and 12:00 America/Chicago
[Timer]
OnCalendar=*-*-* 06,12:00:00 America/Chicago
Persistent=true
[Install]
WantedBy=timers.target
@@ -0,0 +1,17 @@
[Unit]
Description=Push PostgreSQL base backup to object storage
After={{ postgres_service }}.service network-online.target
Wants=network-online.target
[Service]
Type=oneshot
User=postgres
Group=postgres
EnvironmentFile=/etc/wal-g/wal-g.env
ExecStart=/usr/local/bin/wal-g backup-push {{ postgres_data_dir }}
ExecStartPost=/usr/local/bin/wal-g delete retain FULL {{ wal_g_retention_count }} --confirm
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
PrivateTmp=true
NoNewPrivileges=true
@@ -0,0 +1,10 @@
[Unit]
Description=Run hourly PostgreSQL base backup
[Timer]
OnCalendar=hourly
Persistent=true
RandomizedDelaySec=5m
[Install]
WantedBy=timers.target
@@ -0,0 +1,12 @@
#!/bin/bash
set -euo pipefail
{% if deployment_environment == 'staging' %}
if [[ "${1:-}" == "wal-push" ]]; then
echo "WAL uploads are disabled on staging" >&2
exit 1
fi
{% endif %}
set -a
source /etc/wal-g/wal-g.env
set +a
exec /usr/local/bin/wal-g "$@"
@@ -0,0 +1,9 @@
AWS_ACCESS_KEY_ID={{ wal_g_access_key }}
AWS_SECRET_ACCESS_KEY={{ wal_g_secret_key }}
AWS_ENDPOINT={{ wal_g_s3_endpoint }}
AWS_REGION={{ wal_g_s3_region }}
AWS_S3_FORCE_PATH_STYLE=true
WALG_S3_PREFIX={{ wal_g_s3_prefix | regex_replace('/$', '') }}
WALG_COMPRESSION_METHOD=lz4
WALG_UPLOAD_CONCURRENCY=4
WALG_DOWNLOAD_CONCURRENCY=4
+14
View File
@@ -0,0 +1,14 @@
[Unit]
Description=Restore production PostgreSQL backup into staging
After=network-online.target
Wants=network-online.target
ConditionPathExists=/etc/wal-g/db-sync.env
[Service]
Type=oneshot
EnvironmentFile=/etc/wal-g/db-sync.env
ExecStart=/usr/local/sbin/db-sync
TimeoutStartSec=45min
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=7
+9
View File
@@ -0,0 +1,9 @@
[Unit]
Description=Run staging database restore at 06:00 and 12:00 America/Chicago
[Timer]
OnCalendar=*-*-* 06,12:00:00 America/Chicago
Persistent=true
[Install]
WantedBy=timers.target
+9
View File
@@ -0,0 +1,9 @@
AWS_ACCESS_KEY_ID={{ wal_g_access_key }}
AWS_SECRET_ACCESS_KEY={{ wal_g_secret_key }}
AWS_ENDPOINT={{ wal_g_s3_endpoint }}
AWS_REGION={{ wal_g_s3_region }}
AWS_S3_FORCE_PATH_STYLE=true
WALG_S3_PREFIX={{ wal_g_s3_prefix | regex_replace('/$', '') }}
WALG_COMPRESSION_METHOD=lz4
WALG_UPLOAD_CONCURRENCY=4
WALG_DOWNLOAD_CONCURRENCY=4