feat: add Hetzner-hosted todo application stack

Implement the server-rendered Go todo application with PostgreSQL persistence, Tailwind styling, health checks, and test coverage.

Provision production and staging infrastructure with OpenTofu, including private networking, static IPs, Object Storage backups, and load balancing.

Configure Rocky Linux hosts with Ansible, rootless Podman, PostgreSQL, WAL-G backup automation, staging restores, and generated inventory.

ansible/roles/wal_g/defaults/main.yml

@@ -0,0 +1,2 @@
+wal_g_version: v3.0.7
+wal_g_retention_count: 7

ansible/roles/wal_g/handlers/main.yml

@@ -0,0 +1,12 @@
+---
+- name: Restart WAL-G timer
+  ansible.builtin.systemd_service:
+    name: wal-g-backup.timer
+    daemon_reload: true
+    state: restarted
+
+- name: Restart DB sync timer
+  ansible.builtin.systemd_service:
+    name: db-sync.timer
+    daemon_reload: true
+    state: restarted

ansible/roles/wal_g/tasks/main.yml

@@ -0,0 +1,123 @@
+---
+- name: Validate WAL-G configuration
+  ansible.builtin.assert:
+    that:
+      - wal_g_s3_prefix is match('^s3://.+/prod/?$')
+      - wal_g_access_key | length > 0
+      - wal_g_secret_key | length > 0
+    fail_msg: WAL-G must use the production backup prefix and external credentials.
+  no_log: true
+
+- name: Create WAL-G temporary directory
+  ansible.builtin.file:
+    path: "/var/tmp/wal-g-install/{{ wal_g_version }}"
+    state: directory
+    mode: "0755"
+
+- name: Download WAL-G
+  ansible.builtin.unarchive:
+    src: "{{ wal_g_download_url }}"
+    dest: "/var/tmp/wal-g-install/{{ wal_g_version }}"
+    remote_src: true
+    creates: "/var/tmp/wal-g-install/{{ wal_g_version }}/wal-g-pg-ubuntu-20.04-amd64"
+
+- name: Install WAL-G executable
+  ansible.builtin.copy:
+    src: "/var/tmp/wal-g-install/{{ wal_g_version }}/wal-g-pg-ubuntu-20.04-amd64"
+    dest: /usr/local/bin/wal-g
+    remote_src: true
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Create WAL-G configuration directory
+  ansible.builtin.file:
+    path: /etc/wal-g
+    state: directory
+    owner: root
+    group: postgres
+    mode: "0750"
+
+- name: Install protected WAL-G environment
+  ansible.builtin.template:
+    src: wal-g.env.j2
+    dest: /etc/wal-g/wal-g.env
+    owner: postgres
+    group: postgres
+    mode: "0600"
+  no_log: true
+
+- name: Install PostgreSQL WAL-G wrapper
+  ansible.builtin.template:
+    src: wal-g-postgres.j2
+    dest: /usr/local/sbin/wal-g-postgres
+    owner: root
+    group: postgres
+    mode: "0750"
+
+- name: Install production backup service
+  ansible.builtin.template:
+    src: wal-g-backup.service.j2
+    dest: /etc/systemd/system/wal-g-backup.service
+    mode: "0644"
+  when: deployment_environment == 'prod'
+  notify: Restart WAL-G timer
+
+- name: Install production backup timer
+  ansible.builtin.template:
+    src: wal-g-backup.timer.j2
+    dest: /etc/systemd/system/wal-g-backup.timer
+    mode: "0644"
+  when: deployment_environment == 'prod'
+  notify: Restart WAL-G timer
+
+- name: Enable production backup timer
+  ansible.builtin.systemd_service:
+    name: wal-g-backup.timer
+    enabled: true
+    daemon_reload: true
+    state: started
+  when: deployment_environment == 'prod'
+
+- name: Install staging restore configuration
+  ansible.builtin.template:
+    src: db-sync.env.j2
+    dest: /etc/wal-g/db-sync.env
+    owner: root
+    group: root
+    mode: "0600"
+  when: deployment_environment == 'staging'
+  no_log: true
+
+- name: Install staging restore script
+  ansible.builtin.template:
+    src: db-sync.sh.j2
+    dest: /usr/local/sbin/db-sync
+    owner: root
+    group: root
+    mode: "0700"
+  when: deployment_environment == 'staging'
+
+- name: Install staging restore service
+  ansible.builtin.template:
+    src: db-sync.service.j2
+    dest: /etc/systemd/system/db-sync.service
+    mode: "0644"
+  when: deployment_environment == 'staging'
+  notify: Restart DB sync timer
+
+- name: Install staging restore timer
+  ansible.builtin.template:
+    src: db-sync.timer.j2
+    dest: /etc/systemd/system/db-sync.timer
+    mode: "0644"
+  when: deployment_environment == 'staging'
+  notify: Restart DB sync timer
+
+- name: Enable staging restore timer
+  ansible.builtin.systemd_service:
+    name: db-sync.timer
+    enabled: true
+    daemon_reload: true
+    state: started
+  when: deployment_environment == 'staging'

ansible/roles/wal_g/templates/db-sync.env.j2

@@ -0,0 +1,12 @@
+AWS_ACCESS_KEY_ID={{ wal_g_access_key }}
+AWS_SECRET_ACCESS_KEY={{ wal_g_secret_key }}
+AWS_ENDPOINT={{ wal_g_s3_endpoint }}
+AWS_REGION={{ wal_g_s3_region }}
+AWS_S3_FORCE_PATH_STYLE=true
+WALG_S3_PREFIX={{ wal_g_s3_prefix | regex_replace('/$', '') }}
+WALG_COMPRESSION_METHOD=lz4
+PGDATA={{ postgres_data_dir }}
+POSTGRES_SERVICE={{ postgres_service }}
+APP_USER={{ app_user }}
+APP_DB_USER={{ app_db_user }}
+APP_DB_PASSWORD={{ app_db_password }}

ansible/roles/wal_g/templates/db-sync.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Restore production PostgreSQL backup into staging
+After=network-online.target
+Wants=network-online.target
+ConditionPathExists=/etc/wal-g/db-sync.env
+
+[Service]
+Type=oneshot
+EnvironmentFile=/etc/wal-g/db-sync.env
+ExecStart=/usr/local/sbin/db-sync
+TimeoutStartSec=45min
+Nice=10
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7

ansible/roles/wal_g/templates/db-sync.sh.j2

@@ -0,0 +1,54 @@
+#!/bin/bash
+set -Eeuo pipefail
+source /etc/wal-g/db-sync.env
+
+expected_pgdata="/var/lib/pgsql/data"
+if [[ "$PGDATA" != "$expected_pgdata" || ! -d "$PGDATA" || "$(stat -c %U "$PGDATA")" != "postgres" ]]; then
+  echo "Refusing destructive restore for unexpected PGDATA: $PGDATA" >&2
+  exit 1
+fi
+if [[ "$WALG_S3_PREFIX" != s3://*/prod ]]; then
+  echo "Refusing restore from a non-production prefix" >&2
+  exit 1
+fi
+
+systemctl --user --machine="${APP_USER}@" stop app-todo.service || true
+systemctl stop "$POSTGRES_SERVICE"
+
+shopt -s dotglob nullglob
+contents=("$PGDATA"/*)
+if (( ${#contents[@]} > 0 )); then
+  rm -rf --one-file-system -- "${contents[@]}"
+fi
+
+sudo -u postgres --preserve-env=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,AWS_ENDPOINT,AWS_REGION,AWS_S3_FORCE_PATH_STYLE,WALG_S3_PREFIX,WALG_COMPRESSION_METHOD \
+  /usr/local/bin/wal-g backup-fetch "$PGDATA" LATEST
+chown -R postgres:postgres "$PGDATA"
+chmod 0700 "$PGDATA"
+install -o postgres -g postgres -m 0600 /etc/todo-postgresql/postgresql.conf "$PGDATA/postgresql.conf"
+install -o postgres -g postgres -m 0600 /etc/todo-postgresql/pg_hba.conf "$PGDATA/pg_hba.conf"
+
+cat >"$PGDATA/postgresql.auto.conf" <<'RECOVERY'
+restore_command = '/usr/local/sbin/wal-g-postgres wal-fetch %f %p'
+recovery_target_timeline = 'latest'
+RECOVERY
+touch "$PGDATA/recovery.signal"
+chown postgres:postgres "$PGDATA/postgresql.auto.conf" "$PGDATA/recovery.signal"
+
+systemctl start "$POSTGRES_SERVICE"
+for _ in {1..120}; do
+  if sudo -u postgres psql --dbname=postgres --tuples-only --no-align --command="SELECT NOT pg_is_in_recovery()" 2>/dev/null | grep -qx t; then
+    break
+  fi
+  sleep 2
+done
+if ! sudo -u postgres psql --dbname=postgres --tuples-only --no-align --command="SELECT NOT pg_is_in_recovery()" | grep -qx t; then
+  echo "PostgreSQL did not complete recovery" >&2
+  exit 1
+fi
+
+sudo -u postgres psql --dbname=postgres --set="role=$APP_DB_USER" --set="password=$APP_DB_PASSWORD" <<'SQL'
+SELECT format('ALTER ROLE %I LOGIN PASSWORD %L', :'role', :'password') \gexec
+SQL
+
+systemctl --user --machine="${APP_USER}@" start app-todo.service

ansible/roles/wal_g/templates/db-sync.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Run staging database restore at 06:00 and 12:00 America/Chicago
+
+[Timer]
+OnCalendar=*-*-* 06,12:00:00 America/Chicago
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/roles/wal_g/templates/wal-g-backup.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Push PostgreSQL base backup to object storage
+After={{ postgres_service }}.service network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User=postgres
+Group=postgres
+EnvironmentFile=/etc/wal-g/wal-g.env
+ExecStart=/usr/local/bin/wal-g backup-push {{ postgres_data_dir }}
+ExecStartPost=/usr/local/bin/wal-g delete retain FULL {{ wal_g_retention_count }} --confirm
+Nice=10
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+PrivateTmp=true
+NoNewPrivileges=true

ansible/roles/wal_g/templates/wal-g-backup.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Run hourly PostgreSQL base backup
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+RandomizedDelaySec=5m
+
+[Install]
+WantedBy=timers.target

ansible/roles/wal_g/templates/wal-g-postgres.j2

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -euo pipefail
+{% if deployment_environment == 'staging' %}
+if [[ "${1:-}" == "wal-push" ]]; then
+  echo "WAL uploads are disabled on staging" >&2
+  exit 1
+fi
+{% endif %}
+set -a
+source /etc/wal-g/wal-g.env
+set +a
+exec /usr/local/bin/wal-g "$@"

ansible/roles/wal_g/templates/wal-g.env.j2

@@ -0,0 +1,9 @@
+AWS_ACCESS_KEY_ID={{ wal_g_access_key }}
+AWS_SECRET_ACCESS_KEY={{ wal_g_secret_key }}
+AWS_ENDPOINT={{ wal_g_s3_endpoint }}
+AWS_REGION={{ wal_g_s3_region }}
+AWS_S3_FORCE_PATH_STYLE=true
+WALG_S3_PREFIX={{ wal_g_s3_prefix | regex_replace('/$', '') }}
+WALG_COMPRESSION_METHOD=lz4
+WALG_UPLOAD_CONCURRENCY=4
+WALG_DOWNLOAD_CONCURRENCY=4