diff --git a/roles/coolify/tasks/main.yaml b/roles/coolify/tasks/main.yaml index 05114c2..f8a7181 100644 --- a/roles/coolify/tasks/main.yaml +++ b/roles/coolify/tasks/main.yaml @@ -13,6 +13,11 @@ ansible.builtin.include_tasks: detect.yaml tags: always +- name: Disable Coolify proxy watchdog + ansible.builtin.include_tasks: watchdog_disable.yaml + when: coolify_state in ['absent', 'stopped'] + tags: coolify, proxy + - name: Uninstall Coolify ansible.builtin.include_tasks: delete.yaml when: coolify_state == 'absent' @@ -60,6 +65,11 @@ - name: Ensure services are running ansible.builtin.include_tasks: start.yaml +- name: Configure Coolify proxy watchdog + ansible.builtin.include_tasks: watchdog.yaml + when: coolify_state in ['present', 'latest'] + tags: coolify, proxy + - name: Show Coolify status ansible.builtin.debug: msg: diff --git a/roles/coolify/tasks/watchdog.yaml b/roles/coolify/tasks/watchdog.yaml new file mode 100644 index 0000000..c398bd6 --- /dev/null +++ b/roles/coolify/tasks/watchdog.yaml @@ -0,0 +1,29 @@ +--- +- name: Deploy Coolify proxy watchdog script + become: true + ansible.builtin.template: + src: proxy-watchdog.sh.j2 + dest: /usr/local/sbin/coolify-proxy-watchdog + owner: root + group: root + mode: "0750" + +- name: Deploy Coolify proxy watchdog units + become: true + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + owner: root + group: root + mode: "0644" + loop: + - coolify-proxy-watchdog.service + - coolify-proxy-watchdog.timer + +- name: Enable Coolify proxy watchdog timer + become: true + ansible.builtin.systemd: + name: coolify-proxy-watchdog.timer + daemon_reload: true + enabled: true + state: started diff --git a/roles/coolify/tasks/watchdog_disable.yaml b/roles/coolify/tasks/watchdog_disable.yaml new file mode 100644 index 0000000..6f9e079 --- /dev/null +++ b/roles/coolify/tasks/watchdog_disable.yaml @@ -0,0 +1,34 @@ +--- +- name: Check for Coolify proxy watchdog timer + become: true + ansible.builtin.stat: + path: /etc/systemd/system/coolify-proxy-watchdog.timer + register: _coolify_watchdog_timer + +- name: Disable Coolify proxy watchdog timer + become: true + ansible.builtin.systemd: + name: coolify-proxy-watchdog.timer + enabled: false + state: stopped + when: _coolify_watchdog_timer.stat.exists + +- name: Remove Coolify proxy watchdog files + become: true + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /usr/local/sbin/coolify-proxy-watchdog + - /etc/systemd/system/coolify-proxy-watchdog.service + - /etc/systemd/system/coolify-proxy-watchdog.timer + register: _coolify_watchdog_removed + when: coolify_state == 'absent' + +- name: Reload systemd after removing Coolify proxy watchdog + become: true + ansible.builtin.systemd: + daemon_reload: true + when: + - coolify_state == 'absent' + - _coolify_watchdog_removed is changed diff --git a/roles/coolify/templates/coolify-proxy-watchdog.service.j2 b/roles/coolify/templates/coolify-proxy-watchdog.service.j2 new file mode 100644 index 0000000..7bb943e --- /dev/null +++ b/roles/coolify/templates/coolify-proxy-watchdog.service.j2 @@ -0,0 +1,8 @@ +[Unit] +Description=Recover Traefik proxies after Docker socket connection failures +After=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/coolify-proxy-watchdog +TimeoutStartSec=45s diff --git a/roles/coolify/templates/coolify-proxy-watchdog.timer.j2 b/roles/coolify/templates/coolify-proxy-watchdog.timer.j2 new file mode 100644 index 0000000..54edaf7 --- /dev/null +++ b/roles/coolify/templates/coolify-proxy-watchdog.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Check Traefik Docker socket connections + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min +Unit=coolify-proxy-watchdog.service + +[Install] +WantedBy=timers.target diff --git a/roles/coolify/templates/docker-compose.prod.yml.j2 b/roles/coolify/templates/docker-compose.prod.yml.j2 index 6e9e7fc..6c64452 100644 --- a/roles/coolify/templates/docker-compose.prod.yml.j2 +++ b/roles/coolify/templates/docker-compose.prod.yml.j2 @@ -1,4 +1,4 @@ -# Managed by Ansible — manual changes will be overwritten on next deployment +# Managed by Ansible - changes will be overwritten on next deployment services: coolify: image: "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify:${LATEST_IMAGE:-latest}" diff --git a/roles/coolify/templates/proxy-watchdog.sh.j2 b/roles/coolify/templates/proxy-watchdog.sh.j2 new file mode 100644 index 0000000..33049e2 --- /dev/null +++ b/roles/coolify/templates/proxy-watchdog.sh.j2 @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -u + +docker info >/dev/null 2>&1 || exit 0 +containers=$(timeout 10s docker ps --format '{% raw %}{{.Names}} {{.Image}}{% endraw %}') || exit 0 +state_dir=/run/coolify-proxy-watchdog +mkdir -p "$state_dir" + +while read -r container image; do + case "$container:$image" in + coolify-proxy:*|*traefik*) ;; + *) continue ;; + esac + + now=$(date +%s) + state_file="$state_dir/$container" + last_restart=$(cat "$state_file" 2>/dev/null || true) + if [[ "$last_restart" =~ ^[0-9]+$ ]] && (( now - last_restart < 300 )); then + continue + fi + + started_at=$(timeout 10s docker inspect --format '{% raw %}{{.State.StartedAt}}{% endraw %}' "$container") || continue + started_epoch=$(date -d "$started_at" +%s) || continue + if (( started_epoch > now - 120 )); then + since="$started_at" + else + since=2m + fi + + logs=$(timeout 15s docker logs --since "$since" --tail 500 "$container" 2>&1) || continue + if printf '%s\n' "$logs" | grep -Ei 'docker\.sock|docker\.socket' | grep -Eiq 'connection refused|cannot connect|no such file or directory|connection reset|i/o timeout|context deadline exceeded'; then + printf '%s\n' "$now" > "$state_file" + if docker restart --time 10 "$container" >/dev/null; then + echo "Restarted $container after Docker socket connection failure" + else + echo "Failed to restart $container after Docker socket connection failure" >&2 + fi + fi +done <<< "$containers"