maintenance: add proper health checks and alerting

internal/service/assets/alloy.config.tmpl

@@ -58,3 +58,42 @@ prometheus.scrape "node" {
   scrape_interval = "15s"
 }
 
+prometheus.exporter.blackbox "services" {
+  config = `{
+    modules: {
+      tcp:      { prober: tcp },
+      http_2xx: { prober: http }
+    }
+  }`
+  target {
+    name    = "traefik"
+    address = "traefik:443"
+    module  = "tcp"
+  }
+  target {
+    name    = "finch"
+    address = "http://finch:3003/healthz"
+    module  = "http_2xx"
+  }
+  target {
+    name    = "mimir"
+    address = "http://mimir:8080/ready"
+    module  = "http_2xx"
+  }
+  target {
+    name    = "loki"
+    address = "http://loki:3100/ready"
+    module  = "http_2xx"
+  }
+  target {
+    name    = "pyroscope"
+    address = "http://pyroscope:4040/ready"
+    module  = "http_2xx"
+  }
+}
+
+prometheus.scrape "services" {
+  targets         = prometheus.exporter.blackbox.services.targets
+  forward_to      = [prometheus.remote_write.default.receiver]
+  scrape_interval = "15s"
+}

internal/service/assets/docker-compose.yaml.tmpl

@@ -8,6 +8,7 @@ services:
       - GF_SERVER_SERVE_FROM_SUB_PATH=true
     volumes:
       - /var/lib/finch/grafana:/var/lib/grafana
+      - /var/lib/finch/grafana/alerting:/etc/grafana/provisioning/alerting
     entrypoint:
       - sh
       - -euc
@@ -61,6 +62,8 @@ services:
         EOF
         /run.sh
     restart: always
+    labels:
+      finch.config-hash: "{{ .GrafanaConfigHash }}"
 
   loki:
     container_name: loki
@@ -73,6 +76,8 @@ services:
       - /var/lib/finch/loki/etc:/etc/loki
       - /var/lib/finch/loki/data:/var/lib/loki
     restart: always
+    labels:
+      finch.config-hash: "{{ .LokiConfigHash }}"
 
   traefik:
     container_name: traefik
@@ -104,6 +109,8 @@ services:
       - "--disable-reporting=true"
       - "/etc/alloy/alloy.config"
     restart: always
+    labels:
+      finch.config-hash: "{{ .AlloyConfigHash }}"
 
   finch:
     container_name: finch
@@ -122,6 +129,8 @@ services:
       - "--config.file=/etc/mimir/mimir.yaml"
     user: "10001:10001"
     restart: always
+    labels:
+      finch.config-hash: "{{ .MimirConfigHash }}"
 
   pyroscope:
     container_name: pyroscope
@@ -133,11 +142,106 @@ services:
       - "--compactor.blocks-retention-period=72h"
       - "--querier.max-query-lookback=72h"
       - "--querier.max-query-length=72h"
-      - "--querier.max-query-lookback=72h"
       - "--server.log-source-ips-enabled=true"
     user: "10001:10001"
     restart: always
 
+  hc-traefik:
+    container_name: hc-traefik
+    image: curlimages/curl:8.17.0
+    entrypoint:
+      - sleep
+      - infinity
+    healthcheck:
+      test: ["CMD-SHELL", "curl -so /dev/null -w '%{http_code}' http://traefik:80 | grep -qE '^[234][0-9]{2}$' && curl -sko /dev/null -w '%{http_code}' https://traefik:443 | grep -qE '^[234][0-9]{2}$'"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+      start_period: 10s
+    depends_on:
+      - traefik
+    restart: always
+
+  hc-finch:
+    container_name: hc-finch
+    image: curlimages/curl:8.17.0
+    entrypoint:
+      - sleep
+      - infinity
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fs http://finch:3003/healthz"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+      start_period: 10s
+    depends_on:
+      - finch
+    restart: always
+
+  hc-grafana:
+    container_name: hc-grafana
+    image: curlimages/curl:8.17.0
+    entrypoint:
+      - sleep
+      - infinity
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fs http://grafana:3000/api/health"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+      start_period: 10s
+    depends_on:
+      - grafana
+    restart: always
+
+  hc-loki:
+    container_name: hc-loki
+    image: curlimages/curl:8.17.0
+    entrypoint:
+      - sleep
+      - infinity
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fs http://loki:3100/ready"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+      start_period: 10s
+    depends_on:
+      - loki
+    restart: always
+
+  hc-mimir:
+    container_name: hc-mimir
+    image: curlimages/curl:8.17.0
+    entrypoint:
+      - sleep
+      - infinity
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fs http://mimir:8080/ready"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+      start_period: 10s
+    depends_on:
+      - mimir
+    restart: always
+
+  hc-pyroscope:
+    container_name: hc-pyroscope
+    image: curlimages/curl:8.17.0
+    entrypoint:
+      - sleep
+      - infinity
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fs http://pyroscope:4040/ready"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+      start_period: 10s
+    depends_on:
+      - pyroscope
+    restart: always
+
 networks:
   default:
     name: finch