Health Monitoring

#!/bin/bash

PUSH_URL="https://uptime-kuma.your-domain.com/api/push/xxxxxx"

DISK_USAGE=$(df -h | grep -vE '^Filesystem|tmpfs|cdrom' \
  | awk '{ print $5 }' | sed 's/%//' | sort -n | tail -1)

MEM_TOTAL=$(free | awk '/^Mem:/ {print $2}')
MEM_USED=$(free | awk '/^Mem:/ {print $3}')
MEM_USAGE=$(( MEM_USED * 100 / MEM_TOTAL ))

STATUS="up"
MSG="Disk: ${DISK_USAGE}%, RAM: ${MEM_USAGE}%"

if [ "$DISK_USAGE" -gt 85 ] || [ "$MEM_USAGE" -gt 90 ]; then
  STATUS="down"
  MSG="ALERT - Disk: ${DISK_USAGE}%, RAM: ${MEM_USAGE}%"
fi

curl -s "${PUSH_URL}?status=${STATUS}&msg=${MSG}&ping=" > /dev/null

chmod +x /opt/scripts/health-push.sh
crontab -e
# Tambahkan:
*/5 * * * * /opt/scripts/health-push.sh >> /var/log/health-push.log 2>&1

docker run -d \
  --name blackbox_exporter \
  -p 9115:9115 \
  -v $(pwd)/blackbox.yml:/config/blackbox.yml \
  prom/blackbox-exporter:latest \
  --config.file=/config/blackbox.yml

modules:
  http_post_2xx:
    prober: http
    timeout: 10s
    http:
      method: POST
      valid_status_codes: [200]
      preferred_ip_protocol: "ip4"

  http_get_2xx:
    prober: http
    timeout: 10s
    http:
      method: GET
      valid_status_codes: [200]
      preferred_ip_protocol: "ip4"

  tcp_connect:
    prober: tcp
    timeout: 10s

  https_ssl:
    prober: http
    timeout: 10s
    http:
      method: GET
      tls_config:
        insecure_skip_verify: false

scrape_configs:
  - job_name: 'alurkerja-api'
    metrics_path: /probe
    params:
      module: [http_post_2xx]
    static_configs:
      - targets:
          - https://your-domain.com/api/v1/authentication/public/healthz
          - https://your-domain.com/api/v1/probis/public/healthz
          - https://your-domain.com/api/v1/compro/public/healthz
          - https://your-domain.com/api/v1/generate-test/public/healthz
          - https://your-domain.com/api/v1/integration/public/healthz
          - https://your-domain.com/api/v1/migration/public/healthz
          - https://your-domain.com/api/v1/notif/public/healthz
          - https://your-domain.com/api/v1/proxy/public/healthz
          - https://your-domain.com/api/v1/report/public/healthz
          - https://your-domain.com/api/v1/simulation/public/healthz
          - https://your-domain.com/api/v1/tenant/public/healthz
          - https://your-domain.com/api/v1/tasklist/public/healthz
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox_exporter:9115

  - job_name: 'alurkerja-tcp'
    metrics_path: /probe
    params:
      module: [tcp_connect]
    static_configs:
      - targets:
          - your-db-host:5432        # PostgreSQL
          - your-redis-host:6379     # Redis
          - your-rabbitmq-host:5672  # RabbitMQ
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox_exporter:9115

  - job_name: 'alurkerja-storage'
    metrics_path: /probe
    params:
      module: [http_get_2xx]
    static_configs:
      - targets:
          - https://your-storage-host/public/healthz
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox_exporter:9115
    scrape_interval: 2m

  - job_name: 'alurkerja-ssl'
    metrics_path: /probe
    params:
      module: [https_ssl]
    static_configs:
      - targets:
          - https://your-domain.com
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox_exporter:9115
    scrape_interval: 24h

docker run -d \
  --name node_exporter \
  --pid="host" \
  -v "/:/host:ro,rslave" \
  -p 9100:9100 \
  quay.io/prometheus/node-exporter:latest \
  --path.rootfs=/host

  - job_name: 'node-exporter'
    static_configs:
      - targets: ['your-server-host:9100']

# API/Storage service down
probe_success{job=~"alurkerja-api|alurkerja-storage"} == 0

# TCP connection failed (DB / Queue)
probe_success{job="alurkerja-tcp"} == 0

# Response time API > 3 detik
probe_duration_seconds{job="alurkerja-api"} > 3

# SSL expire < 30 hari
probe_ssl_earliest_cert_expiry - time() < 86400 * 30

# Disk usage > 85%
(1 - node_filesystem_avail_bytes / node_filesystem_size_bytes) * 100 > 85

# Memory usage > 90%
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90

docker run -d \
  --name gatus \
  -p 8080:8080 \
  -v $(pwd)/config:/config \
  twinproduction/gatus:latest

# Konfigurasi alerting global
alerting:
  slack:
    webhook-url: "https://hooks.slack.com/services/xxx/yyy/zzz"
    default-alert:
      failure-threshold: 3
      success-threshold: 1
      send-on-resolved: true

endpoints:
  # ───────────────────────────────
  # API Service (12 Service)
  # ───────────────────────────────
  - name: authentication
    group: api-service
    url: https://your-domain.com/api/v1/authentication/public/healthz
    method: POST
    interval: 1m
    conditions:
      - "[STATUS] == 200"
      - "[RESPONSE_TIME] < 3000"
    alerts:
      - type: slack

  - name: bpm
    group: api-service
    url: https://your-domain.com/api/v1/probis/public/healthz
    method: POST
    interval: 1m
    conditions:
      - "[STATUS] == 200"
      - "[RESPONSE_TIME] < 3000"
    alerts:
      - type: slack

  - name: company_profile
    group: api-service
    url: https://your-domain.com/api/v1/compro/public/healthz
    method: POST
    interval: 1m
    conditions:
      - "[STATUS] == 200"
      - "[RESPONSE_TIME] < 3000"
    alerts:
      - type: slack

  - name: generate_test
    group: api-service
    url: https://your-domain.com/api/v1/generate-test/public/healthz
    method: POST
    interval: 1m
    conditions:
      - "[STATUS] == 200"
      - "[RESPONSE_TIME] < 3000"
    alerts:
      - type: slack

  - name: integration
    group: api-service
    url: https://your-domain.com/api/v1/integration/public/healthz
    method: POST
    interval: 1m
    conditions:
      - "[STATUS] == 200"
      - "[RESPONSE_TIME] < 3000"
    alerts:
      - type: slack

  - name: migration
    group: api-service
    url: https://your-domain.com/api/v1/migration/public/healthz
    method: POST
    interval: 1m
    conditions:
      - "[STATUS] == 200"
      - "[RESPONSE_TIME] < 3000"
    alerts:
      - type: slack

  - name: notification
    group: api-service
    url: https://your-domain.com/api/v1/notif/public/healthz
    method: POST
    interval: 1m
    conditions:
      - "[STATUS] == 200"
      - "[RESPONSE_TIME] < 3000"
    alerts:
      - type: slack

  - name: proxy
    group: api-service
    url: https://your-domain.com/api/v1/proxy/public/healthz
    method: POST
    interval: 1m
    conditions:
      - "[STATUS] == 200"
      - "[RESPONSE_TIME] < 3000"
    alerts:
      - type: slack

  - name: report
    group: api-service
    url: https://your-domain.com/api/v1/report/public/healthz
    method: POST
    interval: 1m
    conditions:
      - "[STATUS] == 200"
      - "[RESPONSE_TIME] < 3000"
    alerts:
      - type: slack

  - name: simulation
    group: api-service
    url: https://your-domain.com/api/v1/simulation/public/healthz
    method: POST
    interval: 1m
    conditions:
      - "[STATUS] == 200"
      - "[RESPONSE_TIME] < 3000"
    alerts:
      - type: slack

  - name: tenant_management
    group: api-service
    url: https://your-domain.com/api/v1/tenant/public/healthz
    method: POST
    interval: 1m
    conditions:
      - "[STATUS] == 200"
      - "[RESPONSE_TIME] < 3000"
    alerts:
      - type: slack

  - name: camunda_tasklist
    group: api-service
    url: https://your-domain.com/api/v1/tasklist/public/healthz
    method: POST
    interval: 1m
    conditions:
      - "[STATUS] == 200"
      - "[RESPONSE_TIME] < 3000"
    alerts:
      - type: slack

  # ───────────────────────────────
  # Infrastruktur
  # ───────────────────────────────
  - name: postgresql
    group: infrastructure
    url: tcp://your-db-host:5432
    interval: 1m
    conditions:
      - "[CONNECTED] == true"
    alerts:
      - type: slack

  - name: redis
    group: infrastructure
    url: tcp://your-redis-host:6379
    interval: 1m
    conditions:
      - "[CONNECTED] == true"
    alerts:
      - type: slack

  - name: rabbitmq
    group: infrastructure
    url: tcp://your-rabbitmq-host:5672
    interval: 1m
    conditions:
      - "[CONNECTED] == true"
    alerts:
      - type: slack

  - name: file-storage
    group: infrastructure
    url: https://your-storage-host/public/healthz
    method: GET
    interval: 2m
    conditions:
      - "[STATUS] == 200"
    alerts:
      - type: slack

  - name: ssl-certificate
    group: infrastructure
    url: https://your-domain.com
    method: GET
    interval: 24h
    conditions:
      - "[STATUS] == 200"
      - "[CERTIFICATE_EXPIRATION] > 720h"   # 30 hari
    alerts:
      - type: slack
        failure-threshold: 1
        description: "SSL certificate akan expired dalam < 30 hari"

git clone https://github.com/OneUptime/oneuptime.git
cd oneuptime
cp config.env.example config.env
docker compose up -d

// Script berjalan di server OneUptime
const { execSync } = require('child_process');

const disk = execSync("df / | tail -1 | awk '{print $5}'")
  .toString().trim().replace('%', '');
const mem = execSync("free | awk '/^Mem:/ {printf \"%.0f\", $3/$2*100}'")
  .toString().trim();

if (parseInt(disk) > 85 || parseInt(mem) > 90) {
  throw new Error(`Resource alert — Disk: ${disk}%, RAM: ${mem}%`);
}

define service {
    host_name               alurkerja-prod
    service_description     Health - Authentication
    check_command           check_http!-H your-domain.com -u /api/v1/authentication/public/healthz --ssl --method=POST
    check_interval          1
    max_check_attempts      3
    notification_interval   30
}

/usr/lib/nagios/plugins/check_http \
  -H your-domain.com \
  -u /api/v1/authentication/public/healthz \
  --method=POST \
  --ssl \
  -w 3 -c 5

# PostgreSQL
/usr/lib/nagios/plugins/check_tcp -H your-db-host -p 5432

# Redis
/usr/lib/nagios/plugins/check_tcp -H your-redis-host -p 6379

# RabbitMQ
/usr/lib/nagios/plugins/check_tcp -H your-rabbitmq-host -p 5672

/usr/lib/nagios/plugins/check_http \
  -H your-storage-host \
  -u /public/healthz \
  --ssl -w 5 -c 10

/usr/lib/nagios/plugins/check_http \
  -H your-domain.com \
  --ssl \
  -C 30,14    # warning 30 hari, critical 14 hari

# Disk
/usr/lib/nagios/plugins/check_disk -w 15% -c 10% -p /

# Memory (butuh plugin check_mem)
/usr/lib/nagios/plugins/check_mem -w 90 -c 95

curl -X POST -o /dev/null -s -w "%{http_code}\n" \
  https://your-domain.com/api/v1/authentication/public/healthz

nc -zv your-db-host 5432       # PostgreSQL
nc -zv your-redis-host 6379    # Redis
nc -zv your-rabbitmq-host 5672 # RabbitMQ

echo | openssl s_client -connect your-domain.com:443 2>/dev/null \
  | openssl x509 -noout -dates

#!/bin/bash

BASE_URL="https://your-domain.com"

echo "=============================="
echo " Alurkerja Health Check"
echo " $(date)"
echo "=============================="

# ─── API Service ───────────────
declare -A API_SERVICES=(
  ["authentication"]="${BASE_URL}/api/v1/authentication/public/healthz"
  ["bpm"]="${BASE_URL}/api/v1/probis/public/healthz"
  ["company_profile"]="${BASE_URL}/api/v1/compro/public/healthz"
  ["generate_test"]="${BASE_URL}/api/v1/generate-test/public/healthz"
  ["integration"]="${BASE_URL}/api/v1/integration/public/healthz"
  ["migration"]="${BASE_URL}/api/v1/migration/public/healthz"
  ["notification"]="${BASE_URL}/api/v1/notif/public/healthz"
  ["proxy"]="${BASE_URL}/api/v1/proxy/public/healthz"
  ["report"]="${BASE_URL}/api/v1/report/public/healthz"
  ["simulation"]="${BASE_URL}/api/v1/simulation/public/healthz"
  ["tenant_management"]="${BASE_URL}/api/v1/tenant/public/healthz"
  ["camunda_tasklist"]="${BASE_URL}/api/v1/tasklist/public/healthz"
)

echo ""
echo "── API Service ──"
ALL_OK=true
for SERVICE in "${!API_SERVICES[@]}"; do
  STATUS=$(curl -X POST -o /dev/null -s -w "%{http_code}" \
    --max-time 10 "${API_SERVICES[$SERVICE]}")
  if [ "$STATUS" == "200" ]; then
    echo "  ✅ $SERVICE → $STATUS"
  else
    echo "  ❌ $SERVICE → $STATUS"
    ALL_OK=false
  fi
done

# ─── TCP Check ─────────────────
declare -A TCP_SERVICES=(
  ["postgresql"]="your-db-host:5432"
  ["redis"]="your-redis-host:6379"
  ["rabbitmq"]="your-rabbitmq-host:5672"
)

echo ""
echo "── Infrastruktur (TCP) ──"
for SERVICE in "${!TCP_SERVICES[@]}"; do
  HOST=$(echo "${TCP_SERVICES[$SERVICE]}" | cut -d: -f1)
  PORT=$(echo "${TCP_SERVICES[$SERVICE]}" | cut -d: -f2)
  if nc -z -w 5 "$HOST" "$PORT" 2>/dev/null; then
    echo "  ✅ $SERVICE ($HOST:$PORT)"
  else
    echo "  ❌ $SERVICE ($HOST:$PORT) — connection refused"
    ALL_OK=false
  fi
done

# ─── File Storage ───────────────
echo ""
echo "── File Storage ──"
FS_STATUS=$(curl -o /dev/null -s -w "%{http_code}" \
  --max-time 10 "https://your-storage-host/public/healthz")
if [ "$FS_STATUS" == "200" ]; then
  echo "  ✅ file-storage → $FS_STATUS"
else
  echo "  ❌ file-storage → $FS_STATUS"
  ALL_OK=false
fi

# ─── SSL Certificate ────────────
echo ""
echo "── SSL Certificate ──"
EXPIRY=$(echo | openssl s_client -connect your-domain.com:443 2>/dev/null \
  | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
if [ -n "$EXPIRY" ]; then
  echo "  ✅ SSL expires: $EXPIRY"
else
  echo "  ❌ SSL check failed"
  ALL_OK=false
fi

echo ""
echo "=============================="
if [ "$ALL_OK" = true ]; then
  echo " Semua monitor sehat ✅"
else
  echo " Ada monitor yang bermasalah ❌"
  exit 1
fi

chmod +x check-health.sh
./check-health.sh