ansible-config/playbooks/add-blackbox-correct.yml

---
- name: Add correct Blackbox Exporter job
  hosts: 192.168.0.105
  become: yes

  tasks:
    - name: Backup current config
      copy:
        src: /etc/prometheus/prometheus.yml
        dest: /etc/prometheus/prometheus.yml.backup-pre-blackbox-{{ ansible_date_time.epoch }}
        remote_src: yes
      tags: prometheus

    - name: Check current line numbers
      shell: |
        echo "Last scrape_config job ends at line:"
        grep -n "job_name: postgres" /etc/prometheus/prometheus.yml
        echo ""
        echo "Remote_write starts at line:"
        grep -n "^remote_write:" /etc/prometheus/prometheus.yml
      register: line_info
      changed_when: false
      tags: prometheus

    - name: Create correct blackbox job config
      copy:
        dest: /tmp/blackbox-job.yml
        content: |
          - job_name: blackbox
            honor_timestamps: true
            track_timestamps_staleness: false
            scrape_interval: 15s
            scrape_timeout: 10s
            metrics_path: /probe
            params:
              module: [http_2xx]
            scheme: http
            follow_redirects: true
            enable_http2: true
            static_configs:
              - targets:
                # Internal services
                - "http://192.168.0.110/"
                - "http://192.168.0.111:9187/metrics"
                - "http://192.168.0.112:8080/get"
                - "http://192.168.0.100:3000/"
                - "http://192.168.0.101:9100/metrics"
                - "http://192.168.0.103:8200/ui/"
                - "http://192.168.0.104:8428/metrics"
                - "http://192.168.0.105:9090/metrics"
                - "http://192.168.0.106:3000"
                # External domains
                - "http://forgejo.pvenode.ru/"
                - "http://grafana.pvenode.ru/"
                - "http://prometheus.pvenode.ru/"
                - "http://app1.pvenode.ru/"
                - "http://wiki.pvenode.ru/"
            relabel_configs:
              - source_labels: [__address__]
                target_label: __param_target
              - source_labels: [__param_target]
                target_label: instance
              - target_label: __address__
                replacement: 192.168.0.112:8083
            metric_relabel_configs:
              - source_labels: [__address__]
                separator: ;
                regex: (.*)
                target_label: instance
                replacement: $1
                action: replace
              - source_labels: [__address__]
                separator: ;
                regex: ([^:]+):\d+
                target_label: host
                replacement: ${1}
                action: replace
      tags: prometheus

    - name: Insert blackbox job before remote_write
      shell: |
        # Находим строку с remote_write
        remote_line=$(grep -n "^remote_write:" /etc/prometheus/prometheus.yml | cut -d: -f1)
        
        if [ -z "$remote_line" ]; then
          echo "ERROR: remote_write not found"
          exit 1
        fi
        
        # Создаем новый файл
        cp /etc/prometheus/prometheus.yml /etc/prometheus/prometheus.yml.tmp
        
        # Вставляем blackbox перед remote_write
        head -n $((remote_line - 1)) /etc/prometheus/prometheus.yml > /etc/prometheus/prometheus.yml.new
        cat /tmp/blackbox-job.yml >> /etc/prometheus/prometheus.yml.new
        tail -n +$remote_line /etc/prometheus/prometheus.yml >> /etc/prometheus/prometheus.yml.new
        
        # Заменяем старый файл
        mv /etc/prometheus/prometheus.yml.new /etc/prometheus/prometheus.yml
        rm -f /etc/prometheus/prometheus.yml.tmp
        
        echo "Inserted at line $((remote_line - 1))"
      args:
        executable: /bin/bash
      tags: prometheus

    - name: Check Prometheus configuration
      command: promtool check config /etc/prometheus/prometheus.yml
      register: promtool_check
      failed_when: promtool_check.rc != 0
      changed_when: false
      tags: prometheus

    - name: Show config check result
      debug:
        msg: "{{ promtool_check.stdout_lines }}"
      when: promtool_check.rc == 0
      tags: prometheus

    - name: Reload Prometheus
      systemd:
        name: prometheus
        state: reloaded
      when: promtool_check.rc == 0
      tags: prometheus

    - name: Verify blackbox job added
      shell: |
        sleep 2
        echo "=== Checking if blackbox job exists ==="
        if grep -q "job_name: blackbox" /etc/prometheus/prometheus.yml; then
          echo "✓ Blackbox job found in config"
          echo ""
          echo "=== Checking Prometheus targets ==="
          curl -s "http://localhost:9090/api/v1/targets" | python3 -c "
import json, sys
data = json.load(sys.stdin)
for target in data['data']['activeTargets']:
    job = target['discoveredLabels'].get('job', 'N/A')
    if 'blackbox' in job.lower():
        print(f'✓ Blackbox target: {target[\"health\"]}')
        print(f'  URL: {target[\"scrapeUrl\"]}')
        exit(0)
print('✗ Blackbox not in targets yet (may need 15s scrape interval)')
"
        else
          echo "✗ Blackbox job not found in config"
        fi
      args:
        executable: /bin/bash
      tags: prometheus