feat: add blackbox monitoring and app3 deployment

- Add blackbox exporter role and playbooks
- Add cadvisor, docker, httpbin roles
- Add app3 deployment playbooks
- Configure blackbox monitoring

playbooks/add-blackbox-final.yml

@@ -0,0 +1,105 @@
+---
+- name: Add Blackbox job to Prometheus config
+  hosts: 192.168.0.105
+  become: yes
+
+  tasks:
+    - name: Backup config
+      copy:
+        src: /etc/prometheus/prometheus.yml
+        dest: /etc/prometheus/prometheus.yml.backup-blackbox
+        remote_src: yes
+
+    - name: Create blackbox config file
+      copy:
+        dest: /tmp/blackbox-config.yml
+        content: |
+          # Blackbox Exporter monitoring
+          - job_name: blackbox
+            honor_timestamps: true
+            track_timestamps_staleness: false
+            scrape_interval: 15s
+            scrape_timeout: 10s
+            metrics_path: /probe
+            params:
+              module: [http_2xx]
+            scheme: http
+            follow_redirects: true
+            enable_http2: true
+            static_configs:
+              - targets:
+                - "http://192.168.0.110/"
+                - "http://192.168.0.111:9187/metrics"
+                - "http://192.168.0.112:8080/get"
+                - "http://192.168.0.100:3000/"
+                - "http://192.168.0.101:9100/metrics"
+                - "http://192.168.0.103:8200/ui/"
+                - "http://192.168.0.104:8428/metrics"
+                - "http://192.168.0.105:9090/metrics"
+                - "http://192.168.0.106:3000"
+                - "http://forgejo.pvenode.ru/"
+                - "http://grafana.pvenode.ru/"
+                - "http://prometheus.pvenode.ru/"
+                - "http://app1.pvenode.ru/"
+                - "http://wiki.pvenode.ru/"
+            relabel_configs:
+              - source_labels: [__address__]
+                target_label: __param_target
+              - source_labels: [__param_target]
+                target_label: instance
+              - target_label: __address__
+                replacement: 192.168.0.112:8083
+            metric_relabel_configs:
+              - source_labels: [__address__]
+                separator: ;
+                regex: (.*)
+                target_label: instance
+                replacement: $1
+                action: replace
+              - source_labels: [__address__]
+                separator: ;
+                regex: ([^:]+):\d+
+                target_label: host
+                replacement: ${1}
+                action: replace
+
+    - name: Insert blackbox config before remote_write
+      shell: |
+        # Find remote_write line
+        remote_line=$(grep -n "^remote_write:" /etc/prometheus/prometheus.yml | head -1 | cut -d: -f1)
+        
+        if [ -z "$remote_line" ]; then
+          echo "ERROR: remote_write not found"
+          exit 1
+        fi
+        
+        # Insert blackbox config
+        head -n $((remote_line - 1)) /etc/prometheus/prometheus.yml > /tmp/prometheus-new.yml
+        cat /tmp/blackbox-config.yml >> /tmp/prometheus-new.yml
+        tail -n +$remote_line /etc/prometheus/prometheus.yml >> /tmp/prometheus-new.yml
+        
+        # Replace original
+        mv /tmp/prometheus-new.yml /etc/prometheus/prometheus.yml
+        
+        echo "Inserted at line $((remote_line - 1))"
+
+    - name: Validate config
+      command: promtool check config /etc/prometheus/prometheus.yml
+      register: config_check
+      changed_when: false
+
+    - name: Show validation result
+      debug:
+        msg: "{{ config_check.stdout_lines }}"
+
+    - name: Reload Prometheus
+      systemd:
+        name: prometheus
+        state: reloaded
+      when: config_check.rc == 0
+
+    - name: Check result
+      debug:
+        msg: |
+          Blackbox job {{ "successfully added" if config_check.rc == 0 else "failed to add" }}
+          Backup: /etc/prometheus/prometheus.yml.backup-blackbox