feat: add blackbox monitoring and app3 deployment
- Add blackbox exporter role and playbooks - Add cadvisor, docker, httpbin roles - Add app3 deployment playbooks - Configure blackbox monitoring
This commit is contained in:
Freazzzing
151
playbooks/add-blackbox-correct.yml
Normal file
151
playbooks/add-blackbox-correct.yml
Normal file
@ -0,0 +1,151 @@
|
||||
---
|
||||
- name: Add correct Blackbox Exporter job
|
||||
hosts: 192.168.0.105
|
||||
become: yes
|
||||
|
||||
tasks:
|
||||
- name: Backup current config
|
||||
copy:
|
||||
src: /etc/prometheus/prometheus.yml
|
||||
dest: /etc/prometheus/prometheus.yml.backup-pre-blackbox-{{ ansible_date_time.epoch }}
|
||||
remote_src: yes
|
||||
tags: prometheus
|
||||
|
||||
- name: Check current line numbers
|
||||
shell: |
|
||||
echo "Last scrape_config job ends at line:"
|
||||
grep -n "job_name: postgres" /etc/prometheus/prometheus.yml
|
||||
echo ""
|
||||
echo "Remote_write starts at line:"
|
||||
grep -n "^remote_write:" /etc/prometheus/prometheus.yml
|
||||
register: line_info
|
||||
changed_when: false
|
||||
tags: prometheus
|
||||
|
||||
- name: Create correct blackbox job config
|
||||
copy:
|
||||
dest: /tmp/blackbox-job.yml
|
||||
content: |
|
||||
- job_name: blackbox
|
||||
honor_timestamps: true
|
||||
track_timestamps_staleness: false
|
||||
scrape_interval: 15s
|
||||
scrape_timeout: 10s
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx]
|
||||
scheme: http
|
||||
follow_redirects: true
|
||||
enable_http2: true
|
||||
static_configs:
|
||||
- targets:
|
||||
# Internal services
|
||||
- "http://192.168.0.110/"
|
||||
- "http://192.168.0.111:9187/metrics"
|
||||
- "http://192.168.0.112:8080/get"
|
||||
- "http://192.168.0.100:3000/"
|
||||
- "http://192.168.0.101:9100/metrics"
|
||||
- "http://192.168.0.103:8200/ui/"
|
||||
- "http://192.168.0.104:8428/metrics"
|
||||
- "http://192.168.0.105:9090/metrics"
|
||||
- "http://192.168.0.106:3000"
|
||||
# External domains
|
||||
- "http://forgejo.pvenode.ru/"
|
||||
- "http://grafana.pvenode.ru/"
|
||||
- "http://prometheus.pvenode.ru/"
|
||||
- "http://app1.pvenode.ru/"
|
||||
- "http://wiki.pvenode.ru/"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: 192.168.0.112:8083
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
separator: ;
|
||||
regex: (.*)
|
||||
target_label: instance
|
||||
replacement: $1
|
||||
action: replace
|
||||
- source_labels: [__address__]
|
||||
separator: ;
|
||||
regex: ([^:]+):\d+
|
||||
target_label: host
|
||||
replacement: ${1}
|
||||
action: replace
|
||||
tags: prometheus
|
||||
|
||||
- name: Insert blackbox job before remote_write
|
||||
shell: |
|
||||
# Находим строку с remote_write
|
||||
remote_line=$(grep -n "^remote_write:" /etc/prometheus/prometheus.yml | cut -d: -f1)
|
||||
|
||||
if [ -z "$remote_line" ]; then
|
||||
echo "ERROR: remote_write not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Создаем новый файл
|
||||
cp /etc/prometheus/prometheus.yml /etc/prometheus/prometheus.yml.tmp
|
||||
|
||||
# Вставляем blackbox перед remote_write
|
||||
head -n $((remote_line - 1)) /etc/prometheus/prometheus.yml > /etc/prometheus/prometheus.yml.new
|
||||
cat /tmp/blackbox-job.yml >> /etc/prometheus/prometheus.yml.new
|
||||
tail -n +$remote_line /etc/prometheus/prometheus.yml >> /etc/prometheus/prometheus.yml.new
|
||||
|
||||
# Заменяем старый файл
|
||||
mv /etc/prometheus/prometheus.yml.new /etc/prometheus/prometheus.yml
|
||||
rm -f /etc/prometheus/prometheus.yml.tmp
|
||||
|
||||
echo "Inserted at line $((remote_line - 1))"
|
||||
args:
|
||||
executable: /bin/bash
|
||||
tags: prometheus
|
||||
|
||||
- name: Check Prometheus configuration
|
||||
command: promtool check config /etc/prometheus/prometheus.yml
|
||||
register: promtool_check
|
||||
failed_when: promtool_check.rc != 0
|
||||
changed_when: false
|
||||
tags: prometheus
|
||||
|
||||
- name: Show config check result
|
||||
debug:
|
||||
msg: "{{ promtool_check.stdout_lines }}"
|
||||
when: promtool_check.rc == 0
|
||||
tags: prometheus
|
||||
|
||||
- name: Reload Prometheus
|
||||
systemd:
|
||||
name: prometheus
|
||||
state: reloaded
|
||||
when: promtool_check.rc == 0
|
||||
tags: prometheus
|
||||
|
||||
- name: Verify blackbox job added
|
||||
shell: |
|
||||
sleep 2
|
||||
echo "=== Checking if blackbox job exists ==="
|
||||
if grep -q "job_name: blackbox" /etc/prometheus/prometheus.yml; then
|
||||
echo "✓ Blackbox job found in config"
|
||||
echo ""
|
||||
echo "=== Checking Prometheus targets ==="
|
||||
curl -s "http://localhost:9090/api/v1/targets" | python3 -c "
|
||||
import json, sys
|
||||
data = json.load(sys.stdin)
|
||||
for target in data['data']['activeTargets']:
|
||||
job = target['discoveredLabels'].get('job', 'N/A')
|
||||
if 'blackbox' in job.lower():
|
||||
print(f'✓ Blackbox target: {target[\"health\"]}')
|
||||
print(f' URL: {target[\"scrapeUrl\"]}')
|
||||
exit(0)
|
||||
print('✗ Blackbox not in targets yet (may need 15s scrape interval)')
|
||||
"
|
||||
else
|
||||
echo "✗ Blackbox job not found in config"
|
||||
fi
|
||||
args:
|
||||
executable: /bin/bash
|
||||
tags: prometheus
|
||||
Reference in New Issue
Block a user