Added Prometheus Stuffs

collections/ansible_collections/adhdgirl/minilab/roles/grafana/defaults/main.yaml

@@ -0,0 +1,51 @@
+---
+grafana_version: "latest"
+
+# Network settings
+grafana_http_addr: "0.0.0.0"
+grafana_http_port: 3000
+grafana_domain: "grafana.example.com"
+grafana_root_url: "http://{{ grafana_domain }}:{{ grafana_http_port }}"
+
+# Security settings
+grafana_admin_user: "admin"
+grafana_admin_password: "changeme"
+grafana_secret_key: "SW2YcwTIb9zpOOhoPsMm"
+grafana_disable_gravatar: true
+grafana_allow_sign_up: false
+
+# Database settings (default: sqlite3)
+grafana_database_type: "sqlite3"
+grafana_database_host: ""
+grafana_database_name: "grafana"
+grafana_database_user: ""
+grafana_database_password: ""
+
+# SMTP settings for email alerts
+grafana_smtp_enabled: false
+grafana_smtp_host: "localhost:25"
+grafana_smtp_user: ""
+grafana_smtp_password: ""
+grafana_smtp_from_address: "grafana@example.com"
+
+# Datasources to provision
+grafana_datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://localhost:9090
+    is_default: true
+    editable: false
+
+# Plugins to install
+grafana_plugins:
+  - grafana-clock-panel
+  - grafana-polystat-panel
+
+# Log settings
+grafana_log_mode: "console file"
+grafana_log_level: "info"
+
+# Anonymous access
+grafana_anonymous_enabled: false
+grafana_anonymous_org_role: "Viewer"

collections/ansible_collections/adhdgirl/minilab/roles/grafana/handlers/main.yaml

@@ -0,0 +1,6 @@
+---
+- name: Restart grafana
+  ansible.builtin.systemd:
+    name: grafana-server
+    state: restarted
+    daemon_reload: true

collections/ansible_collections/adhdgirl/minilab/roles/grafana/tasks/configure.yaml

@@ -0,0 +1,21 @@
+---
+- name: Deploy Grafana configuration
+  ansible.builtin.template:
+    src: grafana.ini.j2
+    dest: /etc/grafana/grafana.ini
+    owner: root
+    group: grafana
+    mode: "0640"
+  notify: Restart grafana
+
+- name: Create provisioning directories
+  ansible.builtin.file:
+    path: "/etc/grafana/provisioning/{{ item }}"
+    state: directory
+    owner: root
+    group: grafana
+    mode: "0755"
+  loop:
+    - datasources
+    - dashboards
+    - notifiers

collections/ansible_collections/adhdgirl/minilab/roles/grafana/tasks/datasources.yaml

@@ -0,0 +1,10 @@
+---
+- name: Deploy datasource provisioning file
+  ansible.builtin.template:
+    src: datasource.yaml.j2
+    dest: /etc/grafana/provisioning/datasources/ansible.yaml
+    owner: root
+    group: grafana
+    mode: "0640"
+  become: true
+  notify: Restart grafana

collections/ansible_collections/adhdgirl/minilab/roles/grafana/tasks/install.yaml

@@ -0,0 +1,30 @@
+---
+- name: Install required packages
+  ansible.builtin.apt:
+    name:
+      - apt-transport-https
+      - python3-debian
+      - wget
+      - gnupg
+    state: present
+    update_cache: true
+
+- name: Add Grafana APT repository
+  ansible.builtin.deb822_repository:
+    name: grafana
+    types:
+      - deb
+    uris: https://apt.grafana.com
+    suites:
+      - stable
+    components:
+      - main
+    signed_by: https://apt.grafana.com/gpg-full.key
+    state: present
+
+- name: Install Grafana
+  ansible.builtin.apt:
+    name: "grafana{% if grafana_version != 'latest' %}={{ grafana_version }}{% endif %}"
+    state: "{{ 'present' if grafana_version != 'latest' else 'latest' }}"
+    update_cache: true
+  notify: Restart grafana

collections/ansible_collections/adhdgirl/minilab/roles/grafana/tasks/main.yaml

@@ -0,0 +1,39 @@
+---
+- name: Load distro-specific variables
+  ansible.builtin.include_vars: "{{ item }}"
+  tags: always
+  with_first_found:
+    - files:
+        - "{{ ansible_facts['distribution'] }}.yaml"
+      skip: true
+
+- name: Ensure Grafana is running on this device
+  block:
+    - name: Install Grafana
+      ansible.builtin.include_tasks: install.yaml
+    - name: Configure Grafana
+      ansible.builtin.include_tasks: configure.yaml
+    - name: Provision datasources
+      ansible.builtin.include_tasks: datasources.yaml
+    - name: Install plugins
+      ansible.builtin.include_tasks: plugins.yaml
+    - name: Enable and start Grafana
+      ansible.builtin.systemd:
+        name: grafana-server
+        state: started
+        enabled: true
+        daemon_reload: true
+      become: true
+    - name: Wait for Grafana to be ready
+      ansible.builtin.uri:
+        url: "http://localhost:{{ grafana_http_port }}/api/health"
+        status_code: 200
+      register: grafana_health
+      until: grafana_health.status == 200
+      retries: 15
+      delay: 3
+
+  rescue:
+    - name: Set that this task failed # noqa: var-naming[no-role-prefix]
+      ansible.builtin.set_fact:
+        task_failed: true

collections/ansible_collections/adhdgirl/minilab/roles/grafana/tasks/plugins.yaml

@@ -0,0 +1,18 @@
+---
+- name: Ensure presense of folders for grafana
+  ansible.builtin.file:
+    path: /var/lib/grafana/plugins
+    state: directory
+    recurse: true
+    owner: 1000
+    group: 1000
+    mode: "0755"
+- name: Get list of installed plugins
+  ansible.builtin.command: grafana cli plugins ls
+  register: grafana_installed_plugins
+  changed_when: false
+- name: Install Grafana plugins
+  ansible.builtin.command: "grafana cli plugins install {{ item }}"
+  loop: "{{ grafana_plugins }}"
+  when: item not in grafana_installed_plugins.stdout
+  notify: Restart grafana

collections/ansible_collections/adhdgirl/minilab/roles/grafana/templates/datasource.yaml.j2

@@ -0,0 +1,29 @@
+# Datasource provisioning - managed by Ansible
+apiVersion: 1
+
+deleteDatasources: []
+
+datasources:
+{% for ds in grafana_datasources %}
+  - name: {{ ds.name }}
+    type: {{ ds.type }}
+    access: {{ ds.access | default('proxy') }}
+    url: {{ ds.url }}
+    isDefault: {{ ds.is_default | default(false) | string | lower }}
+    editable: {{ ds.editable | default(false) | string | lower }}
+{% if ds.database is defined %}
+    database: {{ ds.database }}
+{% endif %}
+{% if ds.basic_auth is defined and ds.basic_auth %}
+    basicAuth: true
+    basicAuthUser: {{ ds.basic_auth_user }}
+    secureJsonData:
+      basicAuthPassword: {{ ds.basic_auth_password }}
+{% endif %}
+{% if ds.json_data is defined %}
+    jsonData:
+{% for key, value in ds.json_data.items() %}
+      {{ key }}: {{ value }}
+{% endfor %}
+{% endif %}
+{% endfor %}

collections/ansible_collections/adhdgirl/minilab/roles/grafana/templates/grafana.ini.j2

@@ -0,0 +1,48 @@
+# Grafana configuration - managed by Ansible
+
+[server]
+http_addr = {{ grafana_http_addr }}
+http_port = {{ grafana_http_port }}
+domain = {{ grafana_domain }}
+root_url = {{ grafana_root_url }}
+
+[database]
+type = {{ grafana_database_type }}
+{% if grafana_database_type != 'sqlite3' %}
+host = {{ grafana_database_host }}
+name = {{ grafana_database_name }}
+user = {{ grafana_database_user }}
+password = {{ grafana_database_password }}
+{% endif %}
+
+[security]
+admin_user = {{ grafana_admin_user }}
+admin_password = {{ grafana_admin_password }}
+secret_key = {{ grafana_secret_key }}
+disable_gravatar = {{ grafana_disable_gravatar | string | lower }}
+
+[users]
+allow_sign_up = {{ grafana_allow_sign_up | string | lower }}
+
+[auth.anonymous]
+enabled = {{ grafana_anonymous_enabled | string | lower }}
+{% if grafana_anonymous_enabled %}
+org_role = {{ grafana_anonymous_org_role }}
+{% endif %}
+
+{% if grafana_smtp_enabled %}
+[smtp]
+enabled = true
+host = {{ grafana_smtp_host }}
+user = {{ grafana_smtp_user }}
+password = {{ grafana_smtp_password }}
+from_address = {{ grafana_smtp_from_address }}
+{% endif %}
+
+[log]
+mode = {{ grafana_log_mode }}
+level = {{ grafana_log_level }}
+
+[analytics]
+reporting_enabled = false
+check_for_updates = true

collections/ansible_collections/adhdgirl/minilab/roles/grafana/vars/main.yaml

@@ -0,0 +1,25 @@
+---
+grafana_admin_password: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  39643461386537313733333835393232303535366662333135383064326136386637656663656535
+  3731646539653035636338666638313036313164653964310a333930333130376565386538343036
+  36616661623030643232616162346134363331633462663937313065313861333463653331353536
+  6165636162363165360a633930643933613264613332333461653765663032626630313563633333
+  3331
+grafana_domain: "grafana.local.cobb.lgbt"
+grafana_datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://localhost:9090
+    is_default: true
+    editable: false
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://localhost:3100
+    editable: false
+grafana_plugins:
+  - grafana-clock-panel
+  - grafana-polystat-panel
+grafana_secret_key: "4V0H3pCvZAmQ02TDJJoc"

collections/ansible_collections/adhdgirl/minilab/roles/prometheus/tasks/main.yaml

@@ -0,0 +1,28 @@
+---
+- name: Load distro-specific variables
+  ansible.builtin.include_vars: "{{ item }}"
+  tags: always
+  with_first_found:
+    - files:
+        - "{{ ansible_facts['distribution'] }}.yaml"
+      skip: true
+
+- name: Ensure Prometheus monitoring is running on this device
+  block:
+    - name: Ensure Prometheus is installed and configured
+      tags: packages,monitoring,prometheus
+      community.general.apk:
+        name: prometheus-node-exporter
+        update_cache: true
+        state: latest
+    - name: Make sure exporter to run at boot
+      tags: prometheus,monitoring
+      ansible.builtin.service:
+        name: node-exporter
+        enabled: true
+        state: started
+
+  rescue:
+    - name: Set that this task failed # noqa: var-naming[no-role-prefix]
+      ansible.builtin.set_fact:
+        task_failed: true

collections/ansible_collections/adhdgirl/minilab/roles/readeck/files/docker-compose.yaml

@@ -4,7 +4,7 @@ services:
     image: codeberg.org/readeck/readeck:${READECK_IMAGE_TAG:-latest}
     container_name: readeck
     ports:
-      - 8083:8000
+      - 8084:8000
     volumes:
       - /mnt/storage/readeck:/readeck
     restart: unless-stopped

collections/ansible_collections/adhdgirl/minilab/roles/traefik/templates/traefik-compose.j2

@@ -11,6 +11,7 @@ services:
     ports:
       - "80:80"
       - "443:443"
+      - "7000:7000"
     env_file:
       - path: /opt/traefik/cloudflare.env
         required: true

collections/ansible_collections/adhdgirl/minilab/roles/traefik/templates/traefik.yaml.j2

@@ -12,7 +12,11 @@ api:
   dashboard: true
   insecure: false
 metrics:
-  prometheus: {}
+  prometheus:
+    addEntryPointsLabels: true
+    addRoutersLabels: true
+    addServicesLabels: true
+    entryPoint: "metrics"
 
 providers:
   docker:
@@ -47,6 +51,9 @@ entryPoints:
     address: :443
     http:
       tls: {}
+  
+  metrics:
+    address: :7000
       
 certificatesResolvers:
   letsEncrypt:

collections/requirements.yaml

@@ -5,3 +5,4 @@ collections:
   # - name: ansibleguy.nftables
   #   version: 1.0.3
   - name: community.docker
+  - name: prometheus.prometheus

inventory/host_vars/super.yaml

@@ -0,0 +1,76 @@
+---
+server_name: super
+ansible_python_interpreter: /usr/bin/python3
+prometheus_scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets:
+          - "localhost:9090"
+  - job_name: "adguard_node"
+    static_configs:
+      - targets:
+          - "adguard.local.cobb.lgbt:9100"
+  - job_name: "stephanie_node"
+    static_configs:
+      - targets:
+          - "stephanie.local.cobb.lgbt:9100"
+  - job_name: "nemetona_node"
+    static_configs:
+      - targets:
+          - "nemetona.local.cobb.lgbt:9100"
+  - job_name: "ida_node"
+    static_configs:
+      - targets:
+          - "ida.local.cobb.lgbt:9100"
+  - job_name: "emosen_node"
+    static_configs:
+      - targets:
+          - "emosen.local.cobb.lgbt:9100"
+  - job_name: "adguardpi_node"
+    static_configs:
+      - targets:
+          - "adguardpi.local.cobb.lgbt:9100"
+  - job_name: "knivi_node"
+    static_configs:
+      - targets:
+          - "knivi.local.cobb.lgbt:9100"
+  - job_name: "reir_node"
+    static_configs:
+      - targets:
+          - "reir.local.cobb.lgbt:9100"
+  - job_name: "curren_node"
+    static_configs:
+      - targets:
+          - "curren.local.cobb.lgbt:9100"
+  - job_name: "pve_node"
+    static_configs:
+      - targets:
+          - "10.69.1.7:9100"
+          - "10.69.1.7:9221"
+  - job_name: "snmp"
+    static_configs:
+      - targets:
+          - "10.69.1.1"
+          - "10.69.1.10"
+          - "10.69.1.11"
+          - "10.69.1.126"
+    metrics_path: /snmp
+    params:
+      auth: [public_v2]
+      module: [if_mib]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9116
+  - job_name: "snmp_exporter"
+    static_configs:
+      - targets: ["localhost:9116"]
+  - job_name: "traefik"
+    static_configs:
+      - targets:
+          - "nemetona.local.cobb.lgbt:7000"
+    metrics_path: /metrics
+prometheus_db_dir: /mnt/storage/db

inventory/hosts.yaml

@@ -15,6 +15,8 @@ all:
       ansible_host: 10.69.10.131
     stephanie:
       ansible_host: 10.69.10.52
+    super:
+      ansible_host: 10.69.1.31
     nemetona:
       ansible_host: 10.69.10.50
     ida:
@@ -37,6 +39,7 @@ all:
         nemetona:
         ida:
         emosen:
+        home_assistant:
     debian:
       hosts:
         adguardpi:
@@ -44,6 +47,7 @@ all:
         knivi:
         reir:
         curren:
+        super:
     docker:
       hosts:
         adguard:
@@ -76,3 +80,6 @@ all:
     frigate:
       hosts:
         curren:
+    prometheus:
+      hosts:
+        super:

minilab.yaml

@@ -23,6 +23,18 @@
   become: true
   roles:
     - adhdgirl.minilab.common
+- name: Ensure proper config for Promotheus node-exporter (Alpine)
+  hosts: alpine
+  tags: monitoring,prometheus,exporter,alpine
+  become: true
+  roles:
+    - adhdgirl.minilab.prometheus
+- name: Ensure proper config for Prometheus node-exporter (Debian)
+  hosts: debian,pve,!mors
+  tags: monitoring,prometheus,exporter,debian
+  become: true
+  roles:
+    - prometheus.prometheus.node_exporter
 - name: Ensure docker is installed and running
   hosts: docker
   tags: docker
@@ -70,3 +82,11 @@
   become: true
   roles:
     - adhdgirl.minilab.frigate
+- name: Configure and install Prometheus
+  hosts: super
+  tags: monitoring,prometheus
+  become: true
+  roles:
+    - prometheus.prometheus.prometheus
+    - prometheus.prometheus.snmp_exporter
+    - adhdgirl.minilab.grafana

setup_new_machine

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+clear
+echo "" > ansible.log
+ANSIBLE_PATH=ansible-playbook
+
+if hash ansible-playbook 2>/dev/null; then
+  ANSIBLE_PATH=ansible-playbook
+else
+  ANSIBLE_PATH=./.venv/bin/ansible-playbook
+fi
+
+$ANSIBLE_PATH --vault-password-file vault-password.txt -e 'ansible_user=root' -e 'ansible_private_key_file=/workspaces/minilab/ansible_ed25519' -e 'ansible_become_method=sudo' minilab.yaml $@

update.yaml

@@ -1,6 +1,6 @@
 ---
 - name: Update and reboot all hosts
-  hosts: all, !stephanie
+  hosts: all, !stephanie, !pve
   gather_facts: true
   become: true