Monitoring Exporters and Grafana Installation Guide

# If repository is already cloned:
cd /opt/grafana
sudo bash /opt/grafana/misc/install-docker.sh

# Or via one-command bootstrap (installs Docker automatically):
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- PROMETHEUS_IP

docker -v
docker compose version

# Auto-detect services and deploy (recommended):
cd /opt/grafana
sudo bash /opt/grafana/misc/quick-deploy.sh PROMETHEUS_IP

# Examples:
sudo bash /opt/grafana/misc/quick-deploy.sh 65.108.67.167

# Or deploy specific exporters manually (--exporter flag):
sudo bash /opt/grafana/misc/quick-deploy.sh 65.108.67.167 --exporter base --exporter jmx-wildfly
sudo bash /opt/grafana/misc/quick-deploy.sh 65.108.67.167 --exporter base --exporter mysql

# Or use one-command bootstrap (installs Docker, clones repo, deploys):
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- 65.108.67.167

/opt
└── grafana
    ├── exporters/                    # Modular exporter directories
    │   ├── base/                     # Node + Process + cAdvisor (all servers)
    │   │   ├── docker-compose.yml
    │   │   ├── process-exporter.yml
    │   │   └── .env.example
    │   ├── jmx-wildfly/             # HikariCP exporter (5556) + JVM exporter
    │   │                            #   (5557, compose profile 'jvm', bare-metal only)
    │   │   ├── docker-compose.yml
    │   │   ├── config-hikari.yml
    │   │   ├── config-jvm.yml
    │   │   └── .env.example
    │   ├── mysql/                    # MySQL/MariaDB exporter
    │   │   ├── docker-compose.yml
    │   │   └── .env.example
    │   ├── oracle/                   # Oracle DB exporter
    │   │   ├── docker-compose.yml
    │   │   ├── config.yaml.template
    │   │   ├── generate-config.sh
    │   │   └── .env.example
    │   ├── clickhouse/               # ClickHouse native Prometheus endpoint (no exporter container)
    │   │   ├── clickhouse-prometheus.xml   # config.d snippet (port 9363)
    │   │   └── install-native.sh           # installs the config + optional restart
    │   ├── postgres/                 # PostgreSQL exporter
    │   │   ├── docker-compose.yml
    │   │   └── .env.example
    │   └── nginx/                    # Nginx exporter (for Angular UI)
    │       ├── docker-compose.yml
    │       └── .env.example
    ├── prometheus-grafana-stack/     # Monitoring stack
    │   ├── docker-compose.yml
    │   ├── .env.example
    │   └── prometheus/
    │       ├── prometheus.yml
    │       ├── scripts/
    │       │   └── generate-targets.sh
    │       └── targets/
    │           └── servers.env.example
    ├── docs/
    │   └── architecture.md           # Architecture diagrams
    ├── misc/
    │   ├── bootstrap.sh              # One-command installer
    │   ├── quick-deploy.sh           # Auto-detect and deploy exporters
    │   ├── detect-services.sh        # Service auto-detection
    │   ├── diagnose.sh               # Troubleshooting tool
    │   └── install-docker.sh         # Docker installer
    ├── README.md
    └── sizing.md                     # Capacity planning guide

# Auto-detect services and deploy (recommended)
cd /opt/grafana
sudo bash /opt/grafana/misc/quick-deploy.sh 65.109.58.165

# Base exporters (Node + Process) - all servers
cd /opt/grafana/exporters/base

# Hazelcast: No exporter needed - uses native Prometheus endpoint (port 9101)
# See section "Prerequisites: Enable Prometheus on Hazelcast Servers"

# JMX exporters for WildFly/FTACS
cd /opt/grafana/exporters/jmx-wildfly

cp .env.example .env
vi .env

docker compose up -d

docker compose logs -f

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9100 -j ACCEPT   # node-exporter
iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9256 -j ACCEPT   # process-exporter
iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9183 -j ACCEPT   # cadvisor
# JMX ports (if WildFly detected):
iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 5556 -j ACCEPT   # jmx-hikari
iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 5557 -j ACCEPT   # jmx-jvm
# Hazelcast native Prometheus (if Hazelcast detected):
iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9101 -j ACCEPT   # hazelcast native

iptables -A INPUT -p tcp -s 65.109.58.165 --dport 9100 -j ACCEPT   # node-exporter
iptables -A INPUT -p tcp -s 65.109.58.165 --dport 9256 -j ACCEPT   # process-exporter
iptables -A INPUT -p tcp -s 65.109.58.165 --dport 9183 -j ACCEPT   # cadvisor
# JMX ports (if WildFly detected):
iptables -A INPUT -p tcp -s 65.109.58.165 --dport 5556 -j ACCEPT   # jmx-hikari
iptables -A INPUT -p tcp -s 65.109.58.165 --dport 5557 -j ACCEPT   # jmx-jvm
# Hazelcast native Prometheus (if Hazelcast detected):
iptables -A INPUT -p tcp -s 65.109.58.165 --dport 9101 -j ACCEPT   # hazelcast native

cd /opt/grafana

# Auto-detect (recommended) - deploys appropriate exporters based on detected services
sudo bash /opt/grafana/misc/quick-deploy.sh 65.108.67.167

# Or deploy specific exporters manually:
# For WildFly/FTACS servers:
sudo bash /opt/grafana/misc/quick-deploy.sh 65.108.67.167 --exporter base --exporter jmx-wildfly

# Check containers are running
docker ps | grep jmx

# Test metrics endpoints
# WildFly:
curl -s http://localhost:5556/metrics | head  # HikariCP
curl -s http://localhost:5557/metrics | head  # JVM

# Hazelcast:
curl -s http://localhost:9101/metrics | head  # Hazelcast

JAVA_OPTS="$JAVA_OPTS \
    -Dcom.sun.management.jmxremote \
    -Dcom.sun.management.jmxremote.port=9999 \
    -Dcom.sun.management.jmxremote.rmi.port=9999 \
    -Dcom.sun.management.jmxremote.ssl=false \
    -Dcom.sun.management.jmxremote.authenticate=false \
    -Djava.rmi.server.hostname=127.0.0.1"

registerMbeans=true

# For MySQL:
service jboss restart

# For Oracle:
service jboss restartoracle

export PROMETHEUS_PORT=9101

  -e PROMETHEUS_PORT=9101 \   # Enable native Prometheus metrics
  -p 9101:9101 \              # Expose metrics port to host

# If you already have Docker installed and repository cloned:
cd /opt/grafana
sudo bash /opt/grafana/misc/quick-deploy.sh 65.109.58.165 --exporter base --exporter mysql

# Or use one-command bootstrap (auto-detects MySQL):
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- 65.109.58.165

# Uncomment to override default port (9104)
# - '--web.listen-address=:19104'

CREATE USER IF NOT EXISTS 'exporter'@'%' IDENTIFIED BY 'StR0nG_PassW0rD_hErE' WITH MAX_USER_CONNECTIONS 3;
GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'exporter'@'%';

cd /opt/grafana/exporters/mysql/
cp .env.example .env
vi .env

# MySQL Connection Details (REQUIRED)
MYSQL_HOST=localhost
MYSQL_PORT=3306
MYSQL_USER=exporter
MYSQL_PASSWORD=StR0nG_PassW0rD_hErE  # Replace with your password

docker compose up -d

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9104 -j ACCEPT

# If you already have Docker installed and repository cloned:
cd /opt/grafana
sudo bash /opt/grafana/misc/quick-deploy.sh 65.109.58.165 --exporter base --exporter oracle

# Or use one-command bootstrap (auto-detects Oracle):
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- 65.109.58.165

cd /opt/grafana/exporters/oracle/

cp .env.example .env
vi .env

# Oracle Database Connection (REQUIRED)
ORACLE_HOST=localhost
ORACLE_PORT=1521
ORACLE_SERVICE_NAME=ORCL
ORACLE_USER=system
ORACLE_PASSWORD=your_oracle_password

./generate-config.sh

docker compose up -d

docker compose logs -f

curl http://localhost:9161/metrics

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9161 -j ACCEPT

iptables -A INPUT -p tcp -s 65.109.58.165 --dport 9161 -j ACCEPT

cd /opt/grafana/exporters/oracle
cp .env.example .env
vi .env

ORACLE_HOST=<database-server-ip>    # Remote database server IP (not localhost)
ORACLE_PORT=1521
ORACLE_SERVICE_NAME=<service-name>
ORACLE_USER=<username>
ORACLE_PASSWORD=<password>

cd /opt/grafana/exporters/oracle
bash /opt/grafana/exporters/oracle/generate-config.sh
docker compose up -d

curl http://localhost:9161/metrics | grep oracledb_up
# Expected: oracledb_up 1

cd /opt/grafana/prometheus-grafana-stack/prometheus/targets
vi servers.env
# Add line:
ORACLE 127.0.0.1:9161 <instance-name>

# Regenerate targets:
bash /opt/grafana/prometheus-grafana-stack/prometheus/scripts/generate-targets.sh

hazelcast:
    image: hub.friendly-tech.com/components/hazelcast:ft.5.5.0
    ports:
      - "5701:5701"
      - "5711:5711"
      - "9101:9101"
    environment:
      PROMETHEUS_PORT: 9101

hazelcast:
    image: hub.friendly-tech.com/components/hazelcast:ft.5.5.0
    network_mode: host
    environment:
      PROMETHEUS_PORT: 9101

# Should return metrics immediately (no hang)
curl -s http://localhost:9101/metrics | head

# Auto-detect and deploy all exporters (recommended):
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- PROMETHEUS_IP

# Or if repository already cloned:
cd /opt/grafana
sudo bash /opt/grafana/misc/quick-deploy.sh PROMETHEUS_IP

location /nginx_status {
    stub_status on;
    access_log off;
    allow 127.0.0.1;
    deny all;
}

# Check where nginx config is located inside the container
docker exec portals nginx -T 2>/dev/null | head -5

# Common locations:
#   /etc/nginx/conf.d/default.conf
#   /etc/nginx/nginx.conf

# View the current config
docker exec portals cat /etc/nginx/conf.d/default.conf

# Edit: copy config out, add the location block, copy back
docker cp portals:/etc/nginx/conf.d/default.conf ./default.conf
# ... add the location /nginx_status block inside the server { } section ...
docker cp ./default.conf portals:/etc/nginx/conf.d/default.conf
docker exec portals nginx -s reload

docker exec portals curl -s http://127.0.0.1/nginx_status
# Expected output:
#   Active connections: 5
#   server accepts handled requests
#
1234 1234 5678
#   Reading: 0 Writing: 1 Waiting: 4

cd /opt/grafana/exporters/nginx
cp .env.example .env
vi .env

NGINX_HOST=localhost  # or container name if using Docker network
NGINX_PORT=8880
NGINX_EXPORTER_PORT=9113

docker compose up -d

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9113 -j ACCEPT

curl http://localhost:9113/metrics | grep nginx_

sudo bash /opt/grafana/exporters/clickhouse/install-native.sh --restart

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9363 -j ACCEPT

CLICKHOUSE  <server-ip>:9363  <instance>  role=database

curl -s http://localhost:9363/metrics | grep -E '^ClickHouse(Metrics|ProfileEvents|AsyncMetrics)_'

-- Use a generated password, not a memorable one; this account is reachable
-- from wherever the database is. e.g.  openssl rand -base64 24
CREATE USER exporter WITH PASSWORD 'REPLACE_WITH_GENERATED_PASSWORD';
GRANT pg_monitor TO exporter;
GRANT CONNECT ON DATABASE ftl_ai TO exporter;

postgres:
  ports:
    - "127.0.0.1:5432:5432"   # loopback ONLY — the exporter reaches it, the internet does not

iptables -I DOCKER-USER -p tcp --dport 5432 ! -s <allowed-ip>/32 -j DROP

cd /opt/grafana/exporters/postgres
# find the database's network
docker inspect <postgres-container> --format '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}'
# in .env:
#   POSTGRES_DOCKER_NETWORK=ft-ui-services_default
#   POSTGRES_HOST=postgres
# service name, not 127.0.0.1
#   POSTGRES_PORT=5432
docker compose -f docker-compose.yml -f docker-compose.network.yml up -d

cd /opt/grafana/exporters/postgres
cp .env.example .env
vi .env

POSTGRES_HOST=localhost
POSTGRES_PORT=5432
POSTGRES_USER=exporter
POSTGRES_PASSWORD=<the generated password from Step 1>
POSTGRES_DB=ftl_ai
POSTGRES_EXPORTER_PORT=9187

docker compose up -d

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9187 -j ACCEPT

curl http://localhost:9187/metrics | grep pg_

curl http://localhost:8383/api/actuator/prometheus | grep jvm_

FT_DEVICE_NETWORK 65.109.24.154:8383 DevOps-HC1

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 8383 -j ACCEPT

  ui-ai-agent:
    ports:
      - "8084:8080"

docker compose up -d ui-ai-agent

curl http://localhost:8084/metrics | grep python_

AI_AGENT 65.109.24.154:8084 DevOps-HC1

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 8084 -j ACCEPT

.requestMatchers("/actuator/**").permitAll()

management.endpoints.web.exposure.include=prometheus,health,info

curl http://localhost:8881/actuator/prometheus | grep jvm_

UI_BACKEND 65.109.24.154:8881 DevOps-HC1

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 8881 -j ACCEPT

lodctr.exe /E:Lsa
lodctr.exe /E:PerfProc
lodctr.exe /R

msiexec /i "C:\Users\Administrator\windows_exporter-0.31.3-amd64.msi" ENABLED_COLLECTORS=ad,adfs,cache,cpu,cpu_info,container,dfsr,dhcp,dns,fsrmquota,iis,logical_disk,memory,msmq,mssql,net,os,process,remote_fx,service,tcp,time,vmware LISTEN_PORT=9100 EXTRA_FLAGS="--collector.process.include=""(w3wp|java.*)"" --collector.process.iis --collector.process.counter-version=2"

netsh advfirewall firewall add rule name="Allow Windows Exporter" dir=in action=allow protocol=TCP localport=9100

sc query windows_exporter

# === IIS SERVERS ===
NODE <server-ip>:9100 <Prefix>-IIS service=IIS,role=webserver,node_ip=<server-ip>

cd /opt/grafana/prometheus-grafana-stack/prometheus
/opt/grafana/prometheus-grafana-stack/prometheus/scripts/generate-targets.sh

curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- stack

cd /opt/grafana/prometheus-grafana-stack
cp .env.example .env
vi .env

# Server Configuration (REQUIRED)
SERVER_IP=65.109.58.165  # Replace with your server's IP

# Grafana Admin Credentials (REQUIRED)
# NOTE: Password in .env is used only on FIRST deployment.
# After that, database is the source of truth.
# To change password later, use: docker exec grafana grafana cli admin reset-admin-password NEW_PASS
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=your_secure_password_here

# SMTP Configuration (REQUIRED for alerting)
SMTP_ENABLED=true
SMTP_HOST=smtp.example.com:587
SMTP_USER=your_smtp_user
SMTP_PASSWORD=your_smtp_password

# Grafana Service Token (REQUIRED for nginx proxy)
GRAFANA_SERVICE_TOKEN=your_token_here  # Will be generated in section 8.2

cd /opt/grafana/prometheus-grafana-stack

# Alertmanager config placeholder (regenerated by the container entrypoint)
cat > alertmanager/alertmanager.yml << 'EOF'
# Placeholder - will be regenerated by alertmanager entrypoint.sh
global:
  resolve_timeout: 5m
route:
  receiver: 'default'
receivers:
  - name: 'default'
EOF
chmod 666 alertmanager/alertmanager.yml

# TLS certificate for the nginx proxy (self-signed; replace with a real
# certificate later if you have one — files: ssl/certificate.crt, ssl/private.key).
# ssl/ is git-ignored, so it is NOT in the ZIP — create it first.
mkdir -p ssl
openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
  -keyout ssl/private.key -out ssl/certificate.crt \
  -subj "/C=US/ST=State/L=City/O=FriendlyTech/CN=${GRAFANA_DOMAIN:-$(hostname -f)}"

# Prometheus basic-auth file for nginx (user: prometheus). Also git-ignored, so
# nginx would otherwise mount a directory onto it and fail. htpasswd may be
# absent on RHEL — fall back to openssl apr1; write the password back into .env.
PROM_PASS=$(openssl rand -base64 12 | tr -dc 'a-zA-Z0-9' | head -c16)
if command -v htpasswd >/dev/null 2>&1; then
  htpasswd -bc .htpasswd prometheus "$PROM_PASS"
else
  echo "prometheus:$(openssl passwd -apr1 "$PROM_PASS")" > .htpasswd
fi
grep -q '^PROMETHEUS_PASSWORD=' .env \
  && sed -i "s/^PROMETHEUS_PASSWORD=.*/PROMETHEUS_PASSWORD=$PROM_PASS/" .env \
  || echo "PROMETHEUS_PASSWORD=$PROM_PASS" >> .env
echo "Prometheus UI login: prometheus / $PROM_PASS"

# nginx ACME webroot (bind-mounted; keep it a directory)
mkdir -p acme-challenge

cd /opt/grafana/prometheus-grafana-stack
docker compose up -d

iptables -A INPUT -p tcp --dport 80 -j ACCEPT
iptables -A INPUT -p tcp --dport 443 -j ACCEPT
# If server has IPv6 address, also add ip6tables rules:
ip6tables -A INPUT -p tcp --dport 80 -j ACCEPT
ip6tables -A INPUT -p tcp --dport 443 -j ACCEPT

cd /opt/grafana/prometheus-grafana-stack

# View Grafana password
grep GRAFANA_ADMIN_PASSWORD .env

# View Prometheus password
grep PROMETHEUS_PASSWORD .env

# Reset Grafana password if forgotten
docker exec grafana grafana cli admin reset-admin-password NEW_PASSWORD

cd /opt/grafana/prometheus-grafana-stack/prometheus/targets
vi servers.env

# MySQL database server
NODE 10.0.0.10:9100 Prod-DB
PROCESS 10.0.0.10:9256 Prod-DB
CADVISOR 10.0.0.10:9183 Prod-DB
MYSQL 10.0.0.10:9104 Prod-DB

# WildFly/ACS application server
NODE 10.0.0.20:9100 Prod-ACS1
PROCESS 10.0.0.20:9256 Prod-ACS1
CADVISOR 10.0.0.20:9183 Prod-ACS1
JMX_HIKARI 10.0.0.20:5556 Prod-ACS1
JMX_JVM 10.0.0.20:5557 Prod-ACS1

# Hazelcast cache server (standalone)
NODE 10.0.0.30:9100 Prod-HC1 role=cache
PROCESS 10.0.0.30:9256 Prod-HC1
CADVISOR 10.0.0.30:9183 Prod-HC1
JMX_HC 10.0.0.30:9101 Prod-HC1

# API server (microservices)
NODE 10.0.0.35:9100 Prod-API1 role=api
PROCESS 10.0.0.35:9256 Prod-API1
CADVISOR 10.0.0.35:9183 Prod-API1
PROVISION_API 10.0.0.35:8091 Prod-API1
NORTHBOUND_API 10.0.0.35:9880 Prod-API1
SERVICE_API 10.0.0.35:8085 Prod-API1
SUBSCRIPTION_API 10.0.0.35:8080 Prod-API1

# Oracle database server
NODE 10.0.0.40:9100 Prod-OracleDB
PROCESS 10.0.0.40:9256 Prod-OracleDB
CADVISOR 10.0.0.40:9183 Prod-OracleDB
ORACLE 10.0.0.40:9161 Prod-OracleDB

# Windows/IIS web server
NODE 10.0.0.50:9100 Prod-IIS service=IIS,role=webserver

# === DATABASE SERVERS ===
NODE 65.109.58.165:9100 DevOps-DB service=mysql,role=database
PROCESS 65.109.58.165:9256 DevOps-DB
CADVISOR 65.109.58.165:9183 DevOps-DB
MYSQL 65.109.58.165:9104 DevOps-DB

# === ACS SERVERS ===
NODE 65.109.58.164:9100 DevOps-ACS1 service=ACS,role=application
PROCESS 65.109.58.164:9256 DevOps-ACS1
CADVISOR 65.109.58.164:9183 DevOps-ACS1
JMX_HIKARI 65.109.58.164:5556 DevOps-ACS1
JMX_JVM 65.109.58.164:5557 DevOps-ACS1
# Optional ACS endpoints:
FT_SYSTEM 65.109.58.164:8090 DevOps-ACS1
ACS_METRICS 65.109.58.164:8080 DevOps-ACS1
JMX_HC 65.109.58.164:9101 DevOps-ACS1

# === HAZELCAST SERVERS (standalone) ===
NODE 65.109.24.154:9100 DevOps-HC1 service=hazelcast,role=cache
PROCESS 65.109.24.154:9256 DevOps-HC1
CADVISOR 65.109.24.154:9183 DevOps-HC1
JMX_HC 65.109.24.154:9101 DevOps-HC1

# === API SERVERS (microservices) ===
NODE 65.109.24.155:9100 DevOps-API1 service=api,role=api
PROCESS 65.109.24.155:9256 DevOps-API1
CADVISOR 65.109.24.155:9183 DevOps-API1
PROVISION_API 65.109.24.155:8091 DevOps-API1
NORTHBOUND_API 65.109.24.155:9880 DevOps-API1
SERVICE_API 65.109.24.155:8085 DevOps-API1
SUBSCRIPTION_API 65.109.24.155:8080 DevOps-API1

# === ORACLE SERVERS ===
NODE 65.109.20.174:9100 DevOps-OracleDB service=oracle,role=database
PROCESS 65.109.20.174:9256 DevOps-OracleDB
CADVISOR 65.109.20.174:9183 DevOps-OracleDB
ORACLE 65.109.20.174:9161 DevOps-OracleDB

# === IIS SERVERS ===
NODE 65.109.49.150:9100 devops-ui.friendly-tech.com service=IIS,role=webserver,node_ip=65.109.49.150

cd /opt/grafana/prometheus-grafana-stack/prometheus
/opt/grafana/prometheus-grafana-stack/prometheus/scripts/generate-targets.sh

# show what did not answer this run; changes nothing
bash /opt/grafana/prometheus-grafana-stack/prometheus/scripts/generate-targets.sh --park-dry-run

# comment those rows out, keeping the original line, with a backup
bash /opt/grafana/prometheus-grafana-stack/prometheus/scripts/generate-targets.sh --park

curl -s -o /dev/null -w '%{http_code}\n' \
  http://<ip>:8091/prov-portal/actuator/prometheus     # not /metrics

# prometheus.yml already has this for each job:
relabel_configs:
  - source_labels: [__address__]
    target_label: node_ip
    regex: (.+):.*      # extracts IP from "IP:PORT"
    replacement: $1

cd /opt/grafana/prometheus-grafana-stack
vi .env

# Example: allow access from Angular UI portal
UI_PORTAL_ORIGINS=https://portal.example.com,https://10.0.0.50:8880

grep GRAFANA_SERVICE_TOKEN .env

docker compose down && docker compose up -d

cd /opt/grafana
sudo bash /opt/grafana/misc/generate-grafana-token.sh --restart

cd /opt/grafana/prometheus-grafana-stack
vi .env
# Set: GRAFANA_SERVICE_TOKEN=glsa_YourActualTokenHere_12345678

<iframe
  src="/d/business-metrics/business-metrics?orgId=1&kiosk&refresh=30s"
  width="100%"
  height="600"
  frameborder="0">
</iframe>

# Should return 200 OK without authentication
curl -kI https://YOUR_SERVER_DOMAIN/grafana-ro/

curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- stack

# Configure alert notifications? [y/N]: y
# Select channels: 1,2  (Email + Telegram)

cd /opt/grafana/prometheus-grafana-stack
vi .env

ALERT_EMAIL=alerts@company.com

# Start MailPit (auto-started when option 3 is selected)
docker compose --profile testing up -d mailpit

# Web UI to view emails (opens automatically in bootstrap output)
open http://<server-ip>:8025

TELEGRAM_BOT_TOKEN=123456789:ABCdefGHI...
TELEGRAM_CHAT_ID=-1001234567890

SLACK_WEBHOOK_URL=https://hooks.slack.com/services/T00/B00/XXXX
SLACK_CHANNEL=#alerts

MSTEAMS_WEBHOOK_URL=https://outlook.office.com/webhook/...

APP_WEBHOOK_URL=https://your-app.com/api/alerts

SMS_GATEWAY_URL=https://api.twilio.com/...
SMS_API_TOKEN=your-token

# SNMPv2c
SNMP_TRAP_ADDRESS=nms.company.com:162
SNMP_VERSION=V2c
SNMP_COMMUNITY=public

# SNMPv3 (authentication + encryption)
SNMP_VERSION=V3
SNMP_AUTH_USERNAME=snmpuser
SNMP_AUTH_PASSWORD=authpassword
SNMP_PRIV_PASSWORD=privpassword

# Start SNMP services (auto-started when option 2 is selected)
docker compose --profile snmp up -d

# Web UI to view traps (opens automatically in bootstrap output)
open http://<server-ip>:8089

# Show help
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- --help

# Show available versions
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- --version-list

curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- stack --branch 1.6.6

cat /opt/grafana/VERSION

curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- --version-list

cd /opt/grafana/prometheus-grafana-stack
docker compose down
cp -r prometheus_data /backup/prometheus_data_$(date +%Y%m%d)
cp -r grafana_data /backup/grafana_data_$(date +%Y%m%d)

ss -ltpn | grep <port>

docker ps
docker logs -f <CONTAINER_ID>

docker exec $(docker ps | grep prom/prometheus | awk '{print $1}') \
  wget http://<EXPORTER_IP>:<PORT>/metrics -O /tmp/test

# On the Grafana server:
bash /opt/grafana/misc/bootstrap.sh --reset-password "new_password"

# Or directly:
docker exec grafana grafana cli admin reset-admin-password "new_password"

# Ubuntu/Debian
netfilter-persistent save

# CentOS/Rocky
service iptables save

docker compose down
docker compose up -d

systemctl status mysql
mysql -u exporter -p -e "SHOW GRANTS;"

docker logs nginx_proxy
docker exec nginx_proxy env | grep GRAFANA_SERVICE_TOKEN
docker compose restart nginx

docker info | grep -E 'Storage Driver|driver-type'   # overlayfs + io.containerd.snapshotter.v1
docker inspect --format='{{.Config.Image}}' cadvisor  # must be >= v0.54.0
docker logs --since 10m cadvisor 2>&1 | grep -c 'read-write layer ID'
docker logs --since 10m cadvisor 2>&1 | grep 'docker container factory'
source /opt/grafana/misc/lib/container-helpers.sh && detect_containerd_sock

cd /opt/grafana/exporters/base
sed -i 's|^CADVISOR_VERSION=.*|CADVISOR_VERSION=v0.55.1|' .env   # or add the line if missing
source /opt/grafana/misc/lib/container-helpers.sh
SOCK=$(detect_containerd_sock)
grep -q '^CONTAINERD_SOCK=' .env \
  && sed -i "s|^CONTAINERD_SOCK=.*|CONTAINERD_SOCK=$SOCK|" .env \
  || echo "CONTAINERD_SOCK=$SOCK" >> .env
docker compose up -d cadvisor

curl -fLO https://hub.friendly-tech.com/docs/pdf/cadvisor-v0.55.1.tar.gz
sha256sum cadvisor-v0.55.1.tar.gz
# expect 3439641369dea7121fc3ed1fb2722e979421b0bab3c574d8115fd12d977aa204
# transfer, then on the target host:
docker load -i cadvisor-v0.55.1.tar.gz

docker pull gcr.io/cadvisor/cadvisor:v0.55.1
docker save gcr.io/cadvisor/cadvisor:v0.55.1 -o cadvisor-v0.55.1.tar
tar tf cadvisor-v0.55.1.tar | grep -m1 manifest.json   # classic format, loads anywhere

docker logs --since 3m cadvisor 2>&1 | grep -c 'read-write layer ID'       # expect 0
curl -s 127.0.0.1:9183/metrics | grep -o 'name="[^"]*"' | sort -u | wc -l  # expect your container count

# add both keys to /etc/docker/daemon.json, keeping any existing settings:
#   {"features": {"containerd-snapshotter": false}, "storage-driver": "overlay2"}
systemctl restart docker
docker info | grep 'Storage Driver'   # should show overlay2
# air-gapped: re-load images before starting anything
docker load -i docker-images-bundle.tar.gz
cd /opt/grafana/exporters/base && docker compose up -d

docker logs grafana | grep -i smtp
# Test in UI: Alerting > Contact points > email > Test

cd /opt/grafana/prometheus-grafana-stack
docker compose exec alertmanager-config-ui sh -c 'id; ls -l /app/.env'
# root:root 0644 -> the grant is missing; re-run bootstrap, or:
chgrp "$(getent group docker | cut -d: -f3)" .env && chmod 0660 .env

docker compose up -d alertmanager snmp-notifier

# List available versions
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- --version-list

# Install specific version
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- stack --branch 1.6.4