Runbook: Deploy the Monitoring Stack

curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- stack

cd /opt/grafana/prometheus-grafana-stack
cp .env.example .env
vi .env

# Server Configuration (REQUIRED)
SERVER_IP=65.109.58.165  # Replace with your server's IP

# Grafana Admin Credentials (REQUIRED)
# NOTE: Password in .env is used only on FIRST deployment.
# After that, database is the source of truth.
# To change password later, use: docker exec grafana grafana cli admin reset-admin-password NEW_PASS
GRAFANA_ADMIN_USER=admin
GRAFANA_ADMIN_PASSWORD=your_secure_password_here

# SMTP Configuration (REQUIRED for alerting)
SMTP_ENABLED=true
SMTP_HOST=smtp.example.com:587
SMTP_USER=your_smtp_user
SMTP_PASSWORD=your_smtp_password

# Grafana Service Token (REQUIRED for nginx proxy)
GRAFANA_SERVICE_TOKEN=your_token_here  # Will be generated in section 8.2

cd /opt/grafana/prometheus-grafana-stack

# Alertmanager config placeholder (regenerated by the container entrypoint)
cat > alertmanager/alertmanager.yml << 'EOF'
# Placeholder - will be regenerated by alertmanager entrypoint.sh
global:
  resolve_timeout: 5m
route:
  receiver: 'default'
receivers:
  - name: 'default'
EOF
chmod 666 alertmanager/alertmanager.yml

# TLS certificate for the nginx proxy (self-signed; replace with a real
# certificate later if you have one — files: ssl/certificate.crt, ssl/private.key).
# ssl/ is git-ignored, so it is NOT in the ZIP — create it first.
mkdir -p ssl
openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
  -keyout ssl/private.key -out ssl/certificate.crt \
  -subj "/C=US/ST=State/L=City/O=FriendlyTech/CN=${GRAFANA_DOMAIN:-$(hostname -f)}"

# Prometheus basic-auth file for nginx (user: prometheus). Also git-ignored, so
# nginx would otherwise mount a directory onto it and fail. htpasswd may be
# absent on RHEL — fall back to openssl apr1; write the password back into .env.
PROM_PASS=$(openssl rand -base64 12 | tr -dc 'a-zA-Z0-9' | head -c16)
if command -v htpasswd >/dev/null 2>&1; then
  htpasswd -bc .htpasswd prometheus "$PROM_PASS"
else
  echo "prometheus:$(openssl passwd -apr1 "$PROM_PASS")" > .htpasswd
fi
grep -q '^PROMETHEUS_PASSWORD=' .env \
  && sed -i "s/^PROMETHEUS_PASSWORD=.*/PROMETHEUS_PASSWORD=$PROM_PASS/" .env \
  || echo "PROMETHEUS_PASSWORD=$PROM_PASS" >> .env
echo "Prometheus UI login: prometheus / $PROM_PASS"

# nginx ACME webroot (bind-mounted; keep it a directory)
mkdir -p acme-challenge

cd /opt/grafana/prometheus-grafana-stack
docker compose up -d

iptables -A INPUT -p tcp --dport 80 -j ACCEPT
iptables -A INPUT -p tcp --dport 443 -j ACCEPT
# If server has IPv6 address, also add ip6tables rules:
ip6tables -A INPUT -p tcp --dport 80 -j ACCEPT
ip6tables -A INPUT -p tcp --dport 443 -j ACCEPT

cd /opt/grafana/prometheus-grafana-stack

# View Grafana password
grep GRAFANA_ADMIN_PASSWORD .env

# View Prometheus password
grep PROMETHEUS_PASSWORD .env

# Reset Grafana password if forgotten
docker exec grafana grafana cli admin reset-admin-password NEW_PASSWORD

cd /opt/grafana/prometheus-grafana-stack/prometheus/targets
vi servers.env

# MySQL database server
NODE 10.0.0.10:9100 Prod-DB
PROCESS 10.0.0.10:9256 Prod-DB
CADVISOR 10.0.0.10:9183 Prod-DB
MYSQL 10.0.0.10:9104 Prod-DB

# WildFly/ACS application server
NODE 10.0.0.20:9100 Prod-ACS1
PROCESS 10.0.0.20:9256 Prod-ACS1
CADVISOR 10.0.0.20:9183 Prod-ACS1
JMX_HIKARI 10.0.0.20:5556 Prod-ACS1
JMX_JVM 10.0.0.20:5557 Prod-ACS1

# Hazelcast cache server (standalone)
NODE 10.0.0.30:9100 Prod-HC1 role=cache
PROCESS 10.0.0.30:9256 Prod-HC1
CADVISOR 10.0.0.30:9183 Prod-HC1
JMX_HC 10.0.0.30:9101 Prod-HC1

# API server (microservices)
NODE 10.0.0.35:9100 Prod-API1 role=api
PROCESS 10.0.0.35:9256 Prod-API1
CADVISOR 10.0.0.35:9183 Prod-API1
PROVISION_API 10.0.0.35:8091 Prod-API1
NORTHBOUND_API 10.0.0.35:9880 Prod-API1
SERVICE_API 10.0.0.35:8085 Prod-API1
SUBSCRIPTION_API 10.0.0.35:8080 Prod-API1

# Oracle database server
NODE 10.0.0.40:9100 Prod-OracleDB
PROCESS 10.0.0.40:9256 Prod-OracleDB
CADVISOR 10.0.0.40:9183 Prod-OracleDB
ORACLE 10.0.0.40:9161 Prod-OracleDB

# Windows/IIS web server
NODE 10.0.0.50:9100 Prod-IIS service=IIS,role=webserver

# === DATABASE SERVERS ===
NODE 65.109.58.165:9100 DevOps-DB service=mysql,role=database
PROCESS 65.109.58.165:9256 DevOps-DB
CADVISOR 65.109.58.165:9183 DevOps-DB
MYSQL 65.109.58.165:9104 DevOps-DB

# === ACS SERVERS ===
NODE 65.109.58.164:9100 DevOps-ACS1 service=ACS,role=application
PROCESS 65.109.58.164:9256 DevOps-ACS1
CADVISOR 65.109.58.164:9183 DevOps-ACS1
JMX_HIKARI 65.109.58.164:5556 DevOps-ACS1
JMX_JVM 65.109.58.164:5557 DevOps-ACS1
# Optional ACS endpoints:
FT_SYSTEM 65.109.58.164:8090 DevOps-ACS1
ACS_METRICS 65.109.58.164:8080 DevOps-ACS1
JMX_HC 65.109.58.164:9101 DevOps-ACS1

# === HAZELCAST SERVERS (standalone) ===
NODE 65.109.24.154:9100 DevOps-HC1 service=hazelcast,role=cache
PROCESS 65.109.24.154:9256 DevOps-HC1
CADVISOR 65.109.24.154:9183 DevOps-HC1
JMX_HC 65.109.24.154:9101 DevOps-HC1

# === API SERVERS (microservices) ===
NODE 65.109.24.155:9100 DevOps-API1 service=api,role=api
PROCESS 65.109.24.155:9256 DevOps-API1
CADVISOR 65.109.24.155:9183 DevOps-API1
PROVISION_API 65.109.24.155:8091 DevOps-API1
NORTHBOUND_API 65.109.24.155:9880 DevOps-API1
SERVICE_API 65.109.24.155:8085 DevOps-API1
SUBSCRIPTION_API 65.109.24.155:8080 DevOps-API1

# === ORACLE SERVERS ===
NODE 65.109.20.174:9100 DevOps-OracleDB service=oracle,role=database
PROCESS 65.109.20.174:9256 DevOps-OracleDB
CADVISOR 65.109.20.174:9183 DevOps-OracleDB
ORACLE 65.109.20.174:9161 DevOps-OracleDB

# === IIS SERVERS ===
NODE 65.109.49.150:9100 devops-ui.friendly-tech.com service=IIS,role=webserver,node_ip=65.109.49.150

cd /opt/grafana/prometheus-grafana-stack/prometheus
/opt/grafana/prometheus-grafana-stack/prometheus/scripts/generate-targets.sh

# show what did not answer this run; changes nothing
bash /opt/grafana/prometheus-grafana-stack/prometheus/scripts/generate-targets.sh --park-dry-run

# comment those rows out, keeping the original line, with a backup
bash /opt/grafana/prometheus-grafana-stack/prometheus/scripts/generate-targets.sh --park

curl -s -o /dev/null -w '%{http_code}\n' \
  http://<ip>:8091/prov-portal/actuator/prometheus     # not /metrics

# prometheus.yml already has this for each job:
relabel_configs:
  - source_labels: [__address__]
    target_label: node_ip
    regex: (.+):.*      # extracts IP from "IP:PORT"
    replacement: $1

cd /opt/grafana/prometheus-grafana-stack
vi .env

# Example: allow access from Angular UI portal
UI_PORTAL_ORIGINS=https://portal.example.com,https://10.0.0.50:8880

grep GRAFANA_SERVICE_TOKEN .env

docker compose down && docker compose up -d

cd /opt/grafana
sudo bash /opt/grafana/misc/generate-grafana-token.sh --restart

cd /opt/grafana/prometheus-grafana-stack
vi .env
# Set: GRAFANA_SERVICE_TOKEN=glsa_YourActualTokenHere_12345678

<iframe
  src="/d/business-metrics/business-metrics?orgId=1&kiosk&refresh=30s"
  width="100%"
  height="600"
  frameborder="0">
</iframe>

# Should return 200 OK without authentication
curl -kI https://YOUR_SERVER_DOMAIN/grafana-ro/