Runbook: Install Exporters

# Auto-detect services and deploy (recommended):
cd /opt/grafana
sudo bash /opt/grafana/misc/quick-deploy.sh PROMETHEUS_IP

# Examples:
sudo bash /opt/grafana/misc/quick-deploy.sh 65.108.67.167

# Or deploy specific exporters manually (--exporter flag):
sudo bash /opt/grafana/misc/quick-deploy.sh 65.108.67.167 --exporter base --exporter jmx-wildfly
sudo bash /opt/grafana/misc/quick-deploy.sh 65.108.67.167 --exporter base --exporter mysql

# Or use one-command bootstrap (installs Docker, clones repo, deploys):
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- 65.108.67.167

/opt
└── grafana
    ├── exporters/                    # Modular exporter directories
    │   ├── base/                     # Node + Process + cAdvisor (all servers)
    │   │   ├── docker-compose.yml
    │   │   ├── process-exporter.yml
    │   │   └── .env.example
    │   ├── jmx-wildfly/             # HikariCP exporter (5556) + JVM exporter
    │   │                            #   (5557, compose profile 'jvm', bare-metal only)
    │   │   ├── docker-compose.yml
    │   │   ├── config-hikari.yml
    │   │   ├── config-jvm.yml
    │   │   └── .env.example
    │   ├── mysql/                    # MySQL/MariaDB exporter
    │   │   ├── docker-compose.yml
    │   │   └── .env.example
    │   ├── oracle/                   # Oracle DB exporter
    │   │   ├── docker-compose.yml
    │   │   ├── config.yaml.template
    │   │   ├── generate-config.sh
    │   │   └── .env.example
    │   ├── clickhouse/               # ClickHouse native Prometheus endpoint (no exporter container)
    │   │   ├── clickhouse-prometheus.xml   # config.d snippet (port 9363)
    │   │   └── install-native.sh           # installs the config + optional restart
    │   ├── postgres/                 # PostgreSQL exporter
    │   │   ├── docker-compose.yml
    │   │   └── .env.example
    │   └── nginx/                    # Nginx exporter (for Angular UI)
    │       ├── docker-compose.yml
    │       └── .env.example
    ├── prometheus-grafana-stack/     # Monitoring stack
    │   ├── docker-compose.yml
    │   ├── .env.example
    │   └── prometheus/
    │       ├── prometheus.yml
    │       ├── scripts/
    │       │   └── generate-targets.sh
    │       └── targets/
    │           └── servers.env.example
    ├── docs/
    │   └── architecture.md           # Architecture diagrams
    ├── misc/
    │   ├── bootstrap.sh              # One-command installer
    │   ├── quick-deploy.sh           # Auto-detect and deploy exporters
    │   ├── detect-services.sh        # Service auto-detection
    │   ├── diagnose.sh               # Troubleshooting tool
    │   └── install-docker.sh         # Docker installer
    ├── README.md
    └── sizing.md                     # Capacity planning guide

# Auto-detect services and deploy (recommended)
cd /opt/grafana
sudo bash /opt/grafana/misc/quick-deploy.sh 65.109.58.165

# Base exporters (Node + Process) - all servers
cd /opt/grafana/exporters/base

# Hazelcast: No exporter needed - uses native Prometheus endpoint (port 9101)
# See section "Prerequisites: Enable Prometheus on Hazelcast Servers"

# JMX exporters for WildFly/FTACS
cd /opt/grafana/exporters/jmx-wildfly

cp .env.example .env
vi .env

docker compose up -d

docker compose logs -f

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9100 -j ACCEPT   # node-exporter
iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9256 -j ACCEPT   # process-exporter
iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9183 -j ACCEPT   # cadvisor
# JMX ports (if WildFly detected):
iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 5556 -j ACCEPT   # jmx-hikari
iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 5557 -j ACCEPT   # jmx-jvm
# Hazelcast native Prometheus (if Hazelcast detected):
iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9101 -j ACCEPT   # hazelcast native

iptables -A INPUT -p tcp -s 65.109.58.165 --dport 9100 -j ACCEPT   # node-exporter
iptables -A INPUT -p tcp -s 65.109.58.165 --dport 9256 -j ACCEPT   # process-exporter
iptables -A INPUT -p tcp -s 65.109.58.165 --dport 9183 -j ACCEPT   # cadvisor
# JMX ports (if WildFly detected):
iptables -A INPUT -p tcp -s 65.109.58.165 --dport 5556 -j ACCEPT   # jmx-hikari
iptables -A INPUT -p tcp -s 65.109.58.165 --dport 5557 -j ACCEPT   # jmx-jvm
# Hazelcast native Prometheus (if Hazelcast detected):
iptables -A INPUT -p tcp -s 65.109.58.165 --dport 9101 -j ACCEPT   # hazelcast native

cd /opt/grafana

# Auto-detect (recommended) - deploys appropriate exporters based on detected services
sudo bash /opt/grafana/misc/quick-deploy.sh 65.108.67.167

# Or deploy specific exporters manually:
# For WildFly/FTACS servers:
sudo bash /opt/grafana/misc/quick-deploy.sh 65.108.67.167 --exporter base --exporter jmx-wildfly

# Check containers are running
docker ps | grep jmx

# Test metrics endpoints
# WildFly:
curl -s http://localhost:5556/metrics | head  # HikariCP
curl -s http://localhost:5557/metrics | head  # JVM

# Hazelcast:
curl -s http://localhost:9101/metrics | head  # Hazelcast

JAVA_OPTS="$JAVA_OPTS \
    -Dcom.sun.management.jmxremote \
    -Dcom.sun.management.jmxremote.port=9999 \
    -Dcom.sun.management.jmxremote.rmi.port=9999 \
    -Dcom.sun.management.jmxremote.ssl=false \
    -Dcom.sun.management.jmxremote.authenticate=false \
    -Djava.rmi.server.hostname=127.0.0.1"

registerMbeans=true

# For MySQL:
service jboss restart

# For Oracle:
service jboss restartoracle

export PROMETHEUS_PORT=9101

  -e PROMETHEUS_PORT=9101 \   # Enable native Prometheus metrics
  -p 9101:9101 \              # Expose metrics port to host

# If you already have Docker installed and repository cloned:
cd /opt/grafana
sudo bash /opt/grafana/misc/quick-deploy.sh 65.109.58.165 --exporter base --exporter mysql

# Or use one-command bootstrap (auto-detects MySQL):
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- 65.109.58.165

# Uncomment to override default port (9104)
# - '--web.listen-address=:19104'

CREATE USER IF NOT EXISTS 'exporter'@'%' IDENTIFIED BY 'StR0nG_PassW0rD_hErE' WITH MAX_USER_CONNECTIONS 3;
GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'exporter'@'%';

cd /opt/grafana/exporters/mysql/
cp .env.example .env
vi .env

# MySQL Connection Details (REQUIRED)
MYSQL_HOST=localhost
MYSQL_PORT=3306
MYSQL_USER=exporter
MYSQL_PASSWORD=StR0nG_PassW0rD_hErE  # Replace with your password

docker compose up -d

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9104 -j ACCEPT

# If you already have Docker installed and repository cloned:
cd /opt/grafana
sudo bash /opt/grafana/misc/quick-deploy.sh 65.109.58.165 --exporter base --exporter oracle

# Or use one-command bootstrap (auto-detects Oracle):
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- 65.109.58.165

cd /opt/grafana/exporters/oracle/

cp .env.example .env
vi .env

# Oracle Database Connection (REQUIRED)
ORACLE_HOST=localhost
ORACLE_PORT=1521
ORACLE_SERVICE_NAME=ORCL
ORACLE_USER=system
ORACLE_PASSWORD=your_oracle_password

./generate-config.sh

docker compose up -d

docker compose logs -f

curl http://localhost:9161/metrics

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9161 -j ACCEPT

iptables -A INPUT -p tcp -s 65.109.58.165 --dport 9161 -j ACCEPT

cd /opt/grafana/exporters/oracle
cp .env.example .env
vi .env

ORACLE_HOST=<database-server-ip>    # Remote database server IP (not localhost)
ORACLE_PORT=1521
ORACLE_SERVICE_NAME=<service-name>
ORACLE_USER=<username>
ORACLE_PASSWORD=<password>

cd /opt/grafana/exporters/oracle
bash /opt/grafana/exporters/oracle/generate-config.sh
docker compose up -d

curl http://localhost:9161/metrics | grep oracledb_up
# Expected: oracledb_up 1

cd /opt/grafana/prometheus-grafana-stack/prometheus/targets
vi servers.env
# Add line:
ORACLE 127.0.0.1:9161 <instance-name>

# Regenerate targets:
bash /opt/grafana/prometheus-grafana-stack/prometheus/scripts/generate-targets.sh

hazelcast:
    image: hub.friendly-tech.com/components/hazelcast:ft.5.5.0
    ports:
      - "5701:5701"
      - "5711:5711"
      - "9101:9101"
    environment:
      PROMETHEUS_PORT: 9101

hazelcast:
    image: hub.friendly-tech.com/components/hazelcast:ft.5.5.0
    network_mode: host
    environment:
      PROMETHEUS_PORT: 9101

# Should return metrics immediately (no hang)
curl -s http://localhost:9101/metrics | head

# Auto-detect and deploy all exporters (recommended):
curl -fsSL https://hub.friendly-tech.com/bootstrap/install.sh | sudo bash -s -- PROMETHEUS_IP

# Or if repository already cloned:
cd /opt/grafana
sudo bash /opt/grafana/misc/quick-deploy.sh PROMETHEUS_IP

location /nginx_status {
    stub_status on;
    access_log off;
    allow 127.0.0.1;
    deny all;
}

# Check where nginx config is located inside the container
docker exec portals nginx -T 2>/dev/null | head -5

# Common locations:
#   /etc/nginx/conf.d/default.conf
#   /etc/nginx/nginx.conf

# View the current config
docker exec portals cat /etc/nginx/conf.d/default.conf

# Edit: copy config out, add the location block, copy back
docker cp portals:/etc/nginx/conf.d/default.conf ./default.conf
# ... add the location /nginx_status block inside the server { } section ...
docker cp ./default.conf portals:/etc/nginx/conf.d/default.conf
docker exec portals nginx -s reload

docker exec portals curl -s http://127.0.0.1/nginx_status
# Expected output:
#   Active connections: 5
#   server accepts handled requests
#
1234 1234 5678
#   Reading: 0 Writing: 1 Waiting: 4

cd /opt/grafana/exporters/nginx
cp .env.example .env
vi .env

NGINX_HOST=localhost  # or container name if using Docker network
NGINX_PORT=8880
NGINX_EXPORTER_PORT=9113

docker compose up -d

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9113 -j ACCEPT

curl http://localhost:9113/metrics | grep nginx_

sudo bash /opt/grafana/exporters/clickhouse/install-native.sh --restart

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9363 -j ACCEPT

CLICKHOUSE  <server-ip>:9363  <instance>  role=database

curl -s http://localhost:9363/metrics | grep -E '^ClickHouse(Metrics|ProfileEvents|AsyncMetrics)_'

-- Use a generated password, not a memorable one; this account is reachable
-- from wherever the database is. e.g.  openssl rand -base64 24
CREATE USER exporter WITH PASSWORD 'REPLACE_WITH_GENERATED_PASSWORD';
GRANT pg_monitor TO exporter;
GRANT CONNECT ON DATABASE ftl_ai TO exporter;

postgres:
  ports:
    - "127.0.0.1:5432:5432"   # loopback ONLY — the exporter reaches it, the internet does not

iptables -I DOCKER-USER -p tcp --dport 5432 ! -s <allowed-ip>/32 -j DROP

cd /opt/grafana/exporters/postgres
# find the database's network
docker inspect <postgres-container> --format '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}'
# in .env:
#   POSTGRES_DOCKER_NETWORK=ft-ui-services_default
#   POSTGRES_HOST=postgres
# service name, not 127.0.0.1
#   POSTGRES_PORT=5432
docker compose -f docker-compose.yml -f docker-compose.network.yml up -d

cd /opt/grafana/exporters/postgres
cp .env.example .env
vi .env

POSTGRES_HOST=localhost
POSTGRES_PORT=5432
POSTGRES_USER=exporter
POSTGRES_PASSWORD=<the generated password from Step 1>
POSTGRES_DB=ftl_ai
POSTGRES_EXPORTER_PORT=9187

docker compose up -d

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 9187 -j ACCEPT

curl http://localhost:9187/metrics | grep pg_

curl http://localhost:8383/api/actuator/prometheus | grep jvm_

FT_DEVICE_NETWORK 65.109.24.154:8383 DevOps-HC1

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 8383 -j ACCEPT

  ui-ai-agent:
    ports:
      - "8084:8080"

docker compose up -d ui-ai-agent

curl http://localhost:8084/metrics | grep python_

AI_AGENT 65.109.24.154:8084 DevOps-HC1

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 8084 -j ACCEPT

.requestMatchers("/actuator/**").permitAll()

management.endpoints.web.exposure.include=prometheus,health,info

curl http://localhost:8881/actuator/prometheus | grep jvm_

UI_BACKEND 65.109.24.154:8881 DevOps-HC1

iptables -A INPUT -p tcp -s $PROMETHEUS_IP --dport 8881 -j ACCEPT

lodctr.exe /E:Lsa
lodctr.exe /E:PerfProc
lodctr.exe /R

msiexec /i "C:\Users\Administrator\windows_exporter-0.31.3-amd64.msi" ENABLED_COLLECTORS=ad,adfs,cache,cpu,cpu_info,container,dfsr,dhcp,dns,fsrmquota,iis,logical_disk,memory,msmq,mssql,net,os,process,remote_fx,service,tcp,time,vmware LISTEN_PORT=9100 EXTRA_FLAGS="--collector.process.include=""(w3wp|java.*)"" --collector.process.iis --collector.process.counter-version=2"

netsh advfirewall firewall add rule name="Allow Windows Exporter" dir=in action=allow protocol=TCP localport=9100

sc query windows_exporter

# === IIS SERVERS ===
NODE <server-ip>:9100 <Prefix>-IIS service=IIS,role=webserver,node_ip=<server-ip>

cd /opt/grafana/prometheus-grafana-stack/prometheus
/opt/grafana/prometheus-grafana-stack/prometheus/scripts/generate-targets.sh