Alertmanager API Guide

[
  {
    "labels": {
      "alertname": "HighCpuUsage",
      "severity": "warning",
      "instance": "server-01",
      "service": "api"
    },
    "annotations": {
      "summary": "CPU usage above 80%",
      "description": "Server server-01 CPU at 85% for 5 minutes"
    },
    "startsAt": "2024-01-15T10:00:00Z",
    "endsAt": "2024-01-15T10:30:00Z",
    "generatorURL": "https://monitoring.example.com/graph?query=cpu"
  }
]

curl -X POST http://localhost:9093/api/v2/alerts \
  -H "Content-Type: application/json" \
  -d '[{
    "labels": {
      "alertname": "DatabaseConnectionFailed",
      "severity": "critical",
      "instance": "db-primary",
      "service": "mysql"
    },
    "annotations": {
      "summary": "Cannot connect to MySQL database",
      "description": "Connection refused on db-primary:3306"
    }
  }]'

curl -s http://localhost:9093/api/v2/alerts | jq

curl -s 'http://localhost:9093/api/v2/alerts?filter=severity="critical"' | jq

curl -s 'http://localhost:9093/api/v2/alerts?filter=service="mysql"' | jq

[
  {
    "labels": {
      "alertname": "HighCpuUsage",
      "severity": "warning",
      "instance": "server-01"
    },
    "annotations": {
      "summary": "CPU usage above 80%"
    },
    "startsAt": "2024-01-15T10:00:00.000Z",
    "endsAt": "0001-01-01T00:00:00Z",
    "generatorURL": "",
    "status": {
      "state": "active",
      "silencedBy": [],
      "inhibitedBy": []
    },
    "receivers": ["default"],
    "fingerprint": "abc123def456"
  }
]

curl -X POST http://localhost:9093/api/v2/alerts \
  -H "Content-Type: application/json" \
  -d '[{
    "labels": {
      "alertname": "DatabaseConnectionFailed",
      "severity": "critical",
      "instance": "db-primary",
      "service": "mysql"
    },
    "endsAt": "2024-01-15T10:30:00Z"
  }]'

curl -X POST http://localhost:9093/api/v2/silences \
  -H "Content-Type: application/json" \
  -d '{
    "matchers": [
      {
        "name": "alertname",
        "value": "HighCpuUsage",
        "isRegex": false,
        "isEqual": true
      },
      {
        "name": "instance",
        "value": "server-01",
        "isRegex": false,
        "isEqual": true
      }
    ],
    "startsAt": "2024-01-15T10:00:00Z",
    "endsAt": "2024-01-15T12:00:00Z",
    "createdBy": "admin",
    "comment": "Maintenance window - CPU upgrade"
  }'

{
  "silenceID": "a1b2c3d4-e5f6-7890-abcd-ef1234567890"
}

curl -s http://localhost:9093/api/v2/silences | jq

curl -s http://localhost:9093/api/v2/silence/a1b2c3d4-e5f6-7890-abcd-ef1234567890 | jq

curl -X DELETE http://localhost:9093/api/v2/silence/a1b2c3d4-e5f6-7890-abcd-ef1234567890

curl -X POST http://localhost:9093/api/v2/silences \
  -H "Content-Type: application/json" \
  -d '{
    "matchers": [
      {
        "name": "instance",
        "value": "server-0[1-3]",
        "isRegex": true,
        "isEqual": true
      }
    ],
    "startsAt": "2024-01-15T10:00:00Z",
    "endsAt": "2024-01-15T14:00:00Z",
    "createdBy": "devops",
    "comment": "Cluster maintenance"
  }'

curl -s http://localhost:9093/api/v2/status | jq

curl -s http://localhost:9093/api/v2/receivers | jq

curl -s http://localhost:9093/-/healthy
# Returns: OK

curl -s http://localhost:9093/-/ready
# Returns: OK

import requests
import json
from datetime import datetime, timezone

ALERTMANAGER_URL = "http://localhost:9093"

def send_alert(alertname, severity, instance, summary, description=None, labels=None):
    """Send an alert to Alertmanager."""
    alert = {
        "labels": {
            "alertname": alertname,
            "severity": severity,
            "instance": instance,
            **(labels or {})
        },
        "annotations": {
            "summary": summary
        },
        "startsAt": datetime.now(timezone.utc).isoformat()
    }

    if description:
        alert["annotations"]["description"] = description

    response = requests.post(
        f"{ALERTMANAGER_URL}/api/v2/alerts",
        json=[alert],
        headers={"Content-Type": "application/json"}
    )
    response.raise_for_status()
    return response.status_code == 200

def resolve_alert(alertname, severity, instance, labels=None):
    """Resolve an existing alert."""
    alert = {
        "labels": {
            "alertname": alertname,
            "severity": severity,
            "instance": instance,
            **(labels or {})
        },
        "endsAt": datetime.now(timezone.utc).isoformat()
    }

    response = requests.post(
        f"{ALERTMANAGER_URL}/api/v2/alerts",
        json=[alert],
        headers={"Content-Type": "application/json"}
    )
    response.raise_for_status()
    return response.status_code == 200

def get_active_alerts(severity=None):
    """Get list of active alerts."""
    params = {"active": "true"}
    if severity:
        params["filter"] = f'severity="{severity}"'

    response = requests.get(
        f"{ALERTMANAGER_URL}/api/v2/alerts",
        params=params
    )
    response.raise_for_status()
    return response.json()

def create_silence(alertname, instance, duration_hours, comment, created_by="api"):
    """Create a silence for specific alert."""
    from datetime import timedelta

    now = datetime.now(timezone.utc)
    silence = {
        "matchers": [
            {"name": "alertname", "value": alertname, "isRegex": False, "isEqual": True},
            {"name": "instance", "value": instance, "isRegex": False, "isEqual": True}
        ],
        "startsAt": now.isoformat(),
        "endsAt": (now + timedelta(hours=duration_hours)).isoformat(),
        "createdBy": created_by,
        "comment": comment
    }

    response = requests.post(
        f"{ALERTMANAGER_URL}/api/v2/silences",
        json=silence,
        headers={"Content-Type": "application/json"}
    )
    response.raise_for_status()
    return response.json().get("silenceID")

# Usage examples
if __name__ == "__main__":
    # Send alert
    send_alert(
        alertname="AppError",
        severity="warning",
        instance="api-server-01",
        summary="Application error rate increased",
        description="Error rate above 5% for last 5 minutes",
        labels={"service": "user-api", "environment": "production"}
    )

    # Get critical alerts
    alerts = get_active_alerts(severity="critical")
    print(f"Active critical alerts: {len(alerts)}")

    # Resolve alert
    resolve_alert(
        alertname="AppError",
        severity="warning",
        instance="api-server-01",
        labels={"service": "user-api", "environment": "production"}
    )

    # Create 2-hour silence
    silence_id = create_silence(
        alertname="HighCpuUsage",
        instance="server-01",
        duration_hours=2,
        comment="Planned maintenance"
    )
    print(f"Created silence: {silence_id}")

import org.springframework.http.*;
import org.springframework.web.client.RestTemplate;
import java.time.Instant;
import java.util.*;

public class AlertmanagerClient {

    private final String baseUrl;
    private final RestTemplate restTemplate;

    public AlertmanagerClient(String baseUrl) {
        this.baseUrl = baseUrl;
        this.restTemplate = new RestTemplate();
    }

    public void sendAlert(String alertname, String severity, String instance,
                          String summary, Map<String, String> extraLabels) {

        Map<String, Object> alert = new HashMap<>();

        Map<String, String> labels = new HashMap<>();
        labels.put("alertname", alertname);
        labels.put("severity", severity);
        labels.put("instance", instance);
        if (extraLabels != null) {
            labels.putAll(extraLabels);
        }
        alert.put("labels", labels);

        Map<String, String> annotations = new HashMap<>();
        annotations.put("summary", summary);
        alert.put("annotations", annotations);

        alert.put("startsAt", Instant.now().toString());

        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);

        HttpEntity<List<Map<String, Object>>> request =
            new HttpEntity<>(List.of(alert), headers);

        restTemplate.postForEntity(
            baseUrl + "/api/v2/alerts",
            request,
            String.class
        );
    }

    public void resolveAlert(String alertname, String severity, String instance,
                             Map<String, String> extraLabels) {

        Map<String, Object> alert = new HashMap<>();

        Map<String, String> labels = new HashMap<>();
        labels.put("alertname", alertname);
        labels.put("severity", severity);
        labels.put("instance", instance);
        if (extraLabels != null) {
            labels.putAll(extraLabels);
        }
        alert.put("labels", labels);
        alert.put("endsAt", Instant.now().toString());

        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);

        HttpEntity<List<Map<String, Object>>> request =
            new HttpEntity<>(List.of(alert), headers);

        restTemplate.postForEntity(
            baseUrl + "/api/v2/alerts",
            request,
            String.class
        );
    }

    public List<Map<String, Object>> getActiveAlerts() {
        ResponseEntity<List> response = restTemplate.getForEntity(
            baseUrl + "/api/v2/alerts?active=true",
            List.class
        );
        return response.getBody();
    }
}

// Usage in Spring Boot service
@Service
public class MonitoringService {

    private final AlertmanagerClient alertmanager;

    public MonitoringService() {
        this.alertmanager = new AlertmanagerClient("http://localhost:9093");
    }

    public void reportDatabaseError(String dbHost, String errorMessage) {
        alertmanager.sendAlert(
            "DatabaseError",
            "critical",
            dbHost,
            errorMessage,
            Map.of("service", "database", "component", "mysql")
        );
    }

    public void clearDatabaseError(String dbHost) {
        alertmanager.resolveAlert(
            "DatabaseError",
            "critical",
            dbHost,
            Map.of("service", "database", "component", "mysql")
        );
    }
}

#!/bin/bash
# alertmanager-client.sh - Simple Alertmanager CLI

ALERTMANAGER_URL="${ALERTMANAGER_URL:-http://localhost:9093}"

send_alert() {
    local alertname="$1"
    local severity="$2"
    local instance="$3"
    local summary="$4"

    curl -s -X POST "${ALERTMANAGER_URL}/api/v2/alerts" \
        -H "Content-Type: application/json" \
        -d "[{
            \"labels\": {
                \"alertname\": \"${alertname}\",
                \"severity\": \"${severity}\",
                \"instance\": \"${instance}\"
            },
            \"annotations\": {
                \"summary\": \"${summary}\"
            }
        }]"
}

resolve_alert() {
    local alertname="$1"
    local severity="$2"
    local instance="$3"

    curl -s -X POST "${ALERTMANAGER_URL}/api/v2/alerts" \
        -H "Content-Type: application/json" \
        -d "[{
            \"labels\": {
                \"alertname\": \"${alertname}\",
                \"severity\": \"${severity}\",
                \"instance\": \"${instance}\"
            },
            \"endsAt\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"
        }]"
}

list_alerts() {
    curl -s "${ALERTMANAGER_URL}/api/v2/alerts" | jq -r '.[] | "\(.labels.severity)\t\(.labels.alertname)\t\(.labels.instance)"'
}

# Usage
case "$1" in
    send)
        send_alert "$2" "$3" "$4" "$5"
        ;;
    resolve)
        resolve_alert "$2" "$3" "$4"
        ;;
    list)
        list_alerts
        ;;
    *)
        echo "Usage: $0 {send|resolve|list}"
        echo "  send <alertname> <severity> <instance> <summary>"
        echo "  resolve <alertname> <severity> <instance>"
        echo "  list"
        exit 1
        ;;
esac