Environment Operations Runbook

# Create feature environment with MySQL
./scripts/create-feature-env.sh TICKET-123 mysql

# Create feature environment with Oracle
./scripts/create-feature-env.sh TICKET-123 oracle

# Verify creation
kubectl get namespace feature-ticket-123
kubectl get pods -n feature-ticket-123

# Delete ArgoCD Application
argocd app delete feature-ticket-123 --cascade

# Delete namespace (if not cascaded)
kubectl delete namespace feature-ticket-123

# Remove files from Git
rm helm/ft-services/values-dynamic-feature-ticket-123.yaml
rm argocd/applications/dynamic/feature-ticket-123.yaml
git add -A && git commit -m "Remove feature-ticket-123 environment"
git push

# Sync specific application
argocd app sync feature-ticket-123

# Force sync with prune
argocd app sync feature-ticket-123 --prune

# Hard refresh (clear cache)
argocd app get feature-ticket-123 --hard-refresh

# Check sync status details
argocd app get feature-ticket-123

# View diff
argocd app diff feature-ticket-123

# Check ArgoCD logs
kubectl logs -n argocd -l app.kubernetes.io/name=argocd-application-controller

# Check pod status
kubectl get pods -n feature-ticket-123

# Check events
kubectl get events -n feature-ticket-123 --sort-by='.lastTimestamp'

# Check specific pod
kubectl describe pod <pod-name> -n feature-ticket-123
kubectl logs <pod-name> -n feature-ticket-123

# Check GitHub Actions
# Navigate to: https://github.com/{owner}/{repo}/actions

# Check backend logs
docker-compose logs backend | grep -i build

# Check for existing files
ls helm/ft-services/values-dynamic-*.yaml | grep <name>
ls argocd/applications/dynamic/*.yaml | grep <name>

# Check ArgoCD
argocd app get <name>

# Clean up orphaned resources
argocd app delete <name> --cascade
rm helm/ft-services/values-dynamic-<name>.yaml
rm argocd/applications/dynamic/<name>.yaml
git add -A && git commit -m "Clean up orphaned environment"
git push

# Check backend logs
docker-compose logs backend

# Check if git commit happened
git log --oneline -5

# Check ft-root sync
argocd app get ft-root

# Manual intervention: sync ft-root
argocd app sync ft-root

# Check if application was created
argocd app list | grep <env-name>

# Verify parent environment is running
kubectl get pods -n dev-mysql | grep mysql

# Check MySQL is accepting connections
kubectl exec -it -n dev-mysql mysql-0 -- mysql -u root -p -e "SELECT 1"

# Check service DNS resolution from new env
kubectl run -it --rm debug --image=busybox -n feature-ticket-123 -- nslookup mysql.dev-mysql.svc.cluster.local

# Update parent reference in values file if needed

# Verify image in Harbor
curl -u "$HARBOR_USERNAME:$HARBOR_PASSWORD" \
  "https://$HARBOR_HOST/api/v2.0/projects/$HARBOR_PROJECT/repositories/<service>/tags"

# Check GitHub Actions completed
# View workflow run at: https://github.com/{owner}/{repo}/actions

# Retry build via Environment Manager or manually

#!/bin/bash
# Delete feature environments older than 7 days

THRESHOLD_DAYS=7

for app in $(argocd app list -o name | grep "^feature-"); do
    CREATED=$(argocd app get $app -o json | jq -r '.metadata.creationTimestamp')
    AGE_DAYS=$(( ($(date +%s) - $(date -d "$CREATED" +%s)) / 86400 ))

    if [ $AGE_DAYS -gt $THRESHOLD_DAYS ]; then
        echo "Deleting $app (age: $AGE_DAYS days)"
        argocd app delete $app --cascade -y
    fi
done

# Refresh all dynamic environments
for app in $(argocd app list -o name | grep "^feature-"); do
    echo "Refreshing $app"
    argocd app get $app --refresh
done

#!/bin/bash
# Check all environments

echo "Environment Status Report"
echo "========================="

for ns in $(kubectl get ns -o name | grep -E "^namespace/(dev|feature)" | cut -d/ -f2); do
    PODS=$(kubectl get pods -n $ns --no-headers 2>/dev/null | wc -l)
    RUNNING=$(kubectl get pods -n $ns --field-selector=status.phase=Running --no-headers 2>/dev/null | wc -l)
    echo "$ns: $RUNNING/$PODS pods running"
done