betelgeusebytes/k8s/observability-stack/test-loki-logs.sh

#!/bin/bash

set -e

GREEN='\033[0;32m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}   Loki Log Collection Test${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

PASS=0
FAIL=0

# Test 1: Check Alloy DaemonSet
echo -e "${YELLOW}Test 1: Checking Alloy DaemonSet...${NC}"
if kubectl get pods -n observability -l app=alloy --no-headers 2>/dev/null | grep -q "Running"; then
    ALLOY_COUNT=$(kubectl get pods -n observability -l app=alloy --no-headers | grep -c "Running")
    echo -e "${GREEN}✓ Alloy is running ($ALLOY_COUNT pod(s))${NC}"
    PASS=$((PASS+1))
else
    echo -e "${RED}✗ Alloy is not running${NC}"
    FAIL=$((FAIL+1))
fi
echo ""

# Test 2: Check Loki pod
echo -e "${YELLOW}Test 2: Checking Loki pod...${NC}"
if kubectl get pods -n observability -l app=loki --no-headers 2>/dev/null | grep -q "Running"; then
    echo -e "${GREEN}✓ Loki is running${NC}"
    PASS=$((PASS+1))
else
    echo -e "${RED}✗ Loki is not running${NC}"
    FAIL=$((FAIL+1))
fi
echo ""

# Test 3: Test Loki readiness endpoint
echo -e "${YELLOW}Test 3: Testing Loki readiness endpoint...${NC}"
READY=$(kubectl run test-loki-ready-$RANDOM --rm -i --restart=Never --image=curlimages/curl:latest -- \
    curl -s -m 5 http://loki.observability.svc.cluster.local:3100/ready 2>/dev/null || echo "failed")

if [ "$READY" = "ready" ]; then
    echo -e "${GREEN}✓ Loki is ready${NC}"
    PASS=$((PASS+1))
else
    echo -e "${RED}✗ Loki is not ready (response: $READY)${NC}"
    FAIL=$((FAIL+1))
fi
echo ""

# Test 4: Check Alloy can connect to Loki
echo -e "${YELLOW}Test 4: Checking Alloy → Loki connectivity...${NC}"
ALLOY_ERRORS=$(kubectl logs -n observability -l app=alloy --tail=50 2>/dev/null | grep -i "error.*loki" | wc -l)
if [ "$ALLOY_ERRORS" -eq 0 ]; then
    echo -e "${GREEN}✓ No Alloy → Loki connection errors${NC}"
    PASS=$((PASS+1))
else
    echo -e "${RED}✗ Found $ALLOY_ERRORS error(s) in Alloy logs${NC}"
    kubectl logs -n observability -l app=alloy --tail=20 | grep -i error
    FAIL=$((FAIL+1))
fi
echo ""

# Test 5: Create test pod and verify logs
echo -e "${YELLOW}Test 5: Creating test pod and verifying log collection...${NC}"

# Clean up any existing test pod
kubectl delete pod test-logger-verify --ignore-not-found 2>/dev/null

# Create test pod
echo "  Creating test pod that logs every second..."
kubectl run test-logger-verify --image=busybox --restart=Never -- sh -c \
  'for i in 1 2 3 4 5 6 7 8 9 10; do echo "LOKI-TEST-LOG: Message number $i at $(date)"; sleep 1; done' \
  >/dev/null 2>&1

# Wait for pod to start and generate logs
echo "  Waiting 15 seconds for logs to be collected..."
sleep 15

# Query Loki API for test logs
echo "  Querying Loki for test logs..."
START_TIME=$(date -u -d '2 minutes ago' +%s)000000000
END_TIME=$(date -u +%s)000000000

QUERY_RESULT=$(kubectl run test-loki-query-$RANDOM --rm -i --restart=Never --image=curlimages/curl:latest -- \
    curl -s -m 10 "http://loki.observability.svc.cluster.local:3100/loki/api/v1/query_range" \
    --data-urlencode 'query={pod="test-logger-verify"}' \
    --data-urlencode "start=$START_TIME" \
    --data-urlencode "end=$END_TIME" 2>/dev/null || echo "failed")

if echo "$QUERY_RESULT" | grep -q "LOKI-TEST-LOG"; then
    LOG_COUNT=$(echo "$QUERY_RESULT" | grep -o "LOKI-TEST-LOG" | wc -l)
    echo -e "${GREEN}✓ Found $LOG_COUNT test log messages in Loki${NC}"
    PASS=$((PASS+1))
else
    echo -e "${RED}✗ Test logs not found in Loki${NC}"
    echo "  Response: ${QUERY_RESULT:0:200}"
    FAIL=$((FAIL+1))
fi

# Clean up test pod
kubectl delete pod test-logger-verify --ignore-not-found >/dev/null 2>&1

echo ""

# Test 6: Check observability namespace logs
echo -e "${YELLOW}Test 6: Checking for observability namespace logs...${NC}"

OBS_QUERY=$(kubectl run test-loki-obs-$RANDOM --rm -i --restart=Never --image=curlimages/curl:latest -- \
    curl -s -m 10 "http://loki.observability.svc.cluster.local:3100/loki/api/v1/query_range" \
    --data-urlencode 'query={namespace="observability"}' \
    --data-urlencode "start=$START_TIME" \
    --data-urlencode "end=$END_TIME" \
    --data-urlencode "limit=10" 2>/dev/null || echo "failed")

if echo "$OBS_QUERY" | grep -q '"values":\[\['; then
    echo -e "${GREEN}✓ Observability namespace logs found in Loki${NC}"
    PASS=$((PASS+1))
else
    echo -e "${RED}✗ No logs found for observability namespace${NC}"
    FAIL=$((FAIL+1))
fi

echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}   Test Results${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

TOTAL=$((PASS+FAIL))
echo -e "Passed: ${GREEN}$PASS${NC} / $TOTAL"
echo -e "Failed: ${RED}$FAIL${NC} / $TOTAL"
echo ""

if [ $FAIL -eq 0 ]; then
    echo -e "${GREEN}✓✓✓ All tests passed! Logs are flowing to Loki! ✓✓✓${NC}"
    echo ""
    echo "Next steps:"
    echo "  1. Open Grafana: https://grafana.betelgeusebytes.io"
    echo "  2. Go to Explore → Loki"
    echo "  3. Query: {namespace=\"observability\"}"
    echo ""
else
    echo -e "${RED}✗✗✗ Some tests failed. Check the output above for details. ✗✗✗${NC}"
    echo ""
    echo "Troubleshooting:"
    echo "  - Check Alloy logs: kubectl logs -n observability -l app=alloy"
    echo "  - Check Loki logs: kubectl logs -n observability loki-0"
    echo "  - Verify services: kubectl get svc -n observability"
    echo "  - See full guide: VERIFY-LOKI-LOGS.md"
    echo ""
    exit 1
fi