Incident Debugging Playbook

Error 429: Rate limit exceeded

anthropic-ratelimit-requests-remaining: 0

anthropic-ratelimit-requests-reset: 2025-12-24T15:00:00Z

Diagnosis:

async function diagnoseRateLimits(): Promise<void> {
  // Check recent API calls
  const recentCalls = await queryLogs('SELECT COUNT(*) FROM api_calls WHERE timestamp > NOW() - INTERVAL 1 MINUTE');
  console.log(`API calls in last minute: ${recentCalls}`);

// Check rate limit headers from last successful call
const lastHeaders = await getLastAPIHeaders();
console.log('Remaining requests:', lastHeaders['anthropic-ratelimit-requests-remaining']);
console.log('Reset time:', lastHeaders['anthropic-ratelimit-requests-reset']);
}

Fix:

// Implement token bucket rate limiter
class EmergencyRateLimiter {
  private tokens = 50; // Match API tier
  private lastRefill = Date.now();

async throttle(): Promise<void> {
this.refill();
while (this.tokens < 1) {
await sleep(100);
this.refill();
}
this.tokens--;
}

private refill() {
const now = Date.now();
const elapsed = (now - this.lastRefill) / 1000;
const tokensToAdd = elapsed * (50 / 60); // 50 per minute
this.tokens = Math.min(50, this.tokens + tokensToAdd);
this.lastRefill = now;
}
}

2. Agent Timeout

Symptoms:

Error: Agent execution timed out after 300000ms

Task: code-review

Conversation: abc-123-def

Diagnosis:

# Check for hung processes
ps aux | grep claude | grep -v grep

# Check system load
uptime
# Output: load average: 12.5, 8.3, 5.2 (CPU overload!)

# Check for blocking I/O
iotop -o -d 5

Fix:

// Implement aggressive timeouts
class TimeoutManager {
  async executeWithTimeout<T>(
    fn: () => Promise<T>,
    timeoutMs: number
  ): Promise<T> {
    return Promise.race([
      fn(),
      new Promise<never>((_, reject) =>
        setTimeout(() => reject(new Error(`Timeout after ${timeoutMs}ms`)), timeoutMs)
      )
    ]);
  }
}

// Usage
const timeout = new TimeoutManager();
const result = await timeout.executeWithTimeout(
() => agent.execute(task),
30000 // 30 second hard limit
);

3. Memory Leak

Symptoms:

# Memory usage climbing over time
free -m
#              total   used   free
# Mem:         16384  15892    492  # Critical!

# Process memory
ps aux --sort=-%mem | head -5
# claude-daemon: 8.2GB (!)

Diagnosis:

// Track memory usage over time
setInterval(() => {
  const usage = process.memoryUsage();
  console.log(JSON.stringify({
    timestamp: Date.now(),
    heapUsed: usage.heapUsed / 1024 / 1024, // MB
    heapTotal: usage.heapTotal / 1024 / 1024,
    external: usage.external / 1024 / 1024,
    rss: usage.rss / 1024 / 1024
  }));

// Trigger GC if usage > 80%
if (usage.heapUsed / usage.heapTotal > 0.8) {
global.gc(); // Requires --expose-gc flag
}
}, 60000); // Every minute

Common Causes:

// ❌ Leak: Global cache never cleared
const cache = new Map<string, any>();
function addToCache(key: string, value: any) {
  cache.set(key, value); // Grows forever!
}

// ✅ Fix: LRU cache with size limit
import LRU from 'lru-cache';
const cache = new LRU<string, any>({ max: 1000 });

4. Plugin Crash Loop

Symptoms:

# PM2 showing rapid restarts
pm2 status
# plugin-server | errored | 47 restarts in 2 minutes

# Logs show crash
tail -f /var/log/pm2/plugin-server-error.log
# Error: ECONNREFUSED 127.0.0.1:5432
# (PostgreSQL connection failed)

Diagnosis:

# Check dependencies
docker ps | grep postgres
# (empty - PostgreSQL container not running!)

# Check network
netstat -tulpn | grep 5432
# (no listener on port 5432)

Fix:

# Restart dependency
docker-compose up -d postgres

# Verify connectivity
psql -h localhost -U user -d database -c "SELECT 1"

# Restart plugin
pm2 restart plugin-server

Debugging Techniques

1. Binary Search Debugging

Problem: Unknown change broke production

# Use git bisect to find breaking commit
git bisect start
git bisect bad HEAD              # Current version is broken
git bisect good v1.2.0           # Last known good version

# Git will check out commits for testing
# Test each commit:
npm install && npm run build && npm test

# Mark results
git bisect good   # if tests pass
git bisect bad    # if tests fail

# Git will find the exact breaking commit

2. Correlation Analysis

Find patterns in failures:

interface FailureEvent {
  timestamp: number;
  errorType: string;
  userId?: string;
  pluginName?: string;
  duration: number;
}

function analyzeFailureCorrelations(failures: FailureEvent[]): void {
// Group by time windows
const byHour = groupBy(failures, f => Math.floor(f.timestamp / 3600000));

// Find spike times
const spikes = Object.entries(byHour)
.filter(([_, events]) => events.length > 100)
.map(([hour, events]) => ({
hour: new Date(parseInt(hour) * 3600000),
count: events.length,
topError: mode(events.map(e => e.errorType))
}));

console.log('Failure spikes:', spikes);

// Find common attributes
const byPlugin = groupBy(failures, f => f.pluginName);
const suspiciousPlugin = Object.entries(byPlugin)
.sort((a, b) => b[1].length - a[1].length)[0];

console.log(`Most failures from plugin: ${suspiciousPlugin[0]} (${suspiciousPlugin[1].length} errors)`);
}

3. Distributed Tracing

Track request across services:

import { trace, context, SpanStatusCode } from '@opentelemetry/api';

const tracer = trace.getTracer('claude-code');

async function executeAgent(agentName: string, task: any): Promise<any> {
const span = tracer.startSpan('agent.execute', {
attributes: {
'agent.name': agentName,
'task.id': task.id
}
});

try {
// Execute agent logic
const result = await agent.run(task);

span.setStatus({ code: SpanStatusCode.OK });
span.setAttribute('result.success', true);

return result;
} catch (error) {
span.setStatus({
code: SpanStatusCode.ERROR,
message: error.message
});
span.recordException(error);
throw error;
} finally {
span.end();
}
}

Log Analysis

Parsing Claude Code Logs

Log Format:

[2025-12-24T14:35:22.123Z] [ERROR] [agent:code-review] Rate limit exceeded

conversationId: abc-123-def

userId: user-456

errorCode: 429

retryAfter: 12

stack: Error: Rate limit exceeded

at callClaude (/app/src/api.ts:45:11)

Analysis Script:

import { readFileSync } from 'fs';

interface LogEntry {
timestamp: Date;
level: 'ERROR' | 'WARN' | 'INFO';
component: string;
message: string;
metadata: Record<string, any>;
}

function parseLog(line: string): LogEntry | null {
const match = line.match(/[(.?)] [(.?)] [(.?)] (.)/);
if (!match) return null;

const [, timestamp, level, component, rest] = match;
const lines = rest.split('
');
const message = lines[0];

// Parse metadata
const metadata: Record<string, any> = {};
for (const line of lines.slice(1)) {
const metaMatch = line.match(/^s*(w+): (.+)$/);
if (metaMatch) {
const [, key, value] = metaMatch;
metadata[key] = value;
}
}

return {
timestamp: new Date(timestamp),
level: level as any,
component,
message,
metadata
};
}

function analyzeLogs(logPath: string): void {
const content = readFileSync(logPath, 'utf-8');
const logs = content.split('
')
.map(parseLog)
.filter(Boolean) as LogEntry[];

// Error rate by component
const errorsByComponent = groupBy(
logs.filter(l => l.level === 'ERROR'),
l => l.component
);

console.log('Errors by component:');
Object.entries(errorsByComponent)
.sort((a, b) => b[1].length - a[1].length)
.forEach(([component, errors]) => {
console.log(`  ${component}: ${errors.length}`);
});

// Recent errors (last 5 minutes)
const recentErrors = logs.filter(l =>
l.level === 'ERROR' &&
Date.now() - l.timestamp.getTime() < 300000
);

console.log(`
Recent errors: ${recentErrors.length}`);
recentErrors.slice(0, 10).forEach(err => {
console.log(`  ${err.timestamp.toISOString()} - ${err.message}`);
});
}

Using Analytics Daemon

// Query analytics daemon for incident patterns
const ws = new WebSocket('ws://localhost:3456');

ws.onmessage = (event) => {
const data = JSON.parse(event.data);

// Track rate limit warnings
if (data.type === 'rate_limit.warning') {
console.warn(`⚠️ Rate limit approaching: ${data.current}/${data.limit}`);
}

// Track errors
if (data.type === 'llm.call' && data.error) {
console.error(`❌ LLM call failed: ${data.error}`);
}
};

// Query historical data
const response = await fetch('http://localhost:3333/api/sessions');
const sessions = await response.json();
const failedSessions = sessions.filter(s => s.errorCount > 0);

console.log(`Failed sessions: ${failedSessions.length}/${sessions.length}`);

Root Cause Analysis

The 5 Whys Method

Example: Agent Timeout Incident

• Why did the agent timeout?

→ Because it took > 300 seconds to respond

• Why did it take so long?

→ Because the Claude API call was slow (280s)

• Why was the API call slow?

→ Because we sent a 50,000 token prompt

• Why did we send such a large prompt?

→ Because the code-reviewer agent included entire codebase in context

• Why did it include the entire codebase?

Fix: Update file globbing to exclude node_modules

// Before: includes everything
const files = glob.sync('*/');

// After: exclude dependencies
const files = glob.sync('*/', {
ignore: ['node_modules/', '.git/', 'dist/**']
});

Fishbone Diagram (Ishikawa)

interface RootCauseAnalysis {
  problem: string;
  categories: {
    people?: string[];
    process?: string[];
    technology?: string[];
    environment?: string[];
  };
  rootCause: string;
  fix: string;
}

const analysis: RootCauseAnalysis = {
problem: 'Agent timeout causing 68% error rate',
categories: {
people: [
'Developer added file globbing without testing',
'No code review caught the issue'
],
process: [
'No integration tests for large codebases',
'No performance testing in CI/CD'
],
technology: [
'Glob pattern included node_modules (500MB)',
'No size limit on prompts',
'No timeout on file reading'
],
environment: [
'Production codebase larger than test repos',
'No staging environment for testing'
]
},
rootCause: 'Missing file size validation and glob pattern filtering',
fix: 'Add file exclusion patterns and max prompt size validation'
};

Recovery Procedures

Emergency Rollback

# Immediate rollback to last known good version
git log --oneline | head -5
# c534df4 (HEAD) feat: Add new feature (BROKEN)
# 3946b1f docs: Update README
# fc73caa (tag: v1.2.0) fix: Bug fix (LAST GOOD)

# Rollback
git reset --hard fc73caa
npm install
npm run build
pm2 restart all

# Deploy
./deploy.sh production

# Verify
curl http://api.example.com/health

Circuit Breaker Reset

// Manually reset circuit breaker after fixing issue
class CircuitBreakerManager {
  private breakers = new Map<string, CircuitBreaker>();

reset(serviceName: string): void {
const breaker = this.breakers.get(serviceName);
if (breaker) {
breaker.state = 'closed';
breaker.failures = 0;
console.log(`✓ Reset circuit breaker for ${serviceName}`);
}
}

resetAll(): void {
for (const [service, breaker] of this.breakers) {
this.reset(service);
}
console.log('✓ Reset all circuit breakers');
}
}

Data Recovery

# Recover from backup
BACKUP_DATE="2025-12-24-14:00"

# Stop services
pm2 stop all

# Restore database
pg_restore -d database_prod backups/backup_${BACKUP_DATE}.sql

# Restore files
rsync -av backups/files_${BACKUP_DATE}/ /var/lib/claude-code/

# Restart
pm2 restart all

# Verify data integrity
psql -d database_prod -c "SELECT COUNT(*) FROM conversations"

Postmortem Templates

Incident Postmortem