09｜故障排除手冊

make demo-ps MODE=events
curl --connect-timeout 2 --max-time 8 -fsS http://localhost:28698/actuator/health/readiness
curl --connect-timeout 2 --max-time 8 -fsS http://localhost:28693/connectors/archone-outbox/status
curl --connect-timeout 2 --max-time 8 -fsS 'http://localhost:28690/orders?limit=20'

rg -n 'Allocation optimistic-lock retry exhausted|Contract violation|Downstream service failure|23505|check_violation|HikariPool|Connection is not available' /tmp/archone-incident.log

SELECT pid, application_name, state, wait_event_type, wait_event,
       now() - xact_start AS transaction_age, pg_blocking_pids(pid) AS blocking_pids
FROM pg_stat_activity
WHERE datname = current_database()
ORDER BY xact_start NULLS LAST;

SELECT application_name, state, count(*)
FROM pg_stat_activity
WHERE datname = current_database()
GROUP BY application_name, state;

SELECT slot_name, active, wal_status,
       pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)) AS retained_wal
FROM pg_replication_slots;

SELECT route, count(*) AS retained_rows, min("timestamp") AS oldest_retained
FROM event_outbox
GROUP BY route;

SELECT state, count(*), min(created_at) AS oldest_created_at
FROM cancellation_processes
WHERE state IN ('WAITING_WMS', 'WAITING_ORDERING')
GROUP BY state;

SELECT id, type, route, aggregateid, partition_key, "timestamp"
FROM event_outbox WHERE id = :'event_id'::uuid;
SELECT subscriber_id, event_id, event_type, processed_at
FROM event_inbox WHERE event_id = :'event_id'::uuid;

SELECT p.id, p.on_hand_quantity, p.reserved_quantity,
       COALESCE(SUM(CASE WHEN m.state = 'ASSIGNED' THEN l.quantity ELSE 0 END), 0) AS derived_reserved
FROM stock_pools p
LEFT JOIN stock_move_lines l ON l.stock_pool_id = p.id
LEFT JOIN stock_moves m ON m.id = l.move_id
GROUP BY p.id, p.on_hand_quantity, p.reserved_quantity
HAVING p.reserved_quantity < 0
    OR p.reserved_quantity > p.on_hand_quantity
    OR p.reserved_quantity <> COALESCE(SUM(CASE WHEN m.state = 'ASSIGNED' THEN l.quantity ELSE 0 END), 0);

SELECT m.id, m.state, m.demand_quantity, count(l.id) AS line_count,
       COALESCE(sum(l.quantity), 0) AS line_quantity
FROM stock_moves m
LEFT JOIN stock_move_lines l ON l.move_id = m.id
GROUP BY m.id, m.state, m.demand_quantity
HAVING (m.state IN ('CONFIRMED', 'CANCELLED') AND count(l.id) <> 0)
    OR (m.state IN ('ASSIGNED', 'DONE') AND COALESCE(sum(l.quantity), 0) <> m.demand_quantity);