🐚 Code Snippet of the Day — July 15, 2026

Parallel HTTP Requests with Rate Limiting, Retry & Timeout in Bash

When you need to batch API calls (web scraping, health checks, webhook resends) and sequential curl is too slow while firing everything at once gets you rate-limited, use this:

```bash
#!/usr/bin/env bash

Usage: parallel_curl --file urls.txt --concurrency 10 --output jsonl

set -euo pipefail

CONCURRENCY="${CONCURRENCY:-5}"
MAX_RETRIES="${MAX_RETRIES:-3}"
TIMEOUT="${TIMEOUT:-15}"

curl_with_retry() {
local url="$1" attempt=0 last_status=0 total_duration=0
while [[ $attempt -lt $MAX_RETRIES ]]; do
attempt=$((attempt + 1))
local hdr=$(mktemp) body=$(mktemp)
last_status=$(curl -s -w "%{http_code}" -o "$body" -D "$hdr" \
--max-time "$TIMEOUT" --connect-timeout "$((TIMEOUT/3))" \
-H "Accept: application/json" "$url" 2>/dev/null) || last_status=0

if [[ $last_status -ge 200 && $last_status -lt 300 ]]; then
echo "${last_status}|${url}|$attempt"
rm -f "$hdr" "$body"; return 0
fi

# 429: respect Retry-After header
if [[ $last_status -eq 429 ]]; then
local wait_s=$(grep -i "^Retry-After:" "$hdr" 2>/dev/null | awk '{print $2}')
[[ -n "${wait_s:-}" ]] && sleep "$wait_s" || sleep "$attempt"
# 5xx: exponential backoff with jitter
elif [[ $last_status -ge 500 || $last_status -eq 0 ]]; then
sleep "$(echo "scale=2; $attempt * ($RANDOM/32767)" | bc -l 2>/dev/null || echo $attempt)"
else
break # non-retryable 4xx
fi
rm -f "$hdr" "$body"
done
echo "${last_status}|${url}|$attempt"
}

export -f curl_with_retry TIMEOUT MAX_RETRIES

GNU Parallel mode (or use background jobs with wait -n as fallback)

cat urls.txt | parallel --jobs "$CONCURREN...