Rate Limits & Pagination

import time
import requests

GOVERNOR_THRESHOLD = 0.05  # pause when < 5% remaining

def call_with_governor(url, headers):
    response = requests.get(url, headers=headers)

    limit = int(response.headers.get("RateLimit-Limit", 0))
    remaining = int(response.headers.get("RateLimit-Remaining", limit))
    reset = int(response.headers.get("RateLimit-Reset", 0))

    if limit and remaining / limit < GOVERNOR_THRESHOLD:
        # Sleep until the window resets — never hit a 429.
        time.sleep(reset + 1)

    return response

{
  "error": {
    "code": "RATE_LIMITED",
    "message": "Rate limit exceeded.",
    "status": 429,
    "details": {
      "retry_after_seconds": 12
    }
  }
}

{
  "data": [...],
  "pagination": {
    "page": 1,
    "page_size": 25,
    "total_count": 342,
    "total_pages": 14,
    "has_next": true,
    "has_previous": false
  }
}

import requests

page = 1
all_items = []

while True:
    response = requests.get(
        "https://www.closient.com/products/api/v1/products",
        params={"page": page, "page_size": 100},
        headers={"X-API-Key": "YOUR_API_KEY"},
    )
    data = response.json()
    all_items.extend(data["data"])

    if not data["pagination"]["has_next"]:
        break
    page += 1