Rate Limits

async function fetchWithRateLimit(url, options = {}) {
  const maxRetries = 3;
  let retries = 0;

  while (retries < maxRetries) {
    try {
      const response = await fetch(url, options);

      // Check rate limit headers
      const remaining = parseInt(response.headers.get('X-RateLimit-Remaining') || '0');
      const resetAfter = parseInt(response.headers.get('X-RateLimit-Reset-After') || '0');

      // Log rate limit status
      console.log(`Rate limit remaining: ${remaining}`);

      // Handle rate limit exceeded
      if (response.status === 429) {
        const retryAfter = parseInt(response.headers.get('Retry-After') || '60');
        console.log(`Rate limit exceeded. Retrying after ${retryAfter} seconds...`);

        // Wait before retrying
        await new Promise(resolve => setTimeout(resolve, retryAfter * 1000));
        retries++;
        continue;
      }

      // Success - return response
      if (response.ok) {
        return await response.json();
      }

      // Other error
      throw new Error(`HTTP error! status: ${response.status}`);

    } catch (error) {
      if (retries >= maxRetries - 1) {
        throw error;
      }
      retries++;
      await new Promise(resolve => setTimeout(resolve, 1000));
    }
  }

  throw new Error('Max retries exceeded');
}

// Usage example
const data = await fetchWithRateLimit('https://api.webpeek.dev/metadata?url=https://github.com');
console.log(data);

import requests
import time
from typing import Dict, Any

def fetch_with_rate_limit(url: str, max_retries: int = 3) -> Dict[str, Any]:
    """Fetch data from API with automatic rate limit handling."""
    retries = 0

    while retries < max_retries:
        try:
            response = requests.get(url)

            # Check rate limit headers
            remaining = int(response.headers.get('X-RateLimit-Remaining', 0))
            reset_after = int(response.headers.get('X-RateLimit-Reset-After', 0))

            print(f"Rate limit remaining: {remaining}")

            # Handle rate limit exceeded
            if response.status_code == 429:
                retry_after = int(response.headers.get('Retry-After', 60))
                print(f"Rate limit exceeded. Retrying after {retry_after} seconds...")
                time.sleep(retry_after)
                retries += 1
                continue

            # Raise for other HTTP errors
            response.raise_for_status()

            # Success
            return response.json()

        except requests.exceptions.RequestException as e:
            if retries >= max_retries - 1:
                raise e
            retries += 1
            time.sleep(1)

    raise Exception('Max retries exceeded')

# Usage example
try:
    data = fetch_with_rate_limit('https://api.webpeek.dev/metadata?url=https://github.com')
    print(data)
except Exception as e:
    print(f"Error: {e}")

class RateLimiter {
  constructor(requestsPerMinute = 60) {
    this.requestsPerMinute = requestsPerMinute;
    this.queue = [];
    this.processing = false;
    this.requestTimes = [];
  }

  async add(fn) {
    return new Promise((resolve, reject) => {
      this.queue.push({ fn, resolve, reject });
      this.process();
    });
  }

  async process() {
    if (this.processing || this.queue.length === 0) return;

    this.processing = true;

    while (this.queue.length > 0) {
      // Clean old request times
      const now = Date.now();
      this.requestTimes = this.requestTimes.filter(
        time => now - time < 60000
      );

      // Check if we can make a request
      if (this.requestTimes.length >= this.requestsPerMinute) {
        const oldestRequest = Math.min(...this.requestTimes);
        const waitTime = 60000 - (now - oldestRequest);
        console.log(`Rate limit reached. Waiting ${waitTime}ms...`);
        await new Promise(resolve => setTimeout(resolve, waitTime));
        continue;
      }

      // Process next request
      const { fn, resolve, reject } = this.queue.shift();
      this.requestTimes.push(Date.now());

      try {
        const result = await fn();
        resolve(result);
      } catch (error) {
        reject(error);
      }

      // Small delay between requests
      await new Promise(resolve => setTimeout(resolve, 100));
    }

    this.processing = false;
  }
}

// Usage example
const limiter = new RateLimiter(30); // 30 requests per minute

async function fetchMetadata(url) {
  return limiter.add(async () => {
    const response = await fetch(`https://api.webpeek.dev/metadata?url=${encodeURIComponent(url)}`);
    return response.json();
  });
}

// Batch process URLs
const urls = [
  'https://github.com',
  'https://stripe.com',
  'https://vercel.com',
  // ... more URLs
];

for (const url of urls) {
  const data = await fetchMetadata(url);
  console.log(`Processed: ${url}`);
}