Best Proxies for Google Scraping — We Handle  CAPTCHAs|

# Rotating residential proxy (new IP per request)
Host: gate.birdproxies.com
Port: 7777
Username: YOUR_USER-country-us
Password: YOUR_PASSWORD

# Country-specific for local results
Username: YOUR_USER-country-gb  # UK results
Username: YOUR_USER-country-de  # German results
Username: YOUR_USER-country-jp  # Japanese results

# Python Google SERP scraper with proxy rotation
import requests
from bs4 import BeautifulSoup
import time
import random

PROXY = "http://USER-country-us:[email protected]:7777"
proxies = {"http": PROXY, "https": PROXY}

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/120.0.0.0 Safari/537.36"
}

def scrape_serp(keyword, country="us"):
    proxy = f"http://USER-country-{country}:[email protected]:7777"
    url = f"https://www.google.com/search?q={keyword}&gl={country}&hl=en"

    response = requests.get(
        url,
        proxies={"http": proxy, "https": proxy},
        headers=headers,
        timeout=15
    )

    if response.status_code == 200:
        soup = BeautifulSoup(response.text, 'html.parser')
        results = soup.select('div.g')
        return [r.select_one('h3').text for r in results if r.select_one('h3')]
    return []

# Scrape with delays between requests
keywords = ["best crm software", "project management tools", "email marketing"]
for kw in keywords:
    results = scrape_serp(kw)
    print(f"{kw}: {len(results)} results")
    time.sleep(random.uniform(2, 5))  # Random delay

import time
import random
import requests
from requests.exceptions import RequestException

def scrape_with_retry(url, proxies, max_retries=3):
    """Scrape with exponential backoff retry logic"""
    for attempt in range(max_retries):
        try:
            # Random delay between 2-6 seconds
            time.sleep(random.uniform(2, 6))

            response = requests.get(
                url, proxies=proxies,
                headers=headers, timeout=15
            )

            if response.status_code == 200:
                return response

            if response.status_code == 429:
                # Rate limited - wait longer
                wait = (2 ** attempt) * 5
                print(f"Rate limited. Waiting {wait}s...")
                time.sleep(wait)
                continue

        except RequestException as e:
            print(f"Attempt {attempt+1} failed: {e}")
            time.sleep(2 ** attempt)

    return None

# Batch scraping with progress tracking
keywords = load_keywords("keywords.csv")  # Your keyword list
results = {}

for i, kw in enumerate(keywords):
    url = f"https://www.google.com/search?q={kw}"
    response = scrape_with_retry(url, proxies)
    if response:
        results[kw] = parse_serp(response.text)
    print(f"Progress: {i+1}/{len(keywords)}")