Scrape Google Search results with Browserless

# pip install requests beautifulsoup4
import csv
import os
from urllib.parse import parse_qs, urlencode, urljoin, urlparse

import requests
from bs4 import BeautifulSoup

api_key = os.environ["BROWSERLESS_API_TOKEN"]
api_base = "https://production-sfo.browserless.io/scrapers/google"
google_url = "https://www.google.com/search?" + urlencode({
    "q": "best laptop", "hl": "en", "gl": "us"
})

response = requests.post(
    f"{api_base}/search",
    params={"token": api_key},
    json={"url": google_url},
    timeout=150,
)
response.raise_for_status()
result = response.json()
if not result.get("ok"):
    raise RuntimeError(str(result.get("error")))


def resolve(href):
    url = urljoin("https://www.google.com", href)
    if not urlparse(url).netloc.endswith("google.com"):
        return url
    goto = requests.post(
        f"{api_base}/goto",
        params={"token": api_key},
        json={"url": url},
        timeout=60,
    )
    location = (goto.json().get("data") or {}).get("location") if goto.ok else None
    if location:
        return location
    query = parse_qs(urlparse(url).query)
    return (query.get("url") or query.get("q") or [None])[0]


soup = BeautifulSoup(result["data"]["html"], "html.parser")
results = []
seen = set()
for h3 in soup.select("a[href] h3"):
    title = h3.get_text(" ", strip=True)
    url = resolve(h3.find_parent("a", href=True)["href"])
    if not url or urlparse(url).scheme not in ("http", "https"):
        continue
    if title and url not in seen:
        results.append({"title": title, "url": url})
        seen.add(url)

with open("google-results.csv", "w", newline="", encoding="utf-8") as f:
    writer = csv.DictWriter(f, fieldnames=["title", "url"])
    writer.writeheader()
    writer.writerows(results)
print(f"Exported {len(results)} links to google-results.csv")