| | |
| | """ |
| | GODDESS CRAWLER - High-performance async web scraper |
| | Sellable tool - £20 one-time |
| | |
| | Usage: python goddess_crawler.py seeds.txt output_dir/ --workers 100 |
| | """ |
| | import asyncio |
| | import aiohttp |
| | import argparse |
| | import os |
| | import re |
| | from pathlib import Path |
| | from typing import Set |
| | import time |
| |
|
| | class GoddessCrawler: |
| | def __init__(self, output_dir: str, workers: int = 100, timeout: int = 15): |
| | self.output = Path(output_dir) |
| | self.output.mkdir(exist_ok=True) |
| | self.workers = workers |
| | self.timeout = aiohttp.ClientTimeout(total=timeout) |
| | self.visited: Set[str] = set() |
| | self.count = 0 |
| | self.bytes = 0 |
| | self.errors = 0 |
| | self.start = time.time() |
| | |
| | def strip_html(self, html: str) -> str: |
| | html = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL|re.I) |
| | html = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL|re.I) |
| | html = re.sub(r'<[^>]+>', ' ', html) |
| | html = re.sub(r'\s+', ' ', html) |
| | return html.strip() |
| | |
| | async def fetch(self, session: aiohttp.ClientSession, url: str) -> str | None: |
| | if url in self.visited: |
| | return None |
| | self.visited.add(url) |
| | try: |
| | async with session.get(url) as r: |
| | if r.status == 200 and 'text' in r.content_type: |
| | html = await r.text() |
| | return self.strip_html(html) |
| | except: |
| | self.errors += 1 |
| | return None |
| | |
| | async def process(self, session: aiohttp.ClientSession, url: str): |
| | text = await self.fetch(session, url) |
| | if text and len(text) > 200: |
| | self.count += 1 |
| | path = self.output / f"p_{self.count:08d}.txt" |
| | content = f"URL: {url}\n\n{text}" |
| | path.write_text(content) |
| | self.bytes += len(content) |
| | |
| | async def crawl(self, seeds: list[str]): |
| | headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/121.0.0.0"} |
| | conn = aiohttp.TCPConnector(limit=self.workers, limit_per_host=10) |
| | async with aiohttp.ClientSession(connector=conn, timeout=self.timeout, headers=headers) as session: |
| | sem = asyncio.Semaphore(self.workers) |
| | async def bounded(url): |
| | async with sem: |
| | await self.process(session, url) |
| | |
| | |
| | async def status(): |
| | while True: |
| | await asyncio.sleep(15) |
| | elapsed = time.time() - self.start |
| | rate = self.bytes / elapsed / 1e6 |
| | print(f"[{elapsed:.0f}s] {self.count} pages | {self.bytes/1e9:.2f}GB | {self.errors} err | {rate:.1f}MB/s") |
| | |
| | status_task = asyncio.create_task(status()) |
| | await asyncio.gather(*[bounded(url) for url in seeds]) |
| | status_task.cancel() |
| | |
| | print(f"\nDone! {self.count} pages, {self.bytes/1e9:.2f}GB") |
| |
|
| | def main(): |
| | p = argparse.ArgumentParser(description="High-performance web crawler") |
| | p.add_argument("seeds", help="File with URLs, one per line") |
| | p.add_argument("output", help="Output directory") |
| | p.add_argument("--workers", type=int, default=100, help="Concurrent connections") |
| | p.add_argument("--timeout", type=int, default=15, help="Request timeout") |
| | args = p.parse_args() |
| | |
| | seeds = Path(args.seeds).read_text().strip().split('\n') |
| | print(f"Loaded {len(seeds)} seeds, {args.workers} workers") |
| | |
| | crawler = GoddessCrawler(args.output, args.workers, args.timeout) |
| | asyncio.run(crawler.crawl(seeds)) |
| |
|
| | if __name__ == "__main__": |
| | main() |
| |
|
