scrapekit.py

#!/usr/bin/env python3
"""
ScrapeKit - Fast, flexible web scraping toolkit.
Extract structured data from any website in seconds.

Free tier:  100 records, CSV output, static pages
Premium:    Unlimited records, all formats, dynamic JS pages, proxy support

Usage:
  scrapekit --url "https://example.com" --output data.csv
  scrapekit --url "https://example.com" --format json --mode structured
  scrapekit --urls urls.txt --output results.csv --dynamic
  scrapekit --activate YOUR-LICENSE-KEY
"""

__version__ = "1.2.0"

import argparse
import csv
import json
import sys
import time
from pathlib import Path

try:
    import requests
    from bs4 import BeautifulSoup
except ImportError:
    print("Install dependencies: pip install requests beautifulsoup4 lxml")
    sys.exit(1)

# License gate
sys.path.insert(0, str(Path(__file__).parent))
try:
    from license_gate import LicenseGate
except ImportError:
    # Standalone mode - create minimal gate
    class LicenseGate:
        def __init__(self, n): pass
        def check(self, silent=False): return "trial"
        def is_premium(self): return True
        def require_premium(self, f=""): return True
        def handle_activate_flag(self, a=None): return None
        @staticmethod
        def add_activate_arg(p): p.add_argument('--activate', help='License key')

gate = LicenseGate("scrapekit")

FREE_RECORD_LIMIT = 100
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'}

PLAYWRIGHT_AVAILABLE = False
try:
    from playwright.sync_api import sync_playwright
    PLAYWRIGHT_AVAILABLE = True
except ImportError:
    pass


def scrape_static(url):
    resp = requests.get(url, headers=HEADERS, timeout=30)
    resp.raise_for_status()
    return BeautifulSoup(resp.text, 'lxml')


def scrape_dynamic(url):
    if not PLAYWRIGHT_AVAILABLE:
        print("  Playwright not available. Install: pip install playwright && playwright install chromium")
        return scrape_static(url)
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto(url, wait_until='networkidle', timeout=30000)
        content = page.content()
        browser.close()
    return BeautifulSoup(content, 'lxml')


def extract_links(soup, base_url=''):
    links = []
    for a in soup.find_all('a', href=True):
        href = a['href']
        text = a.get_text(strip=True)
        if href.startswith('/'):
            href = base_url.rstrip('/') + href
        links.append({'text': text, 'url': href})
    return links


def extract_tables(soup):
    results = []
    for table in soup.find_all('table'):
        headers = [th.get_text(strip=True) for th in table.find_all('th')]
        for tr in table.find_all('tr'):
            cells = [td.get_text(strip=True) for td in tr.find_all(['td', 'th'])]
            if cells and cells != headers:
                if headers:
                    results.append(dict(zip(headers, cells)))
                else:
                    results.append({'data': ', '.join(cells)})
    return results


def extract_structured(soup):
    items = []
    for selector in ['article', '.product', '.item', '.card', '.listing', '.result', 'li']:
        elements = soup.select(selector)
        if len(elements) >= 3:
            for el in elements:
                item = {}
                title_el = el.find(['h1', 'h2', 'h3', 'h4', 'a'])
                if title_el:
                    item['title'] = title_el.get_text(strip=True)
                    if title_el.name == 'a' and title_el.get('href'):
                        item['link'] = title_el['href']
                price_el = el.find(class_=lambda c: c and 'price' in c.lower()) if el.attrs else None
                if price_el:
                    item['price'] = price_el.get_text(strip=True)
                desc_el = el.find(['p', 'span'])
                if desc_el:
                    item['description'] = desc_el.get_text(strip=True)[:200]
                img_el = el.find('img')
                if img_el and img_el.get('src'):
                    item['image'] = img_el['src']
                if item:
                    items.append(item)
            if items:
                break
    return items


def extract_text(soup):
    for tag in soup(['script', 'style', 'nav', 'footer', 'header']):
        tag.decompose()
    blocks = []
    for el in soup.find_all(['h1', 'h2', 'h3', 'h4', 'p', 'li']):
        text = el.get_text(strip=True)
        if len(text) > 20:
            blocks.append({'tag': el.name, 'text': text[:500]})
    return blocks


def save_output(data_list, output_path, fmt='csv'):
    if not data_list:
        print("  No data extracted.")
        return 0

    # Free tier limit
    if not gate.is_premium() and len(data_list) > FREE_RECORD_LIMIT:
        print(f"  Free tier: limited to {FREE_RECORD_LIMIT} records ({len(data_list)} found)")
        print(f"  Upgrade for unlimited: https://tirandev.gumroad.com")
        data_list = data_list[:FREE_RECORD_LIMIT]

    path = Path(output_path)
    if fmt == 'json':
        path.write_text(json.dumps(data_list, indent=2, ensure_ascii=False), encoding='utf-8')
    elif fmt == 'excel':
        if not gate.require_premium("Excel export"):
            fmt = 'csv'
            path = path.with_suffix('.csv')
            # fall through to csv
        else:
            try:
                import openpyxl
                wb = openpyxl.Workbook()
                ws = wb.active
                if isinstance(data_list[0], dict):
                    keys = list(data_list[0].keys())
                    ws.append(keys)
                    for item in data_list:
                        ws.append([item.get(k, '') for k in keys])
                wb.save(str(path))
                print(f"  Saved {len(data_list)} records to {path}")
                return len(data_list)
            except ImportError:
                print("  openpyxl not installed. Falling back to CSV.")
                fmt = 'csv'
                path = path.with_suffix('.csv')

    if fmt == 'csv':
        if isinstance(data_list[0], dict):
            keys = list(data_list[0].keys())
            for item in data_list[1:]:
                for k in item:
                    if k not in keys:
                        keys.append(k)
            with open(path, 'w', newline='', encoding='utf-8') as f:
                w = csv.DictWriter(f, fieldnames=keys)
                w.writeheader()
                w.writerows(data_list)
        else:
            with open(path, 'w', newline='', encoding='utf-8') as f:
                csv.writer(f).writerows(data_list)

    print(f"  Saved {len(data_list)} records to {path}")
    return len(data_list)


def main():
    parser = argparse.ArgumentParser(
        description='ScrapeKit - Fast web scraping toolkit',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""Examples:
  scrapekit --url "https://books.toscrape.com" --output books.csv
  scrapekit --url "https://example.com" --mode links --format json
  scrapekit --url "https://spa-site.com" --dynamic --output data.csv
  scrapekit --activate YOUR-KEY""")
    parser.add_argument('--url', help='URL to scrape')
    parser.add_argument('--urls', help='File with URLs (one per line)')
    parser.add_argument('--output', '-o', default='output.csv', help='Output file')
    parser.add_argument('--format', '-f', choices=['csv', 'json', 'excel'], default='csv')
    parser.add_argument('--mode', '-m', choices=['auto', 'links', 'tables', 'text', 'structured'], default='auto')
    parser.add_argument('--dynamic', action='store_true', help='[Premium] Use Playwright for JS pages')
    parser.add_argument('--selector', '-s', help='CSS selector to target elements')
    parser.add_argument('--delay', type=float, default=1.0, help='Delay between requests (seconds)')
    parser.add_argument('--version', '-v', action='version', version=f'ScrapeKit {__version__}')
    LicenseGate.add_activate_arg(parser)
    args = parser.parse_args()

    gate.handle_activate_flag(args)
    if hasattr(args, 'activate') and args.activate:
        return

    gate.check()

    if args.dynamic and not gate.require_premium("Dynamic page scraping"):
        print("  Falling back to static scraping.")
        args.dynamic = False

    if args.format == 'excel' and not gate.is_premium():
        print("  Excel export is Premium. Using CSV instead.")
        args.format = 'csv'
        if args.output.endswith('.xlsx'):
            args.output = args.output.replace('.xlsx', '.csv')

    urls = []
    if args.url:
        urls = [args.url]
    elif args.urls:
        with open(args.urls) as f:
            urls = [l.strip() for l in f if l.strip()]
    else:
        parser.print_help()
        return

    all_data = []
    for url in urls:
        print(f"  Scraping: {url}")
        try:
            soup = scrape_dynamic(url) if args.dynamic else scrape_static(url)

            if args.selector:
                for el in soup.select(args.selector):
                    all_data.append({'url': url, 'text': el.get_text(strip=True)[:500]})
            elif args.mode == 'links':
                all_data.extend(extract_links(soup, url))
            elif args.mode == 'tables':
                all_data.extend(extract_tables(soup))
            elif args.mode == 'text':
                all_data.extend(extract_text(soup))
            elif args.mode == 'structured':
                all_data.extend(extract_structured(soup))
            else:
                structured = extract_structured(soup)
                all_data.extend(structured if structured else extract_text(soup))

            if len(urls) > 1:
                time.sleep(args.delay)
        except Exception as e:
            print(f"  Error: {e}")

    fmt = args.format
    if args.output.endswith('.json'):
        fmt = 'json'
    save_output(all_data, args.output, fmt)


if __name__ == '__main__':
    main()