{
 "id": "rfa7b566958a",
 "found": true,
 "parent_id": "re05aff784d1",
 "created_at": "2026-09-01 15:26:04.441613+00:00",
 "rows": [
  {
   "url": "https://raw.githubusercontent.com/KonstantinMB/exploreyc/master/backend/main.py",
   "title": null,
   "content_excerpt": "\"\"\"\nFastAPI backend for YC Company Scraper\n\"\"\"\n\n# Load environment variables from .env file (for local development)\nfrom dotenv import load_dotenv\nload_dotenv()\n\nfrom fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect, BackgroundTasks, Request, Depends, Header\nfrom fastapi.middleware.cors import CORSMiddleware\nfrom fastapi.responses import StreamingResponse, FileResponse, JSONResponse, Response, HTMLResponse\nfrom pydantic import BaseModel\nfrom typing import Optional, List\nimport asyncio\nimport csv\nimport hashlib\nimport io\nimport json\nimport logging\nimport os\nimport secrets\nimport requests\nfrom urllib.parse import urlparse\nfrom datetime import datetime, timedelta, timezone\nfrom collections import defaultdict\nfrom time import time\n\nfrom database_factory import get_database\nfrom scraper_service import ScraperService\nfrom email_service import EmailService\nfrom embedding_service import get_embedding_service\nfrom idea_filter import get_search_text_for_embedding\nfrom og_image_generator import get_og_image_generator\nfrom company_cache import CompanyCache\nfrom coresignal_service import coresignal_service\nfrom hiring_service import get_hiring_service\nfrom gamification_scoring import GamificationScorer\nfrom perplexity_service import get_perplexity_service\nfrom hero_service import build_verdict\nimport ratelimit\n\nlogging.basicConfig(level=logging.INFO)\nlogger = logging.getLogger(__name__)\n\n# Hide the internal API's docs/OpenAPI in production so admin/cron routes aren't published.\n# The public API keeps its own docs at /api/v1/docs.\n_IS_PROD = os.environ.get(\"ENV\", \"\").lower() == \"production\"\napp = FastAPI(\n    title=\"YC Company Scraper API\",\n    version=\"1.0.0\",\n    docs_url=None if _IS_PROD else \"/docs\",\n    redoc_url=None if _IS_PROD else \"/redoc\",\n    openapi_url=None if _IS_PROD else \"/openapi.json\",\n)\n\n# Static mount for re-hosted founder avatars (spec \u00a75.4, dev path).\nfrom fastapi.staticfiles import StaticFiles\n_STATIC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), \"static\")\nos.makedirs(os.path.join(_STATIC_DIR, \"avatars\"), exist_ok=True)\napp.mount(\"/static\", StaticFiles(directory=_STATIC_DIR), name=\"static\")\n\n\n# Rate Limiter for Research Endpoint\nclass ResearchRateLimiter:\n    \"\"\"Simple rate limiter for research requests (5 requests per minute per IP)\"\"\"\n    def __init__(self, max_requests: int = 5, window_seconds: int = 60):\n        self.max_requests = max_requests\n        self.window_seconds = window_seconds\n        self.requests: dict[str, list[float]] = defaultdict(list)\n\n    def is_allowed(self, client_ip: str) -> bool:\n        \"\"\"Check if request is allowed for this client IP\"\"\"\n        now = time()\n        # Remove old requests outside the window\n        self.requests[client_ip] = [\n            req_time for req_time in self.requests[client_ip]\n            if now - req_time < self.window_seconds\n        ]\n\n        # Check if within limit\n        if len(self.requests[client_ip]) >= self.max_requests:\n            return False\n\n        # Add new request\n        self.requests[client_ip].append(now)\n        return True\n\n    def get_reset_time(self, client_ip: str) -> int:\n        \"\"\"Get seconds until rate limit resets\"\"\"\n        if not self.requests[client_ip]:\n            return 0\n        oldest_request = min(self.requests[client_ip])\n        reset_time = int(oldest_request + self.window_seconds - time())\n        return max(0, reset_time)\n\n\nresearch_rate_limiter = ResearchRateLimiter(max_requests=5, window_seconds=60)\n\n# Additional IP-based limiters for expensive / abuse-prone endpoints.\n# Each one wraps an LLM call, an email send, or a long-running background job \u2014\n# without these an unauthenticated `for` loop can drain provider balances.\nvalidate_idea_limiter = ResearchRateLimiter(max_requests=5, window_seconds=60)\ngamified_predict_limiter = ResearchRateLimiter(max_requests=5, window_seconds=60)\nresearch_query_limiter = ResearchRateLimiter(max_requests=5, window_seconds=60)\nsubscribe_limiter = ResearchRateLimiter(max_requests=3, window_seconds=60)\nscrape_limiter = ResearchRateLimiter(max_requests=3, window_seconds=3600)\ndev_auth_limiter = ResearchRateLimiter(max_requests=10, window_seconds=3600)  # signup/login per IP\n\n\ndef _enforce_rate_limit(limiter: ResearchRateLimiter, request_obj: Request, label: str) -> None:\n    \"\"\"Raise HTTP 429 if the caller's IP has exceeded `limiter`.\"\"\"\n    client_ip = request_obj.client.host if request_obj.client else \"unknown\"\n    if not limiter.is_allowed(client_ip):\n        reset = limiter.get_reset_time(client_ip)\n        raise HTTPException(\n            status_code=429,\n            detail=f\"Rate limit exceeded for {label}. Try again in {reset} seconds.\",\n        )\n\n\ndef _log_startup_warnings():\n    \"\"\"Log warnings for missing config at startup\"\"\"\n    if not os.environ.get(\"RESEND_API_KEY\"):\n        logger.warning(\"RESEND_API_KEY not set - email subscriptions will save but verification emails will NOT be sent\")\n\n\ndef _safe_string(value) -> str:\n    \"\"\"Safely convert any value to string, handling lists and None\"\"\"\n    if isinstance(value, list):\n        value = value[0] if value else \"\"\n    if value is None:\n        return \"\"\n    return str(value)\n\n\n# CORS - allow localhost + production frontend (including www variant)\n_cors_origins = [\"http://localhost:5173\", \"http://localhost:3000\", \"https://www.workatastartup.com\"]\nif os.environ.get(\"FRONTEND_URL\"):\n    url = os.environ[\"FRONTEND_URL\"].rstrip(\"/\")\n    if url not in _cors_origins:\n        _cors_origins.append(url)\n    # Also allow www variant (e.g. www.exploreyc.com when FRONTEND_URL is exploreyc.com)\n    if url.startswith(\"https://\"):\n        domain = url.replace(\"https://\", \"\")\n        if not domain.startswith(\"www.\"):\n            www_url = f\"https://www.{domain}\"\n            if www_url not in _cors_origins:\n                _cors_origins.append(www_url)\n    elif url.startswith(\"http://\"):\n        domain = url.replace(\"http://\", \"\")\n        if not domain.startswith(\"www.\"):\n            www_url = f\"http://www.{domain}\"\n            if www_url not in _cors_origins:\n                _cors_origins.append(www_url)\nif os.environ.get(\"VERCEL_URL\"):\n    u = f\"https://{os.environ['VERCEL_URL']}\"\n    if u not in _cors_origins:\n        _cors_origins.append(u)\n\napp.add_middleware(\n    CORSMiddleware,\n    allow_origins=_cors_origins,\n    allow_credentials=True,\n    allow_methods=[\"GET\", \"POST\", \"PUT\", \"DELETE\", \"OPTIONS\", \"PATCH\"],\n    allow_headers=[\"Content-Type\", \"Authorization\", \"Accept\", \"Origin\", \"X-Requested-With\"],\n)\n\n# Initialize database, scraper service, email service, and company cache\ndb = get_database()\nscraper = ScraperService(db)\nfrom a16z_scraper_service import A16ZScraperService\na16z_scraper = A16ZScraperService(db)\nemail_service = EmailService()\ncompany_cache = CompanyCache()\n\n\ndef get_last_2_years_batches() -> List[str]:\n    \"\"\"Get YC batch names for the last 2 years (e.g., Summer 2024, Winter 2024, ...)\"\"\"\n    from datetime import date\n    current_year = date.today().year\n    seasons = [\"Winter\", \"Spring\", \"Summer\", \"Fall\"]\n    batches = []\n    for year in [current_year - 2, current_year - 1, current_year]:\n        for season in seasons:\n            batches.append(f\"{season} {year}\")\n    return batches\n\n\n@app.on_event(\"startup\")\nasync def startup_check_database():\n    \"\"\"Check database status on startup and load company cache\"\"\"\n    _log_startup_warnings()\n    company_cache.load(db)\n    stats = company_cache.get_stats()\n    logger.info(f\"Startup: Company cache loaded with {stats['total_companies']:,} companies (hiring: {stats['hiring']:,}, batches: {len(stats['by_batch'])})\")\n\n    # Load hiring board cache\n    hiring_service = get_hiring_service()\n    hiring_service.cache.load(db)\n    hiring_stats = hiring_service.get_hiring_stats()\n    logger.info(f\"Startup: Hiring cache loaded with {hiring_stats.get('totalJobs', 0):,} jobs from {hiring_stats.get('hiringCompanies', 0):,} companies\")\n\n    # Note: Removed automatic scraping on startup since we now have a complete database\n    # Use the /api/scrape endpoint or run scrape_all_by_batch.py to update companies\n\n# WebSocket connection manager\nclass ConnectionManager:\n    def __init__(self):\n        self.active_connections: List[WebSocket] = []\n\n    async def connect(self, websocket: WebSocket):\n        await websocket.accept()\n        self.active_connections.append(websocket)\n\n    def disconnect(self, websocket: WebSocket):\n        self.active_connections.remove(websocket)\n\n    async def broadcast(self, message: dict):\n        for connection in self.active_connections:\n            try:\n                await connection.send_json(message)\n            except:\n                pass\n\nmanager = ConnectionManager()\n\n\n# Admin Session Management\nadmin_sessions = {}  # {token: {username: str, expires: datetime}}\n\ndef create_admin_session(username: str) -> str:\n    \"\"\"Create a new admin session token\"\"\"\n    token = secrets.token_urlsafe(32)\n    admin_sessions[token] = {\n        \"username\": username,\n        \"expires\": datetime.utcnow() + timedelta(hours=24)\n    }\n    return token\n\ndef verify_admin_session(authorization: Optional[str] = Header(None)) -> dict:\n    \"\"\"Verify admin session token from Authorization header\"\"\"\n    if not authorization:\n        raise HTTPException(status_code=401, detail=\"Missing authorization header\")\n\n    if not authorization.startswith(\"Bearer \"):\n        raise HTTPException(status_code=401, detail=\"Invalid authorization header format\")\n\n    token = authorization.replace(\"Bearer \", \"\")\n\n    session = admin_sessions.get(token)\n    if not session:\n        raise HTTPException(status_code=401, detail=\"Invalid or expired session\")\n\n    if session[\"expires\"] < datetime.utcnow():\n        del admin_sessions[token]\n        raise HTTPException(status_code=401, detail=\"Session expired\")\n\n    return session\n\n\n# Pydantic models\nclass AdminLoginRequest(BaseModel):\n    username: str\n    password: str\n\n\n\nclass ScrapeRequest(BaseModel):\n    query: str = \"\"\n    batch: Optional[List[str]] = None\n    industry: Optional[List[str]] = None\n    region: Optional[List[str]] = None\n    is_hiring: Optional[bool] = None\n    top_company: Optional[bool] = None\n    nonprofit: Optional[bool] = None\n    hits_per_page: int = 1000\n    max_pages: int = 10\n    source: str = \"yc\"  # 'yc' (Algolia) or 'a16z' (portfolio page)\n\n\nclass CompanyFilter(BaseModel):\n    limit: int = 100\n    offset: int = 0\n    batch: Optional[str] = None\n    is_hiring: Optional[bool] = None\n    industry: Optional[str] = None\n    country: Optional[str] = None\n    search: Optional[str] = None\n    top_company: Optional[bool] = None\n    source: Optional[str] = None  # None -> YC only, 'all' -> every source, or a source key\n    merged: bool = False  # collapse same-domain rows across sources into one card\n    has_logo: bool = False  # only companies with a real, renderable logo\n\n\n# Background task storage\nactive_jobs = {}\n\n\n# Routes\n@app.get(\"/\")\nasync def root():\n    return {\"message\": \"YC Company Scraper API\", \"version\": \"1.0.0\"}\n\n\n@app.get(\"/api/health\")\nasync def health():\n    \"\"\"Health check endpoint\"\"\"\n    return {\"status\": \"healthy\", \"timestamp\": datetime.utcnow().isoformat()}\n\n\n# Admin Authentication Endpoints\n@app.post(\"/api/admin/login\")\nasync def admin_login(request: AdminLoginRequest):\n    \"\"\"Admin login endpoint\"\"\"\n    admin_username = os.environ.get(\"ADMIN_USERNAME\")\n    admin_password = os.environ.get(\"ADMIN_PASSWORD\")\n\n    if not admin_username or not admin_password:\n        raise HTTPException(status_code=500, detail=\"Admin credentials not configured\")\n\n    if request.username != admin_username or request.password != admin_password:\n        raise HTTPException(status_code=401, detail=\"Invalid credentials\")\n\n    token = create_admin_session(request.username)\n\n    return {\n        \"token\": token,\n        \"username\": request.username,\n        \"expires_in\": 86400  # 24 hours in seconds\n    }\n\n\n@app.post(\"/api/admin/logout\")\nasync def admin_logout(session: dict = Depends(verify_admin_session), authorization: str = Header(None)):\n    \"\"\"Admin logout endpoint\"\"\"\n    token = authorization.replace(\"Bearer \", \"\")\n    if token in admin_sessions:\n        del admin_sessions[token]\n\n    return {\"message\": \"Logged out successfully\"}\n\n\n@app.get(\"/api/admin/session\")\nasync def admin_session_check(session: dict = Depends(verify_admin_session)):\n    \"\"\"Check if admin session is valid\"\"\"\n    return {\n        \"username\": session[\"username\"],\n        \"expires\": session[\"expires\"].isoformat()\n    }\n\n\n@app.get(\"/api/admin/email-config\")\nasync def get_email_config(session: dict = Depends(verify_admin_session)):\n    \"\"\"Get email configuration status\"\"\"\n    from email_service import EmailService\n\n    email_service = EmailService()\n\n    return {\n        \"resend_configured\": bool(email_service.api_key and not email_service.api_key.startswith(\"re_your\")),\n        \"from_email\": email_service.from_email,\n        \"cron_secret_configured\": bool(os.environ.get(\"CRON_SECRET\"))\n    }\n\n\n@app.post(\"/api/admin/test-verification-email\")\nasync def send_test_verification_email(\n    request: dict,\n    session: dict = Depends(verify_admin_session)\n):\n    \"\"\"Send a test verification email\"\"\"\n    from email_service import EmailService\n    import secrets\n\n    email = request.get(\"email\")\n    if not email:\n        raise HTTPException(status_code=400, detail=\"Email address required\")\n\n    email_service = EmailService()\n\n    if not email_service.api_key or email_service.api_key.startswith(\"re_your\"):\n        raise HTTPException(\n            status_code=500,\n            detail=\"RESEND_API_KEY not configured. Set it in environment variables.\"\n        )\n\n    # Generate a test token\n    test_token = secrets.token_urlsafe(32)\n\n    # Send the email\n    success = email_service.send_verification_email(email, test_token)\n\n    if not success:\n        raise HTTPException(status_code=500, detail=\"Failed to send email. Check logs for details.\")\n\n    return {\n        \"success\": True,\n        \"message\": f\"Test verification email sent to {email}\",\n        \"email\": email\n    }\n\n\n@app.get(\"/api/admin/enrichment/stats\")\nasync def get_enrichment_stats(session: dict = Depends(verify_admin_session)):\n    \"\"\"Get enrichment statistics and progress\"\"\"\n    try:\n        with db.get_connection() as conn:\n            cursor = conn.cursor()\n\n            # Total companies (YC only \u2014 Coresignal enrichment is YC-scoped)\n            cursor.execute(\"SELECT COUNT(*) FROM companies WHERE source = 'yc'\")\n            total_companies = cursor.fetchone()[0]\n\n            # Companies enriched (have coresignal data)\n            cursor.execute(\"SELECT COUNT(*) FROM companies WHERE coresignal_last_updated IS NOT NULL\")\n            enriched_count = cursor.fetchone()[0]\n\n            # Companies with funding data\n            cursor.execute(\"SELECT COUNT(*) FROM companies WHERE funding_total_usd IS NOT NULL\")\n            with_funding_amount = cursor.fetchone()[0]\n\n            # Companies with funding rounds (but maybe no amount)\n            cursor.execute(\"SELECT COUNT(*) FROM companies WHERE funding_last_round_name IS NOT NULL\")\n            with_funding_rounds = cursor.fetchone()[0]\n\n            # Recent enrichments (last 24 hours)\n            # PostgreSQL syntax\n            cursor.execute(\"\"\"\n                SELECT COUNT(*) FROM companies\n                WHERE coresignal_last_updated IS NOT NULL\n                AND coresignal_last_updated > NOW() - INTERVAL '1 day'\n            \"\"\")\n            recent_enrichments = cursor.fetchone()[0]\n\n            # Top enriched companies\n            cursor.execute(\"\"\"\n                SELECT id, name, batch, funding_total_usd, funding_last_round_name, funding_last_round_date, investors_count\n                FROM companies\n                WHERE coresignal_last_updated IS NOT NULL\n                ORDER BY funding_total_usd DESC NULLS LAST\n                LIMIT 10\n            \"\"\")\n            top_enriched = []\n            for row in cursor.fetchall():\n                top_enriched.append({\n                    \"id\": row[0],\n                    \"name\": row[1],\n                    \"batch\": row[2],\n                    \"funding_total_usd\": row[3],\n                    \"funding_last_round_name\": row[4],\n                    \"funding_last_round_date\": row[5],\n                    \"investors_count\": row[6]\n                })\n\n            return {\n                \"total_companies\": total_companies,\n                \"enriched_count\": enriched_count,\n                \"unenriched_count\": total_companies - enriched_count,\n                \"with_funding_amount\": with_funding_amount,\n                \"with_funding_rounds\": with_funding_rounds,\n                \"recent_enrichments_24h\": recent_enrichments,\n                \"enrichment_percentage\": round((enriched_count / total_companies * 100), 2) if total_companies > 0 else 0,\n                \"funding_data_percentage\": round((with_funding_amount / enriched_count * 100), 2) if enriched_count > 0 else 0,\n                \"top_enriched\": top_enriched,\n                \"coresignal_enabled\": coresignal_service.enabled\n            }\n\n    except Exception as e:\n        logger.error(f\"Error getting enrichment stats: {e}\")\n        raise HTTPException(status_code=500, detail=str(e))\n\n\n@app.post(\"/api/scrape\")\nasync def start_scrape(request: ScrapeRequest, background_tasks: BackgroundTasks, request_obj: Request):\n    \"\"\"Start a new scraping job\"\"\"\n    _enforce_rate_limit(scrape_limiter, request_obj, \"scrape jobs\")\n\n    # Create job in database\n    job_id = db.create_scrape_job({\n        'source': request.source,\n        'query': request.query,\n        'batch': request.batch,\n        'industry': request.industry,\n        'region': request.region,\n        'is_hiring': request.is_hiring,\n        'top_company': request.top_company,\n        'nonprofit': request.nonprofit,\n        'hits_per_page': request.hits_per_page,\n        'max_pages': request.max_pages,\n    })\n\n    # Progress callback for WebSocket updates\n    async def progress_callback(data):\n        await manager.broadcast(data)\n\n    # Start scraping in background\n    async def run_scrape():\n        try:\n            if request.source == \"a16z\":\n                total = await a16z_scraper.scrape_companies(\n                    job_id=job_id,\n                    progress_callback=progress_callback,\n                )\n            else:\n                total = await scraper.scrape_companies(\n                    job_id=job_id,\n                    query=request.query,\n                    batch=request.batch,\n                    industry=request.industry,\n                    region=request.region,\n                    is_hiring=request.is_hiring,\n                    top_company=request.top_company,\n                    nonprofit=request.nonprofit,\n                    hits_per_page=request.hits_per_page,\n                    max_pages=request.max_pages,\n                    progress_callback=progress_callback\n                )\n            active_jobs[job_id] = {'status': 'completed', 'total': total}\n        except Exception as e:\n            active_jobs[job_id] = {'status': 'failed', 'error': str(e)}\n        finally:\n            # Refresh in-memory cache after scrape completes (success or failure)\n            company_cache.refresh(db)\n\n    # Schedule background task\n    asyncio.create_task(run_scrape())\n    active_jobs[job_id] = {'status': 'running'}\n\n    return {\n        \"job_id\": job_id,\n        \"status\": \"started\",\n        \"message\": \"Scraping job started\"\n    }\n\n\n@app.get(\"/api/scrape/status/{job_id}\")\nasync def get_scrape_status(job_id: int):\n    \"\"\"Get the status of a scraping job\"\"\"\n    job = db.get_scrape_job(job_id)\n\n    if not job:\n        raise HTTPException(status_code=404, detail=\"Job not found\")\n\n    return {\n        \"job_id\": job_id,\n        \"status\": job['status'],\n        \"total_scraped\": job['total_scraped'],\n        \"current_page\": job['current_page'],\n        \"error\": job['erro"
  }
 ],
 "notes": [
  "WARNING: 2 of 3 URLs failed and were skipped: https://raw.githubusercontent.com/KonstantinMB/exploreyc/master/backend/scraper.py, https://github.com/KonstantinMB/exploreyc/blob/master/backend/scraper.py",
  "Fetch result set: re05aff784d1 (1 rows), reference it as FROM re05aff784d1 in follow-up queries"
 ]
}