{ "id": "re25a0d539fd", "found": true, "parent_id": "recb5c3b80cc", "created_at": "2026-09-01 15:26:13.330228+00:00", "rows": [ { "url": "https://raw.githubusercontent.com/KonstantinMB/exploreyc/master/backend/scraper_service.py", "title": null, "content_excerpt": "\"\"\"\nScraper service for background job processing\n\"\"\"\n\nimport asyncio\nimport requests\nimport json\nfrom typing import Dict, Optional, List, Callable\nfrom urllib.parse import urlencode\nfrom database import Database\nfrom enhanced_geocoding import EXTENDED_LOCATION_COORDS\n\n\nclass ScraperService:\n \"\"\"Service to handle background scraping jobs\"\"\"\n\n ALGOLIA_URL = \"https://45bwzj1sgc-dsn.algolia.net/1/indexes/*/queries\"\n APP_ID = \"45BWZJ1SGC\"\n API_KEY = \"NzllNTY5MzJiZGM2OTY2ZTQwMDEzOTNhYWZiZGRjODlhYzVkNjBmOGRjNzJiMWM4ZTU0ZDlhYTZjOTJiMjlhMWFuYWx5dGljc1RhZ3M9eWNkYyZyZXN0cmljdEluZGljZXM9WUNDb21wYW55X3Byb2R1Y3Rpb24lMkNZQ0NvbXBhbnlfQnlfTGF1bmNoX0RhdGVfcHJvZHVjdGlvbiZ0YWdGaWx0ZXJzPSU1QiUyMnljZGNfcHVibGljJTIyJTVE\"\n\n # Use enhanced geocoding database with 150+ cities worldwide\n LOCATION_COORDS = EXTENDED_LOCATION_COORDS\n\n def __init__(self, db: Database):\n self.db = db\n self.session = requests.Session()\n self.session.headers.update({\n 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',\n 'Accept': 'application/json',\n 'Content-Type': 'application/x-www-form-urlencoded',\n 'Origin': 'https://www.ycombinator.com',\n 'Referer': 'https://www.ycombinator.com/'\n })\n\n def _build_facet_filters(self, filters: Dict[str, List[str]]) -> List:\n \"\"\"Build facet filters for Algolia query\"\"\"\n facet_filters = []\n for key, values in filters.items():\n if values:\n facet_filters.append([f\"{key}:{value}\" for value in values])\n return facet_filters\n\n def _geocode_location(self, location: str) -> Optional[Dict]:\n \"\"\"Simple geocoding using predefined coordinates\"\"\"\n if not location:\n return None\n\n location = location.strip()\n\n # Try exact match\n if location in self.LOCATION_COORDS:\n return self.LOCATION_COORDS[location]\n\n # Try partial match\n for city, coords in self.LOCATION_COORDS.items():\n if city.lower() in location.lower():\n return coords\n\n return None\n\n def _enrich_company_data(self, company: Dict) -> Dict:\n \"\"\"Enrich company data with geocoding\"\"\"\n location = company.get('all_locations', '')\n\n if location:\n coords = self._geocode_location(location)\n if coords:\n company['latitude'] = coords['lat']\n company['longitude'] = coords['lng']\n company['country'] = coords['country']\n\n return company\n\n async def scrape_companies(self,\n job_id: int,\n query: str = \"\",\n batch: Optional[List[str]] = None,\n industry: Optional[List[str]] = None,\n region: Optional[List[str]] = None,\n is_hiring: Optional[bool] = None,\n top_company: Optional[bool] = None,\n nonprofit: Optional[bool] = None,\n hits_per_page: int = 1000,\n max_pages: int = 10,\n progress_callback: Optional[Callable] = None) -> int:\n \"\"\"\n Scrape YC companies with filters (async)\n \"\"\"\n\n try:\n # Build facet filters\n filters = {}\n if batch:\n filters['batch'] = batch\n if industry:\n filters['industries'] = industry\n if region:\n filters['regions'] = region\n if is_hiring is not None:\n filters['isHiring'] = [str(is_hiring).lower()]\n if top_company is not None:\n filters['top_company'] = [str(top_company).lower()]\n if nonprofit is not None:\n filters['nonprofit'] = [str(nonprofit).lower()]\n\n facet_filters = self._build_facet_filters(filters)\n\n total_scraped = 0\n page = 0\n\n while page < max_pages:\n # Build request payload\n params = {\n 'facetFilters': json.dumps(facet_filters) if facet_filters else '',\n 'facets': json.dumps([\n \"app_answers\", \"app_video_public\", \"batch\", \"demo_day_video_public\",\n \"industries\", \"isHiring\", \"nonprofit\", \"question_answers\",\n \"regions\", \"subindustry\", \"top_company\"\n ]),\n 'hitsPerPage': hits_per_page,\n 'maxValuesPerFacet': 1000,\n 'page': page,\n 'query': query,\n 'tagFilters': '',\n 'analyticsTags': 'ycdc',\n 'restrictIndices': 'YCCompany_production,YCCompany_By_Launch_Date_production',\n 'tagFilters': '[\"ycdc_public\"]'\n }\n\n # Build request body\n request_body = {\n \"requests\": [\n {\n \"indexName\": \"YCCompany_production\",\n \"params\": urlencode(params)\n }\n ]\n }\n\n # Make request\n url = f\"{self.ALGOLIA_URL}?x-algolia-agent=Algolia%20for%20JavaScript%20(3.35.1)&x-algolia-application-id={self.APP_ID}&x-algolia-api-key={self.API_KEY}\"\n\n # Run in thread pool to not block async\n loop = asyncio.get_event_loop()\n response = await loop.run_in_executor(\n None,\n lambda: self.session.post(url, json=request_body)\n )\n response.raise_for_status()\n data = response.json()\n\n if not data.get('results') or not data['results']:\n break\n\n result = data['results'][0]\n hits = result.get('hits', [])\n\n if not hits:\n break\n\n # Insert companies into database with change tracking\n for company in hits:\n # Enrich with geocoding\n enriched_company = self._enrich_company_data(company)\n\n # Check if company exists and track changes\n company_id = enriched_company.get('id')\n if company_id:\n existing = self.db.get_company_by_id(company_id)\n\n # Insert/update company\n self.db.insert_company(enriched_company)\n\n # Track changes\n if existing is None:\n # New company created\n self.db.log_change(\n company_id=company_id,\n change_type='created',\n new_value=enriched_company.get('name')\n )\n else:\n # Check for hiring status changes\n old_hiring = existing.get('is_hiring')\n new_hiring = enriched_company.get('isHiring')\n\n if old_hiring != new_hiring:\n if new_hiring:\n self.db.log_change(\n company_id=company_id,\n change_type='hiring_started',\n field_name='is_hiring',\n old_value='false',\n new_value='true'\n )\n else:\n self.db.log_change(\n company_id=company_id,\n change_type='hiring_stopped',\n field_name='is_hiring',\n old_value='true',\n new_value='false'\n )\n\n # Check for batch changes\n old_batch = existing.get('batch')\n new_batch = enriched_company.get('batch')\n\n if old_batch and new_batch and old_batch != new_batch:\n self.db.log_change(\n company_id=company_id,\n change_type='batch_changed',\n field_name='batch',\n old_value=old_batch,\n new_value=new_batch\n )\n else:\n # No ID, just insert\n self.db.insert_company(enriched_company)\n\n total_scraped += 1\n\n # Update job status\n self.db.update_scrape_job(job_id, 'running', total_scraped, page + 1)\n\n # Call progress callback if provided\n if progress_callback:\n await progress_callback({\n 'job_id': job_id,\n 'status': 'running',\n 'total_scraped': total_scraped,\n 'current_page': page + 1\n })\n\n # Check if there are more pages (continue until we get fewer results than requested)\n # Don't rely on nbPages as YC API seems to return incorrect values\n if len(hits) < hits_per_page:\n # Got fewer results than requested, we've reached the end\n break\n\n page += 1\n\n # Small delay to be nice to the API\n await asyncio.sleep(0.5)\n\n # Mark job as completed\n self.db.update_scrape_job(job_id, 'completed', total_scraped, page)\n\n if progress_callback:\n await progress_callback({\n 'job_id': job_id,\n 'status': 'completed',\n 'total_scraped': total_scraped,\n 'current_page': page\n })\n\n return total_scraped\n\n except Exception as e:\n # Mark job as failed\n self.db.update_scrape_job(job_id, 'failed', total_scraped, page, str(e))\n\n if progress_callback:\n await progress_callback({\n 'job_id': job_id,\n 'status': 'failed',\n 'error': str(e),\n 'total_scraped': total_scraped,\n 'current_page': page\n })\n\n raise e\n" }, { "url": "https://api.exploreyc.com/api/stats", "title": null, "content_excerpt": "{\"total_companies\":6217,\"total_all_companies\":24983,\"hiring\":1556,\"by_batch\":{\"Winter 2022\":399,\"Summer 2021\":391,\"Winter 2021\":336,\"Winter 2023\":275,\"Winter 2024\":251,\"Summer 2024\":248,\"Summer 2026\":239,\"Summer 2022\":234,\"Winter 2020\":229,\"Summer 2023\":220,\"Summer 2020\":208,\"Winter 2026\":200,\"Spring 2026\":197,\"Winter 2019\":195,\"Summer 2019\":176,\"Winter 2025\":167,\"Summer 2025\":167,\"Fall 2025\":147,\"Winter 2018\":146,\"Spring 2025\":145,\"Summer 2018\":131,\"Summer 2017\":125,\"Winter 2016\":122,\"Winter 2017\":116,\"Winter 2015\":111,\"Summer 2015\":105,\"Summer 2016\":102,\"Fall 2024\":93,\"Summer 2012\":83,\"Summer 2014\":78,\"Winter 2014\":74,\"Winter 2012\":66,\"Summer 2011\":60,\"Summer 2013\":52,\"Winter 2013\":46,\"Winter 2011\":44,\"Summer 2010\":36,\"Fall 2026\":30,\"Winter 2010\":27,\"Summer 2009\":26,\"Summer 2008\":22,\"Winter 2008\":21,\"Summer 2007\":19,\"Winter 2009\":16,\"Winter 2007\":13,\"Summer 2006\":11,\"Summer 2005\":9,\"Winter 2006\":7,\"Winter 2027\":1,\"Unspecified\":1},\"by_industry\":{\"B2B\":3167,\"Consumer\":892,\"Healthcare\":701,\"Fintech\":655,\"Industrials\":453,\"Real Estate and Construction\":162,\"Education\":125,\"Government\":43,\"Unspecified\":18,\"Software\":1},\"by_country\":{\"United States\":4569,\"Remote\":178,\"India\":156,\"Canada\":126,\"Mexico\":71,\"France\":59,\"Singapore\":53,\"Germany\":47,\"Brazil\":39,\"United Kingdom\":32},\"by_status\":{\"Active\":4425,\"Acquired\":752,\"Inactive\":1017,\"Public\":23},\"by_batch_industry\":{\"Fall 2025\":{\"B2B\":90,\"Industrials\":13,\"Healthcare\":11,\"Fintech\":8,\"Consumer\":18,\"Real Estate and Construction\":5,\"Government\":2},\"Summer 2026\":{\"Healthcare\":22,\"Industrials\":57,\"B2B\":123,\"Fintech\":17,\"Consumer\":11,\"Education\":1,\"Real Estate and Construction\":6,\"Government\":2},\"Winter 2025\":{\"B2B\":104,\"Healthcare\":11,\"Fintech\":12,\"Industrials\":18,\"Consumer\":10,\"Education\":6,\"Government\":4,\"Real Estate and Construction\":2},\"Fall 2026\":{\"B2B\":16,\"Consumer\":5,\"Industrials\":6,\"Government\":1,\"Fintech\":2},\"Summer 2021\":{\"Healthcare\":65,\"B2B\":180,\"Fintech\":62,\"Consumer\":40,\"Industrials\":20,\"Education\":15,\"Real Estate and Construction\":8,\"Government\":1},\"Summer 2025\":{\"B2B\":116,\"Real Estate and Construction\":4,\"Healthcare\":13,\"Industrials\":14,\"Fintech\":7,\"Consumer\":10,\"Government\":1,\"Education\":2},\"Summer 2013\":{\"Education\":2,\"Healthcare\":4,\"Fintech\":4,\"B2B\":26,\"Consumer\":15,\"Real Estate and Construction\":1},\"Winter 2017\":{\"Healthcare\":18,\"B2B\":41,\"Real Estate and Construction\":5,\"Fintech\":12,\"Consumer\":22,\"Industrials\":12,\"Government\":1,\"Unspecified\":3,\"Education\":2},\"Winter 2026\":{\"B2B\":126,\"Software\":1,\"Fintech\":18,\"Consumer\":8,\"Healthcare\":16,\"Industrials\":28,\"Real Estate and Construction\":3},\"Winter 2020\":{\"B2B\":101,\"Fintech\":32,\"Consumer\":29,\"Industrials\":17,\"Healthcare\":36,\"Real Estate and Construction\":9,\"Education\":5},\"Winter 2021\":{\"B2B\":161,\"Healthcare\":42,\"Industrials\":23,\"Fintech\":51,\"Education\":13,\"Consumer\":40,\"Real Estate and Construction\":5,\"Government\":1},\"Winter 2012\":{\"Education\":2,\"Consumer\":21,\"B2B\":30,\"Fintech\":5,\"Industrials\":3,\"Real Estate and Construction\":3,\"Healthcare\":2},\"Winter 2024\":{\"Healthcare\":29,\"B2B\":158,\"Industrials\":9,\"Consumer\":26,\"Fintech\":21,\"Real Estate and Construction\":3,\"Government\":1,\"Education\":4},\"Spring 2026\":{\"B2B\":116,\"Industrials\":26,\"Fintech\":20,\"Consumer\":13,\"Real Estate and Construction\":4,\"Healthcare\":17,\"Government\":1},\"Summer 2020\":{\"Fintech\":23,\"B2B\":109,\"Real Estate and Construction\":5,\"Consumer\":26,\"Healthcare\":33,\"Industrials\":6,\"Government\":2,\"Education\":4},\"Summer 2017\":{\"B2B\":27,\"Education\":6,\"Consumer\":33,\"Industrials\":16,\"Healthcare\":19,\"Government\":4,\"Fintech\":14,\"Real Estate and Construction\":6},\"Winter 2010\":{\"B2B\":17,\"Consumer\":9,\"Real Estate and Construction\":1},\"Winter 2022\":{\"Fintech\":90,\"Consumer\":39,\"B2B\":180,\"Healthcare\":44,\"Industrials\":22,\"Real Estate and Construction\":15,\"Education\":8,\"Government\":1},\"Summer 2011\":{\"Consumer\":18,\"Healthcare\":3,\"B2B\":30,\"Education\":3,\"Fintech\":4,\"Real Estate and Construction\":2},\"Winter 2009\":{\"Real Estate and Construction\":2,\"Consumer\":9,\"B2B\":5},\"Summer 2016\":{\"Consumer\":32,\"B2B\":31,\"Industrials\":10,\"Healthcare\":11,\"Fintech\":7,\"Education\":4,\"Government\":1,\"Real Estate and Construction\":4,\"Unspecified\":2},\"Winter 2019\":{\"B2B\":81,\"Fintech\":23,\"Consumer\":37,\"Industrials\":15,\"Education\":7,\"Real Estate and Construction\":5,\"Healthcare\":25,\"Government\":2},\"Summer 2012\":{\"Consumer\":27,\"B2B\":43,\"Education\":5,\"Healthcare\":4,\"Industrials\":1,\"Fintech\":3},\"Summer 2023\":{\"B2B\":152,\"Fintech\":17,\"Healthcare\":28,\"Consumer\":10,\"Real Estate and Construction\":6,\"Education\":3,\"Industrials\":4},\"Winter 2018\":{\"Healthcare\":27,\"B2B\":55,\"Consumer\":36,\"Government\":2,\"Fintech\":11,\"Industrials\":8,\"Education\":4,\"Real Estate and Construction\":3},\"Winter 2015\":{\"Fintech\":9,\"B2B\":44,\"Government\":3,\"Consumer\":22,\"Healthcare\":20,\"Industrials\":4,\"Real Estate and Construction\":6,\"Unspecified\":3},\"Winter 2023\":{\"B2B\":186,\"Industrials\":6,\"Fintech\":30,\"Healthcare\":23,\"Consumer\":22,\"Real Estate and Construction\":4,\"Education\":3,\"Government\":1},\"Winter 2016\":{\"Consumer\":22,\"Fintech\":10,\"B2B\":52,\"Healthcare\":19,\"Unspecified\":3,\"Government\":1,\"Real Estate and Construction\":3,\"Education\":2,\"Industrials\":10},\"Summer 2018\":{\"B2B\":45,\"Consumer\":19,\"Fintech\":14,\"Healthcare\":36,\"Education\":4,\"Industrials\":9,\"Real Estate and Construction\":3,\"Government\":1},\"Summer 2010\":{\"Fintech\":2,\"Consumer\":16,\"B2B\":16,\"Education\":1,\"Real Estate and Construction\":1},\"Summer 2015\":{\"Healthcare\":11,\"Consumer\":26,\"Fintech\":9,\"B2B\":41,\"Industrials\":10,\"Education\":3,\"Real Estate and Construction\":3,\"Unspecified\":2},\"Summer 2022\":{\"Industrials\":14,\"B2B\":120,\"Fintech\":44,\"Consumer\":21,\"Healthcare\":25,\"Real Estate and Construction\":9,\"Education\":1},\"Summer 2007\":{\"Consumer\":12,\"B2B\":7},\"Winter 2008\":{\"Consumer\":8,\"B2B\":12,\"Fintech\":1},\"Summer 2014\":{\"Consumer\":22,\"B2B\":27,\"Healthcare\":10,\"Fintech\":10,\"Industrials\":6,\"Real Estate and Construction\":1,\"Government\":1,\"Unspecified\":1},\"Fall 2024\":{\"B2B\":56,\"Consumer\":11,\"Real Estate and Construction\":4,\"Government\":1,\"Education\":1,\"Healthcare\":9,\"Fintech\":6,\"Industrials\":5},\"Winter 2014\":{\"B2B\":37,\"Consumer\":18,\"Fintech\":4,\"Industrials\":5,\"Unspecified\":4,\"Education\":2,\"Healthcare\":2,\"Real Estate and Construction\":2},\"Summer 2024\":{\"B2B\":159,\"Fintech\":12,\"Consumer\":20,\"Government\":4,\"Industrials\":25,\"Healthcare\":23,\"Real Estate and Construction\":4,\"Education\":1},\"Spring 2025\":{\"B2B\":99,\"Healthcare\":10,\"Government\":2,\"Industrials\":16,\"Consumer\":8,\"Education\":3,\"Fintech\":6,\"Real Estate and Construction\":1},\"Summer 2008\":{\"Consumer\":14,\"B2B\":8},\"Summer 2019\":{\"B2B\":69,\"Consumer\":27,\"Industrials\":12,\"Real Estate and Construction\":11,\"Government\":2,\"Education\":5,\"Healthcare\":25,\"Fintech\":25},\"Winter 2011\":{\"B2B\":22,\"Consumer\":14,\"Healthcare\":2,\"Real Estate and Construction\":1,\"Education\":2,\"Fintech\":3},\"Winter 2007\":{\"Consumer\":7,\"B2B\":3,\"Industrials\":2,\"Fintech\":1},\"Summer 2009\":{\"B2B\":9,\"Consumer\":12,\"Real Estate and Construction\":2,\"Education\":1,\"Fintech\":2},\"Summer 2006\":{\"Consumer\":9,\"B2B\":2},\"Winter 2013\":{\"B2B\":25,\"Industrials\":1,\"Consumer\":12,\"Healthcare\":6,\"Fintech\":2},\"Summer 2005\":{\"B2B\":5,\"Consumer\":3,\"Fintech\":1},\"Winter 2006\":{\"B2B\":4,\"Consumer\":3},\"Winter 2027\":{\"B2B\":1},\"Unspecified\":{\"Fintech\":1}}}" } ], "notes": [ "WARNING: 2 of 4 URLs failed and were skipped: https://api.exploreyc.com/api/companies?limit=10&offset=0, https://api.exploreyc.com/api/v1/companies?limit=10&offset=0", "Fetch result set: recb5c3b80cc (2 rows), reference it as FROM recb5c3b80cc in follow-up queries" ] }