{"schema_version":"newruntime-reader-dossiers-v1","type":"reader_dossier","id":"reader:googlebot","slug":"googlebot","name":"Googlebot","kind":"search","kind_label":"Search engines","operator":{"name":"Google","url":"https://www.google.com/"},"docs_url":"https://developers.google.com/search/docs/crawling-indexing/googlebot","robots_tokens":["Googlebot","Googlebot-Image","Googlebot-Video","Googlebot-News"],"purpose":"Generic name for Google Search's smartphone and desktop crawlers that fetch pages for the Google Search index (incl. Discover, Images, Video, News). Gemini training/grounding is controlled separately by the Google-Extended robots token.","feeds":["search_index"],"respects_robots":"documented_yes","robots_note":"robots.txt blocks crawling, not indexing; use noindex to keep pages out. Google-Extended token governs Gemini training/grounding without affecting Search.","verification":{"method":"both","url":"https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot"},"why_it_matters":{"text":"Googlebot coverage is the gate to Google Search and AI Overviews traffic; we currently have very few pages indexed, so its visit patterns are a direct indexing signal.","kind":"new_runtime_interpretation"},"confidence":"high","researched_at":"2026-10-23","sources":["https://developers.google.com/search/docs/crawling-indexing/googlebot","https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot","https://developers.google.com/static/search/apis/ipranges/googlebot.json"],"icon_url":"https://newruntime.com/readers/icons/google.png","matching":{"aliases":["googlebot"],"rule":"Reduce each agent_days label to a token: for labels starting with Mozilla/ take the product after 'compatible;', else a versioned crawler token such as 'FooBot/1.0' (a truncated 'Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile' prefix logged before 23 Oct 2026 is PetalBot; anything else is unidentified-android, unidentified-linux-chrome or unidentified-browser); otherwise take the text before the first '/', '(' or ';' and drop a trailing ' token probe'. Lowercase it. A dossier owns the tokens in its aliases."},"observed_baseline":{"as_of":"2026-10-23T08:01:01.990Z","window_first_day":"2026-07-23","window_last_day":"2026-10-23","observed_days":93,"note":"Refreshed daily in production from the live aggregates (dated by as_of). For current numbers aggregate live_sources.history agent_days by matching.aliases.","total_requests":1788,"first_seen":"2026-07-23T08:41:40.576Z","last_seen":"2026-10-23T07:36:43.946Z","days_active":73,"recency":"regular","categories":["search_crawler"],"label_variants":1,"journeys":{"requests":201,"content_reads":0,"feeds_or_discovery_reads":193,"machine_consumption_reads":0,"unique_materials":0,"unique_canonical_nodes":1,"verification":{"verified":{"requests":193,"method":"official_cidr"},"claimed_unverified":{"requests":8,"method":"official_cidr"}}}},"identity_note":"A family name is what a client claims in its user agent. It is verified only when its request came from the operator's published IP ranges or passed forward-confirmed reverse DNS, as reported per family in /crawler-journeys.json. Anyone can claim any name.","privacy":{"scope":"family-level aggregates only","ip_addresses":"not_output","raw_user_agents":"not_output (labels are sanitized; URLs inside them are redacted)","visitor_ids":"not_output","query_strings":"not_output"},"live_sources":{"history":"https://newruntime.com/bot-traffic-history.json","latest":"https://newruntime.com/bot-traffic-latest.json","machine_consumption":"https://newruntime.com/machine-consumption.json","crawler_journeys":"https://newruntime.com/crawler-journeys.json"},"routes":{"html":"https://newruntime.com/readers/googlebot/","markdown":"https://newruntime.com/readers/googlebot.md","json":"https://newruntime.com/readers/googlebot.json","index":"https://newruntime.com/readers.json","index_html":"https://newruntime.com/readers/"}}