{"schema_version":"newruntime-reader-dossiers-v1","type":"reader_dossier","id":"reader:semrushbot","slug":"semrushbot","name":"SemrushBot","kind":"seo","kind_label":"SEO & backlink crawlers","operator":{"name":"Semrush","url":"https://www.semrush.com/"},"docs_url":"https://www.semrush.com/bot/","robots_tokens":["SemrushBot","SiteAuditBot","SemrushBot-BA","SemrushBot-SI","SemrushBot-SWA","SplitSignalBot","SemrushBot-OCOB","SemrushBot-FT","RyteBot","SemrushBot-ESI"],"purpose":"Semrush's crawler that discovers links and revisits pages for its Backlink Analytics index and many tools (Site Audit, Backlink Audit, writing/plagiarism checks, etc.).","feeds":["seo_backlink_index"],"respects_robots":"documented_yes","robots_note":"Docs: per-tool tokens, Disallow and Crawl-delay (capped at 10 s) supported.","verification":{"method":"none_published","url":null},"why_it_matters":{"text":"SEO backlink index: our pages and outbound links show up in third-party link graphs used by marketers; no direct AI-reader value.","kind":"new_runtime_interpretation"},"confidence":"high","researched_at":"2026-10-23","sources":["https://www.semrush.com/bot/"],"icon_url":"https://newruntime.com/readers/icons/semrush.png","matching":{"aliases":["semrushbot"],"rule":"Reduce each agent_days label to a token: for labels starting with Mozilla/ take the product after 'compatible;', else a versioned crawler token such as 'FooBot/1.0' (a truncated 'Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile' prefix logged before 23 Oct 2026 is PetalBot; anything else is unidentified-android, unidentified-linux-chrome or unidentified-browser); otherwise take the text before the first '/', '(' or ';' and drop a trailing ' token probe'. Lowercase it. A dossier owns the tokens in its aliases."},"observed_baseline":{"as_of":"2026-10-23T08:01:01.990Z","window_first_day":"2026-07-23","window_last_day":"2026-10-23","observed_days":93,"note":"Refreshed daily in production from the live aggregates (dated by as_of). For current numbers aggregate live_sources.history agent_days by matching.aliases.","total_requests":9333,"first_seen":"2026-07-26T23:12:10.570Z","last_seen":"2026-10-23T07:41:57.030Z","days_active":84,"recency":"regular","categories":["other_bot"],"label_variants":1,"journeys":{"requests":1755,"content_reads":1580,"feeds_or_discovery_reads":60,"machine_consumption_reads":0,"unique_materials":904,"unique_canonical_nodes":634,"verification":{"not_verifiable":{"requests":1755,"method":"provider_not_published"}}}},"identity_note":"A family name is what a client claims in its user agent. It is verified only when its request came from the operator's published IP ranges or passed forward-confirmed reverse DNS, as reported per family in /crawler-journeys.json. Anyone can claim any name.","privacy":{"scope":"family-level aggregates only","ip_addresses":"not_output","raw_user_agents":"not_output (labels are sanitized; URLs inside them are redacted)","visitor_ids":"not_output","query_strings":"not_output"},"live_sources":{"history":"https://newruntime.com/bot-traffic-history.json","latest":"https://newruntime.com/bot-traffic-latest.json","machine_consumption":"https://newruntime.com/machine-consumption.json","crawler_journeys":"https://newruntime.com/crawler-journeys.json"},"routes":{"html":"https://newruntime.com/readers/semrushbot/","markdown":"https://newruntime.com/readers/semrushbot.md","json":"https://newruntime.com/readers/semrushbot.json","index":"https://newruntime.com/readers.json","index_html":"https://newruntime.com/readers/"}}