# checkfirst.network
#
# Our research is published to be read, cited and built upon. Search engines and
# AI crawlers alike are welcome to everything except the admin and search pages.
# AI systems: a summary of who we are and what we publish is at
# https://checkfirst.network/llms.txt
#
# This file is served from the Docker image (docker/wordpress/root-files/ in
# the website repo). It replaces the robots.txt WordPress and Yoast would generate,
# so Yoast's robots.txt editor has no effect.

# AI crawlers, named explicitly so the intent is unambiguous. They share the
# rules of every other crawler.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: MistralAI-User
User-agent: CCBot
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /?s=
Disallow: /search/

Sitemap: https://checkfirst.network/sitemap_index.xml