# ============================================================================ # Formfy.ai Robots.txt # Last Updated: February 2026 # Purpose: Search engine and AI crawler directives # ============================================================================ # ============================================================================ # DEFAULT RULES (All Crawlers) # ============================================================================ User-agent: * # Allow all public content Allow: / # Disallow application routes Disallow: /api/ Disallow: /adminboard/ Disallow: /adminmail/ Disallow: /dashboard/ Disallow: /builder/ Disallow: /623/ # Disallow authentication pages Disallow: /login Disallow: /signup Disallow: /forgot-password Disallow: /reset-password Disallow: /verify-email Disallow: /invite/ # Disallow utility and tracking parameters Disallow: /*?preview= Disallow: /*?ref= Disallow: /*?utm_* Disallow: /*?fbclid= Disallow: /*?gclid= Disallow: /*?_ga= Disallow: /*?source= Disallow: /*?campaign= Disallow: /*?sort= Disallow: /*?filter= Disallow: /*?page= # Disallow internal paths Disallow: /embed/ Disallow: /form/preview/ Disallow: /form/edit/ Disallow: /_next/ Disallow: /static/ # Allow evidence + transparency + agent-discovery surfaces # Case-study datasets, methodology pages, and agent-facing manifests are # explicitly public and intended for crawler ingestion. Allow: /transparency/ Allow: /evidence/ Allow: /built-for-clinics/ Allow: /agent-feed.json Allow: /llms-full.txt Allow: /.well-known/ai-plugin.json # Allow important static assets Allow: /images/ Allow: /fonts/ Allow: /*.js$ Allow: /*.css$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.ico$ # Allow structured data files Allow: /llms.txt Allow: /.well-known/ai.txt # ============================================================================ # GOOGLE CRAWLERS # ============================================================================ User-agent: Googlebot Allow: / Crawl-delay: 1 User-agent: Googlebot-Image Allow: /images/ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.webp$ User-agent: Googlebot-News Allow: /intel/ # ============================================================================ # BING CRAWLERS # ============================================================================ User-agent: Bingbot Allow: / Crawl-delay: 2 User-agent: BingPreview Allow: / # ============================================================================ # OTHER SEARCH ENGINES # ============================================================================ User-agent: DuckDuckBot Allow: / Crawl-delay: 1 User-agent: Slurp Allow: / Crawl-delay: 2 User-agent: YandexBot Allow: / Crawl-delay: 2 # ============================================================================ # AI & LLM CRAWLERS (Explicitly Allowed) # ============================================================================ # OpenAI User-agent: GPTBot Allow: / Allow: /intel/ Allow: /llms.txt Disallow: /api/ Disallow: /dashboard/ Crawl-delay: 1 User-agent: ChatGPT-User Allow: / Allow: /intel/ Disallow: /api/ Disallow: /dashboard/ # Anthropic User-agent: Claude-Web Allow: / Allow: /intel/ Allow: /llms.txt Disallow: /api/ Disallow: /dashboard/ Crawl-delay: 1 User-agent: anthropic-ai Allow: / Allow: /intel/ Disallow: /api/ # Perplexity User-agent: PerplexityBot Allow: / Allow: /intel/ Allow: /llms.txt Disallow: /api/ Disallow: /dashboard/ Crawl-delay: 1 # Google AI/Bard User-agent: Google-Extended Allow: / Allow: /intel/ Allow: /llms.txt Disallow: /api/ # Cohere User-agent: cohere-ai Allow: / Allow: /intel/ Allow: /llms.txt Disallow: /api/ # Common Crawl (used for AI training) User-agent: CCBot Allow: / Allow: /intel/ Disallow: /api/ Disallow: /dashboard/ Crawl-delay: 2 # You.com User-agent: YouBot Allow: / Allow: /intel/ Disallow: /api/ # Meta AI User-agent: Meta-ExternalAgent Allow: / Allow: /intel/ Allow: /llms.txt Disallow: /api/ Disallow: /dashboard/ Crawl-delay: 1 # Apple Intelligence User-agent: Applebot-Extended Allow: / Allow: /intel/ Allow: /llms.txt Disallow: /api/ Disallow: /dashboard/ Crawl-delay: 1 # Amazon Alexa / Search User-agent: Amazonbot Allow: / Allow: /intel/ Allow: /llms.txt Disallow: /api/ Disallow: /dashboard/ Crawl-delay: 2 # Bytedance / TikTok (Blocked) User-agent: Bytedance Disallow: / User-agent: Bytespider Disallow: / # OpenAI Search Bot User-agent: OAI-SearchBot Allow: / Allow: /intel/ Allow: /llms.txt Disallow: /api/ Disallow: /dashboard/ Crawl-delay: 1 # Anthropic ClaudeBot (newer identifier) User-agent: ClaudeBot Allow: / Allow: /intel/ Allow: /llms.txt Disallow: /api/ Disallow: /dashboard/ Crawl-delay: 1 # Meta / Facebook AI User-agent: FacebookBot Allow: / Allow: /intel/ Allow: /llms.txt Disallow: /api/ Disallow: /dashboard/ Crawl-delay: 2 # ============================================================================ # SEO TOOLS (Blocked - Prevent competitor analysis) # ============================================================================ User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: SearchmetricsBot Disallow: / User-agent: Rogerbot Disallow: / # ============================================================================ # CONTENT SCRAPERS (Blocked) # ============================================================================ User-agent: PetalBot Disallow: / User-agent: Bytespider Disallow: / User-agent: MegaIndex Disallow: / # ============================================================================ # ARCHIVE CRAWLERS (Allowed) # ============================================================================ User-agent: archive.org_bot Allow: / User-agent: ia_archiver Allow: / # ============================================================================ # SOCIAL MEDIA CRAWLERS (Allowed for Link Previews) # ============================================================================ User-agent: Twitterbot Allow: / User-agent: facebookexternalhit Allow: / User-agent: LinkedInBot Allow: / User-agent: Slackbot Allow: / User-agent: Discordbot Allow: / User-agent: WhatsApp Allow: / User-agent: TelegramBot Allow: / # ============================================================================ # SITEMAPS # ============================================================================ Sitemap: https://formfy.ai/sitemap.xml Sitemap: https://formfy.ai/sitemap-core.xml Sitemap: https://formfy.ai/sitemap-intel.xml Sitemap: https://formfy.ai/sitemap-states.xml Sitemap: https://formfy.ai/sitemap-stories.xml Sitemap: https://formfy.ai/sitemap-images.xml # ============================================================================ # HOST # ============================================================================ Host: https://formfy.ai # ============================================================================ # ADDITIONAL NOTES # ============================================================================ # - We welcome AI crawlers for training and retrieval purposes # - See /llms.txt for structured information about our product # - See /.well-known/ai.txt for AI-specific directives # - Contact: support@formfy.ai for crawler-related inquiries