User-Agent: * Allow: / Disallow: /directory-export/ Disallow: /dashboard/ Disallow: /admin/ Disallow: /api/ Disallow: /dev/ Disallow: /verify/ Disallow: /review/ Disallow: /claim/sent/ Disallow: /contact-sent/ Disallow: /widget/ Disallow: /*?* User-Agent: OAI-SearchBot Allow: / Disallow: /directory-export/ Disallow: /dashboard/ Disallow: /admin/ Disallow: /api/ Disallow: /dev/ Disallow: /verify/ Disallow: /review/ Disallow: /claim/sent/ Disallow: /contact-sent/ Disallow: /widget/ Disallow: /*?* User-Agent: ChatGPT-User Allow: / Disallow: /directory-export/ Disallow: /dashboard/ Disallow: /admin/ Disallow: /api/ Disallow: /dev/ Disallow: /verify/ Disallow: /review/ Disallow: /claim/sent/ Disallow: /contact-sent/ Disallow: /widget/ Disallow: /*?* User-Agent: GPTBot Allow: / Disallow: /directory-export/ Disallow: /dashboard/ Disallow: /admin/ Disallow: /api/ Disallow: /dev/ Disallow: /verify/ Disallow: /review/ Disallow: /claim/sent/ Disallow: /contact-sent/ Disallow: /widget/ Disallow: /*?* User-Agent: PerplexityBot Allow: / Disallow: /directory-export/ Disallow: /dashboard/ Disallow: /admin/ Disallow: /api/ Disallow: /dev/ Disallow: /verify/ Disallow: /review/ Disallow: /claim/sent/ Disallow: /contact-sent/ Disallow: /widget/ Disallow: /*?* User-Agent: Perplexity-User Allow: / Disallow: /directory-export/ Disallow: /dashboard/ Disallow: /admin/ Disallow: /api/ Disallow: /dev/ Disallow: /verify/ Disallow: /review/ Disallow: /claim/sent/ Disallow: /contact-sent/ Disallow: /widget/ Disallow: /*?* User-Agent: ClaudeBot Allow: / Disallow: /directory-export/ Disallow: /dashboard/ Disallow: /admin/ Disallow: /api/ Disallow: /dev/ Disallow: /verify/ Disallow: /review/ Disallow: /claim/sent/ Disallow: /contact-sent/ Disallow: /widget/ Disallow: /*?* User-Agent: Claude-User Allow: / Disallow: /directory-export/ Disallow: /dashboard/ Disallow: /admin/ Disallow: /api/ Disallow: /dev/ Disallow: /verify/ Disallow: /review/ Disallow: /claim/sent/ Disallow: /contact-sent/ Disallow: /widget/ Disallow: /*?* User-Agent: Claude-SearchBot Allow: / Disallow: /directory-export/ Disallow: /dashboard/ Disallow: /admin/ Disallow: /api/ Disallow: /dev/ Disallow: /verify/ Disallow: /review/ Disallow: /claim/sent/ Disallow: /contact-sent/ Disallow: /widget/ Disallow: /*?* User-Agent: Google-Extended Allow: / Disallow: /directory-export/ Disallow: /dashboard/ Disallow: /admin/ Disallow: /api/ Disallow: /dev/ Disallow: /verify/ Disallow: /review/ Disallow: /claim/sent/ Disallow: /contact-sent/ Disallow: /widget/ Disallow: /*?* User-Agent: Applebot-Extended Allow: / Disallow: /directory-export/ Disallow: /dashboard/ Disallow: /admin/ Disallow: /api/ Disallow: /dev/ Disallow: /verify/ Disallow: /review/ Disallow: /claim/sent/ Disallow: /contact-sent/ Disallow: /widget/ Disallow: /*?* User-Agent: ShapBot Disallow: / User-Agent: CCBot Disallow: / User-Agent: Brightbot Disallow: / User-Agent: FirecrawlAgent Disallow: / User-Agent: Bytespider Disallow: / User-Agent: Diffbot Disallow: / User-Agent: omgili Disallow: / User-Agent: omgilibot Disallow: / User-Agent: Webzio-Extended Disallow: / User-Agent: ExaBot Disallow: / User-Agent: TavilyBot Disallow: / User-Agent: JinaBot Disallow: / User-Agent: Crawl4AI Disallow: / User-Agent: DataForSeoBot Disallow: / User-Agent: AhrefsBot Disallow: / User-Agent: SemrushBot Disallow: / User-Agent: MJ12bot Disallow: / User-Agent: DotBot Disallow: / User-Agent: BLEXBot Disallow: / User-Agent: PetalBot Disallow: / User-Agent: YouBot Disallow: / User-Agent: Timpibot Disallow: / User-Agent: AI2Bot Disallow: / User-Agent: cohere-ai Disallow: / User-Agent: cohere-training-data-crawler Disallow: / User-Agent: ImagesiftBot Disallow: / User-Agent: Kangaroo Bot Disallow: / User-Agent: PanguBot Disallow: / User-Agent: LinkupBot Disallow: / User-Agent: img2dataset Disallow: / User-Agent: ia_archiver Disallow: / User-Agent: archive.org_bot Disallow: / User-Agent: Scrapy Disallow: / Sitemap: https://agentinc.com/sitemap.xml