[{"data":1,"prerenderedAt":190},["ShallowReactive",2],{"copy:zh:site:":3,"copy:zh:agentweb:bots,ui":123,"copy:zh:site:common":188},{"common":4,"nav":16,"footer":44,"home":51,"error":116},{"skip":5,"menu":6,"language":7,"sections":8,"footer":9,"tool":10,"home":11,"breadcrumbs":12,"copy":13,"copied":14,"download":15},"跳转到正文","菜单","语言","章节","页脚","工具","首页","面包屑导航","复制","已复制","下载",{"groups":17,"items":25},{"build":18,"connect":19,"instruct":20,"run":21,"web":22,"secure":23,"track":24},"构建","连接","编写指令","运行","开放你的网站","安全","跟踪",{"models":26,"cost":27,"frameworks":28,"mcp":29,"mcpConfig":30,"mcpClients":31,"agentsMd":32,"files":33,"openclaw":34,"agentReady":35,"llmsTxt":36,"robotsTxt":37,"apiCatalog":38,"bots":39,"security":40,"incidents":41,"checklist":42,"changelog":43},"模型与 API 价格","Agent 成本计算器","框架与 SDK","MCP 服务器","MCP 配置生成器","MCP 客户端","AGENTS.md 生成器","指令文件","自托管 OpenClaw","适配 Agent 的网站","llms.txt 生成器","面向 AI 机器人的 robots.txt","API 目录生成器","AI 机器人目录","Agent 安全","事故记录","安全检查清单","更新日志",{"about":45,"privacy":46,"terms":47,"sitemap":48,"disclaimer":49,"checked":50},"关于","隐私","条款","网站地图","为 AI agent 开发者提供的独立参考资料。与此处提及的任何厂商均无关联。","事实核查日期：{date}",{"seo":52,"eyebrow":55,"title":56,"lead":57,"ctaMcp":58,"ctaCost":59,"ctaAgents":60,"sheetLabel":61,"sheetTitle":62,"sheet":63,"sheetNote":69,"sectionsTitle":70,"sections":71,"latestTitle":99,"allChanges":100,"rulesTitle":101,"rules":102,"aboutLink":115},{"title":53,"description":54},"DotsAgent：构建 AI agent 的参考资料与工具","LLM API 价格、agent 成本计算器、适用于 13 个客户端的 MCP 配置、AGENTS.md 和 llms.txt 生成器，以及 agent 安全指南。免费提供，支持 18 种语言。","面向 AI agent 开发者","Agent 开发者的案头参考","涵盖 agent 开发所需的价格、协议、配置文件和安全指南，所有信息均经厂商官方文档核实。查找信息、生成所需文件，然后继续编写代码。","生成 MCP 配置","计算 agent 任务成本","编写 AGENTS.md","最新数据","技术栈概览",{"mcp":64,"openclaw":65,"cheapest":66,"models":67,"servers":29,"bots":68},"MCP 规范","OpenClaw","最低价模型，$\u002F百万输入\u002F输出 tokens","已统计价格的模型","已跟踪的 AI 机器人","每项数据均经人工对照厂商文档核实。","网站内容",{"models":72,"mcp":76,"instructions":79,"frameworks":82,"openclaw":85,"agentReady":88,"security":92,"changelog":95},{"title":73,"body":74,"unit":75},"模型与价格","并列比较每百万输入、输出和缓存 token 的价格，并提供计算器，估算完整 agent 运行的费用。","个模型",{"title":29,"body":77,"unit":78},"Model Context Protocol 服务器目录，以及配置生成器，可为每种客户端生成对应的配置文件。","台服务器",{"title":33,"body":80,"unit":81},"编写 AGENTS.md，并了解各工具如何读取 CLAUDE.md、.cursor\u002Frules 等类似文件。","种文件格式",{"title":28,"body":83,"unit":84},"按语言、许可证、MCP 支持和多 agent 功能对比 agent 框架与厂商 SDK。","个框架",{"title":34,"body":86,"unit":87},"逐步指导你在自己的设备或服务器上安装、配置和更新 OpenClaw。","当前版本",{"title":89,"body":90,"unit":91},"适配 agent 的网站","生成 llms.txt、面向 AI crawler 的 robots.txt 规则和 api-catalog，并通过 user agent 查询任意机器人。","个已收录机器人",{"title":40,"body":93,"unit":94},"了解 agent 面临的 OWASP 风险、真实事件记录，以及上线前可逐项核对的清单。","起已记录事件",{"title":96,"body":97,"unit":98},"API 更新日志","按日期记录模型 API、SDK 和协议的变更，并附上厂商公告链接。","条记录","最新变更","所有变更","我们如何确保信息准确",[103,106,109,112],{"title":104,"body":105},"事实均有来源。","每项价格、版本和 flag 都链接到对应的供应商页面，方便你自行核实。",{"title":107,"body":108},"定期核查。","每个页面都会显示上次核查信息的时间；过时的条目会重新核查或移除。",{"title":110,"body":111},"不接受付费推广。","没有人能付费获得收录、提高排名或得到更好的描述。",{"title":113,"body":114},"工具在浏览器中运行。","生成器和计算器都在本地运行，你输入的内容不会发送到服务器。","关于 DotsAgent",{"title":117,"body":118,"home":119,"popular":120,"failed":121,"failedBody":122},"找不到页面","此地址没有对应的页面。页面可能已迁移，或者链接中有拼写错误。","前往首页","热门工具","出了点问题","我们这边未能加载此页面。请稍后再试，或从首页开始浏览。",{"bots":124,"ui":180},{"seo":125,"crumb":39,"eyebrow":128,"title":129,"lead":130,"purposeFilter":131,"all":132,"search":133,"operator":134,"anyOperator":135,"cols":136,"noToken":142,"marks":143,"signed":147,"ips":148,"empty":149,"purposesTitle":150,"purposeHelp":151,"verifyTitle":157,"verify":158,"toGenerator":162,"faqTitle":163,"faq":164},{"title":126,"description":127},"AI bot 目录：40 种爬虫、令牌和 IP 列表","来自 OpenAI、Anthropic、Google、Apple、Meta、Amazon 等公司的 40 个 AI 爬虫和代理，列出 robots.txt 令牌、用途、合规情况和 IP 列表。","{n} 个爬虫和代理","AI 爬虫和代理目录","这里列出了我们能够确认的所有 AI 机器人，包括 robots.txt 中使用的令牌、抓取目的、运营方是否声明遵守相关规则，以及如何验证请求确实来自该机器人。令牌仅见于社区维护的 ai.robots.txt 登记表时，会在对应页面特别标注。","按用途筛选","全部","按名称、令牌或运营方搜索","运营方","任意运营方",{"bot":137,"token":138,"purpose":139,"robots":140,"verify":141},"机器人","令牌","用途","遵守 robots.txt","验证","无令牌",{"yes":144,"no":145,"unknown":146},"是","否","未说明","签名请求","IP 列表","没有符合这些筛选条件的机器人。","用途说明",{"training":152,"search":153,"user":154,"agent":155,"ads":156},"收集网页，用于模型训练或一般研究爬取。Google-Extended 和 Applebot-Extended 是控制令牌，本身不会抓取任何内容。","为搜索产品编入网页索引，涵盖 Google Search 以及 ChatGPT search、Perplexity 和 DuckAssist 等 AI 问答引擎。屏蔽此类机器人，就无法出现在对应产品中。","根据用户在聊天或工具中的请求抓取网页。多家运营方表示，robots.txt 可能不适用于这类请求。","代表用户浏览网页并执行操作，通常使用完整浏览器。许多代理没有令牌，因此只能通过签名识别（如果能够识别的话）。","检查广告主提交的广告落地页。这些机器人不会包含在 robots.txt 生成器中。","验证爬虫身份",[159,160,161],"伪造 User-Agent 字符串毫不费力。大多数大型运营方会以 JSON 文件公布其爬虫使用的 IP 地址段，表格中标记为「IP 列表」。将请求的 IP 地址与该 token 对应的列表进行比对；仅凭名称匹配不能视为已验证。","部分运营方还会说明反向 DNS 信息。例如，Common Crawl 表示 CCBot 主机名会解析到 *.crawl.commoncrawl.org；在信任该主机前，请确认该主机名解析回同一个 IP 地址。","越来越多的智能体依据 Web Bot Auth 草案，使用 HTTP Message Signatures (RFC 9421) 对请求进行签名。请求通过 Signature-Agent 声明其运营方，你可以使用该域名下 \u002F.well-known\u002Fhttp-message-signatures-directory 中的公钥验证签名。有效签名只能证明运营方身份，无法证明背后的用户是谁，也无法证明该用户有权执行哪些操作。","根据此列表生成 robots.txt","关于 AI 爬虫的常见问题",[165,168,171,174,177],{"q":166,"a":167},"哪些 AI 爬虫会忽略 robots.txt？","根据运营方自己的文档，ChatGPT-User、Perplexity-User、Google-Agent、Google-GeminiNotebook、Amzn-User、meta-externalfetcher 和 facebookexternalhit 可能不会遵守 robots.txt，主要是因为抓取由用户触发。社区报告称 Bytespider 也不遵守该规则。",{"q":169,"a":170},"GPTBot、OAI-SearchBot 和 ChatGPT-User 有什么区别？","GPTBot 会收集网页，用于训练 OpenAI 模型；OAI-SearchBot 会为 ChatGPT 搜索建立网页索引；ChatGPT-User 则会在用户提出请求时抓取网页。前两者遵守 robots.txt；OpenAI 表示，其规则可能不适用于第三者。",{"q":172,"a":173},"哪些 AI 智能体没有 robots.txt token？","ChatGPT agent、Gemini Spark 和 Meta 的 Muse。可以通过来自 https:\u002F\u002Fchatgpt.com 的 Web Bot Auth 签名验证 ChatGPT agent。Gemini Spark 在本地 Chrome 模式下使用的标识与用户自己的浏览器相同；Muse 则没有公开的 User-Agent。",{"q":175,"a":176},"如何验证请求确实来自 Googlebot？","Google 会以 JSON 文件公布 Googlebot 的 IP 地址段，链接可在此目录对应的条目中找到。请将请求的 IP 地址与这些地址段进行比对，不要只依赖 User-Agent 字符串。",{"q":178,"a":179},"这份 AI 爬虫列表来自哪里？","每个条目都链接到运营方的相关文档（如果有）；缺少相关文档的 token 则来自社区维护的 ai.robots.txt 注册表。由于找不到运营方文档，我们未收录 DeepSeekBot、Kimi-User、Manus-User、NovaAct、Devin 和 PetalBot。此列表于 2026年10月1日核查。",{"crumb":35,"purposes":181,"sources":187},{"training":182,"search":183,"user":184,"agent":185,"ads":186},"训练","搜索","用户请求","Agent","广告","来源",{"common":189},{"skip":5,"menu":6,"language":7,"sections":8,"footer":9,"tool":10,"home":11,"breadcrumbs":12,"copy":13,"copied":14,"download":15},1790863381520]