[{"data":1,"prerenderedAt":198},["ShallowReactive",2],{"copy:zh:site:":3,"copy:zh:agentweb:robots,ui":123,"copy:zh:site:common":196},{"common":4,"nav":16,"footer":44,"home":51,"error":116},{"skip":5,"menu":6,"language":7,"sections":8,"footer":9,"tool":10,"home":11,"breadcrumbs":12,"copy":13,"copied":14,"download":15},"跳转到正文","菜单","语言","章节","页脚","工具","首页","面包屑导航","复制","已复制","下载",{"groups":17,"items":25},{"build":18,"connect":19,"instruct":20,"run":21,"web":22,"secure":23,"track":24},"构建","连接","编写指令","运行","开放你的网站","安全","跟踪",{"models":26,"cost":27,"frameworks":28,"mcp":29,"mcpConfig":30,"mcpClients":31,"agentsMd":32,"files":33,"openclaw":34,"agentReady":35,"llmsTxt":36,"robotsTxt":37,"apiCatalog":38,"bots":39,"security":40,"incidents":41,"checklist":42,"changelog":43},"模型与 API 价格","Agent 成本计算器","框架与 SDK","MCP 服务器","MCP 配置生成器","MCP 客户端","AGENTS.md 生成器","指令文件","自托管 OpenClaw","适配 Agent 的网站","llms.txt 生成器","面向 AI 机器人的 robots.txt","API 目录生成器","AI 机器人目录","Agent 安全","事故记录","安全检查清单","更新日志",{"about":45,"privacy":46,"terms":47,"sitemap":48,"disclaimer":49,"checked":50},"关于","隐私","条款","网站地图","为 AI agent 开发者提供的独立参考资料。与此处提及的任何厂商均无关联。","事实核查日期：{date}",{"seo":52,"eyebrow":55,"title":56,"lead":57,"ctaMcp":58,"ctaCost":59,"ctaAgents":60,"sheetLabel":61,"sheetTitle":62,"sheet":63,"sheetNote":69,"sectionsTitle":70,"sections":71,"latestTitle":99,"allChanges":100,"rulesTitle":101,"rules":102,"aboutLink":115},{"title":53,"description":54},"DotsAgent：构建 AI agent 的参考资料与工具","LLM API 价格、agent 成本计算器、适用于 13 个客户端的 MCP 配置、AGENTS.md 和 llms.txt 生成器，以及 agent 安全指南。免费提供，支持 18 种语言。","面向 AI agent 开发者","Agent 开发者的案头参考","涵盖 agent 开发所需的价格、协议、配置文件和安全指南，所有信息均经厂商官方文档核实。查找信息、生成所需文件，然后继续编写代码。","生成 MCP 配置","计算 agent 任务成本","编写 AGENTS.md","最新数据","技术栈概览",{"mcp":64,"openclaw":65,"cheapest":66,"models":67,"servers":29,"bots":68},"MCP 规范","OpenClaw","最低价模型，$\u002F百万输入\u002F输出 tokens","已统计价格的模型","已跟踪的 AI 机器人","每项数据均经人工对照厂商文档核实。","网站内容",{"models":72,"mcp":76,"instructions":79,"frameworks":82,"openclaw":85,"agentReady":88,"security":92,"changelog":95},{"title":73,"body":74,"unit":75},"模型与价格","并列比较每百万输入、输出和缓存 token 的价格，并提供计算器，估算完整 agent 运行的费用。","个模型",{"title":29,"body":77,"unit":78},"Model Context Protocol 服务器目录，以及配置生成器，可为每种客户端生成对应的配置文件。","台服务器",{"title":33,"body":80,"unit":81},"编写 AGENTS.md，并了解各工具如何读取 CLAUDE.md、.cursor\u002Frules 等类似文件。","种文件格式",{"title":28,"body":83,"unit":84},"按语言、许可证、MCP 支持和多 agent 功能对比 agent 框架与厂商 SDK。","个框架",{"title":34,"body":86,"unit":87},"逐步指导你在自己的设备或服务器上安装、配置和更新 OpenClaw。","当前版本",{"title":89,"body":90,"unit":91},"适配 agent 的网站","生成 llms.txt、面向 AI crawler 的 robots.txt 规则和 api-catalog，并通过 user agent 查询任意机器人。","个已收录机器人",{"title":40,"body":93,"unit":94},"了解 agent 面临的 OWASP 风险、真实事件记录，以及上线前可逐项核对的清单。","起已记录事件",{"title":96,"body":97,"unit":98},"API 更新日志","按日期记录模型 API、SDK 和协议的变更，并附上厂商公告链接。","条记录","最新变更","所有变更","我们如何确保信息准确",[103,106,109,112],{"title":104,"body":105},"事实均有来源。","每项价格、版本和 flag 都链接到对应的供应商页面，方便你自行核实。",{"title":107,"body":108},"定期核查。","每个页面都会显示上次核查信息的时间；过时的条目会重新核查或移除。",{"title":110,"body":111},"不接受付费推广。","没有人能付费获得收录、提高排名或得到更好的描述。",{"title":113,"body":114},"工具在浏览器中运行。","生成器和计算器都在本地运行，你输入的内容不会发送到服务器。","关于 DotsAgent",{"title":117,"body":118,"home":119,"popular":120,"failed":121,"failedBody":122},"找不到页面","此地址没有对应的页面。页面可能已迁移，或者链接中有拼写错误。","前往首页","热门工具","出了点问题","我们这边未能加载此页面。请稍后再试，或从首页开始浏览。",{"robots":124,"ui":188},{"seo":125,"crumb":128,"eyebrow":129,"title":130,"lead":131,"form":132,"presets":141,"purposeNotes":145,"comments":150,"signalNote":154,"limits":155,"botsLink":156,"explainTitle":157,"explain":158,"faqTitle":171,"faq":172},{"title":126,"description":127},"AI 爬虫和智能体 robots.txt 生成器","生成拒绝 AI 训练、但允许搜索的 robots.txt：按用途分组的 35 个 bot token、Content-Signal 行、私有路径和 sitemap 行。","适用于 AI bot 的 robots.txt","开放你的网站 · 生成器","AI bot robots.txt 生成器","为每类 AI bot 选择策略，单独覆盖特定爬虫，然后复制符合 RFC 9309 的文件。训练、搜索和用户触发的抓取使用不同的 token，因此你可以拒绝其中一类，同时允许另一类。",{"preset":133,"allow":134,"block":135,"tapHint":136,"unset":137,"disallow":138,"disallowHint":139,"sitemap":140},"从以下预设开始","允许","阻止","点击 bot，即可切换它相对于所属组设置的状态。","未设置","私有路径","每行填写一个路径。已允许的 bot 和所有其他爬虫都无法访问这些路径。","Sitemap URL",{"open":142,"noTraining":143,"closed":144},"全部允许","仅阻止训练","阻止所有 AI bot",{"training":146,"search":147,"user":148,"agent":149},"阻止这些 bot 可将你的页面排除在这些运营方的模型训练之外，同时不影响搜索结果中的收录。","屏蔽这些爬虫也会屏蔽 Googlebot 和 Applebot，导致你的网站无法出现在 Google 搜索以及 Apple Siri、Spotlight 和 Safari 的结果中。","这些爬虫会在有人请求某个页面时抓取该页面，且有多个运营方表示 robots.txt 规则可能不适用于它们。因此，这里的屏蔽只是请求，并非保证。","Google-Agent 通常会忽略 robots.txt，而且大多数浏览代理根本没有对应的 token，因此这个分组在实际中影响不大。",{"blocked":151,"allowed":152,"everyone":153},"已屏蔽访问整个网站的 AI 爬虫","允许的 AI 爬虫，排除私有路径","所有其他爬虫","Content-Signal 用于说明抓取的页面可以如何使用：search 表示用于搜索结果，ai-input 表示用于生成 AI 答案，ai-train 表示用于模型训练。它源自 Cloudflare 的 CC0 政策，表达的是一种使用偏好，不会屏蔽任何内容。","robots.txt 是自愿遵守的规则。ChatGPT-User、Perplexity-User 及其他由用户触发的抓取工具表示，这些规则可能不适用于它们；有些代理甚至没有对应的 token。若要强制屏蔽，请根据公开的 IP 列表或特征匹配请求，并在防火墙或 WAF 上拒绝这些请求。","查看目录中的所有 bot","生成的文件如何生效",[159,162,165,168],{"title":160,"body":161},"爬虫遵循自己的分组","如果爬虫在 User-agent 行中找到自己的 token，就会遵循该分组，并跳过 User-agent: * 下的规则。因此，生成器会为你按名称允许的 bot 重复列出私有路径，并为已屏蔽的 bot 设置单条 Disallow: \u002F 规则。",{"title":163,"body":164},"有些 token 不会执行抓取","Google-Extended 和 Applebot-Extended 没有自己的 user agent。实际抓取由 Googlebot 和 Applebot 执行；扩展 token 则告知 Google 和 Apple 是否可以使用这些内容训练模型。Google 也会将 Google-Extended 用于 Gemini grounding。",{"title":166,"body":167},"多数运营方会区分训练和搜索","OpenAI、Anthropic、Amazon 和 Mistral 都为训练和搜索分别设置了独立的 token，因此选择“禁止训练”预设后，你的网站仍会保留在它们的搜索产品中。Meta 是个例外：meta-externalagent 同时负责训练和索引。",{"title":169,"body":170},"Applebot 会回退到 Googlebot 规则","如果文件中没有 Applebot 分组，Apple 表示 Applebot 会改为遵循 Googlebot 规则。Applebot 和 Amazonbot 都会忽略 Crawl-delay，而 Anthropic 表示 ClaudeBot 会遵守该规则。","关于 robots.txt 和 AI bot 的常见问题",[173,176,179,182,185],{"q":174,"a":175},"如何屏蔽 GPTBot，同时保留 ChatGPT 搜索？","禁止 GPTBot，并允许 OAI-SearchBot。GPTBot 收集训练数据，OAI-SearchBot 为 ChatGPT 搜索索引页面，ChatGPT-User 则处理用户请求抓取的页面。OpenAI 在同一页面说明了这三者，并分别提供各自的 IP 列表。",{"q":177,"a":178},"屏蔽 Google-Extended 会让我的网站从 Google 搜索中消失吗？","Google-Extended 是一个没有自己 user agent 的控制 token；抓取工作由 Googlebot 执行。禁止 Google-Extended 会告知 Google 不要将你的内容用于 Gemini 训练和 grounding；哪些内容会被抓取并用于搜索，则由 Googlebot 规则决定。",{"q":180,"a":181},"AI 爬虫会遵守 Crawl-delay 吗？","有些会。Anthropic 表示 ClaudeBot 会遵守该规则，而 Apple 和 Amazon 表示 Applebot 和 Amazonbot 不会。更可靠的做法是在服务器上进行速率限制。",{"q":183,"a":184},"我屏蔽了 ChatGPT-User，它为什么还会访问？","OpenAI 表示，由于页面是用户请求的，robots.txt 规则可能不适用于 ChatGPT-User。Perplexity-User、Amzn-User 和 meta-externalfetcher 也有类似说明。如果运营方公布了 IP 列表，可以根据该列表匹配请求，并在防火墙上屏蔽。",{"q":186,"a":187},"robots.txt 中的 Content-Signal 有什么作用？","它通过三个信号说明内容抓取后可以如何使用：search、ai-input 和 ai-train，每项的值为 yes 或 no。它表达的是偏好，不会阻止任何操作。Cloudflare 的 Markdown for Agents 默认会添加 ai-train=yes、search=yes、ai-input=yes；如果启用了此功能，请检查其输出。",{"crumb":35,"purposes":189,"sources":195},{"training":190,"search":191,"user":192,"agent":193,"ads":194},"训练","搜索","用户请求","Agent","广告","来源",{"common":197},{"skip":5,"menu":6,"language":7,"sections":8,"footer":9,"tool":10,"home":11,"breadcrumbs":12,"copy":13,"copied":14,"download":15},1790863381500]