10:["$","div",null,{"className":"Post-module__aBOVGG__post","style":{"display":"flex","flexDirection":"column","alignItems":"center","gap":"8px","flex":"1","width":"100%","padding":"12px"},"children":[["$","div",null,{"className":"$undefined","style":{"display":"flex","gap":"0.4rem","flexWrap":"wrap","width":"100%"},"children":[false,[["$","a","B2C3",{"href":"/B2C3","className":"Post-module__aBOVGG__fg","style":{},"children":"『t_wの輪郭』で実施した作業"}],["$","a","E441",{"href":"/E441","className":"Post-module__aBOVGG__fg","style":{},"children":"robot.txt"}],["$","a","9231",{"href":"/9231","className":"Post-module__aBOVGG__fg","style":{},"children":"User-Agentを一般のコンピュータに偽装してクロールする輩"}]]]}],["$","article",null,{"className":"article","children":[["$","div",null,{"className":"$undefined","style":{"display":"flex","justifyContent":"space-between","flexWrap":"wrap","width":"100%"},"children":[["$","h2",null,{"style":{"margin":"0.2rem"},"children":["$","a",null,{"href":"/8E3E","children":"bot.txtで有力な検索エンジン以外を除外"}]}],["$","div",null,{"className":"$undefined","style":{"display":"flex","justifyContent":"end","alignItems":"center","flex":"1","textAlign":"right","whiteSpace":"nowrap"},"children":["$","div",null,{"children":"2022/10/25 10:28:00"}]}]]}],["$","div",null,{"className":"article_body","dangerouslySetInnerHTML":{"__html":"

　User-Agentを一般のコンピュータに偽装してクロールする輩が出てきたので、robot.txtで有力な検索エンジン以外を除外。\n

　「これも入れるべきだ」みたいなのがあればコメント求む。\n

robot.txtの中身

User-agent: *\nDisallow: /\n\nUser-agent: Googlebot\nUser-agent: bingbot\nUser-agent: DuckDuckBot\nAllow: /\n

"}}]]}],["$","div",null,{"className":"$undefined","style":{"display":"flex","gap":"0.4rem","flexWrap":"wrap","width":"100%"},"children":[["$","a","",{"href":"/","className":"Post-module__aBOVGG__bg","style":{"opacity":0},"children":"あれ"}],[]]}]]}]