# robots.txt User-agent: dotbot Disallow: / User-agent: twiceler Disallow: / User-agent: * Disallow: Crawl-delay: 120