# robots.txt # User-agent: * # Disallow: User-agent: rdfbot/1.0 Crawl-Delay: 1800