User-agent: Scrapy
Allow: /

# global
 User-agent: *
 Disallow: /xmlrpc.php