# For all robots
User-agent: *
Disallow: /beta/

User-agent: Googlebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: Mediapartners-Google
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: AdsBot-Google
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: Bingbot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: MSNBot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: MSNBot-Media
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-Agent: Alexabot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Allow search crawlers to discover the sitemap
Sitemap: https://au.rollingstone.com/sitemap_index.xml

User-agent: SemrushBot-SA
Disallow:

User-agent: Twitterbot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php