User-agent: Scrapy
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: *
# disallow all files in these directories
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /contact/
Allow: /search/

User-agent: AhrefsSiteAudit
Allow: /


Sitemap: https://ukesong.com/sitemap_index.xml

User-agent: Googlebot
# disallow all files ending with these extensions
Disallow: /*.pdf$
Disallow: /*.swf$

# allow google image bot to search all images
User-agent: Googlebot-Image
User-agent: AhrefsBot-Image

Allow: /*

# allow adsense bot on entire site
User-agent: Mediapartners-Google*
Disallow:
Allow: /*