# If you are regularly crawling WordPress.com sites, please use our firehose to receive real-time push updates instead.
# Please see https://developer.wordpress.com/docs/firehose/ for more details.

Sitemap: http://time.com/news-sitemap.xml

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
# exclude urls of format time.com/page/7456/?search
User-agent: Googlebot
Disallow: /page/*/?search=
Disallow: /page/*/?a_aid=
Disallow: /page/*/?iq_id=
Disallow: /page/*/?hc_location=
Disallow: /*?ref=binfind.com
Disallow: /*?__rmid___get___page
Disallow: /page/*/?pcd=
Disallow: /feed
Disallow: ?search
Disallow: /uncategorized
Disallow: ?sa=
Disallow: ?EGOTECpage
Disallow: ?p=
Disallow: ?ref=
Disallow: ?pagespeed
Disallow: ?hc_location

# Sitemap archive
Sitemap: http://time.com/sitemap.xml

# Video Sitemap archive
Video Sitemap: http://time.com/video-sitemap.xml

Disallow: /wp-login.php
Disallow: /activate/ # har har
Disallow: /cgi-bin/ # MT refugees
Disallow: /mshots/v1/
Disallow: /next/
Disallow: /public.api/

User-agent: IRLbot
Crawl-delay: 3600

# This file was generated on Sun, 11 Mar 2018 03:23:56 +0000