Стандартный robots.txt для WordPress

Стандартный файл robots.txt для WP должен содержать следующие директивы:

User-agent: *

Disallow: /cgi-bin

Disallow: /wp-admin

Disallow: /wp-includes

Disallow: /wp-content/plugins

Disallow: /wp-content/cache

Disallow: /wp-content/themes

Disallow: */trackback

Disallow: */feed

Disallow: /*?

Disallow: /author/

Disallow: /transfers.js

Disallow: /go.php

Disallow: /xmlrpc.php

User-agent: Yandex

Disallow: /cgi-bin

Disallow: /wp-admin

Disallow: /wp-includes

Disallow: /wp-content/plugins

Disallow: /wp-content/cache

Disallow: /wp-content/themes

Disallow: */trackback

Disallow: */feed

Disallow: /*?

Disallow: /author/

Disallow: /transfers.js

Disallow: /go.php

Disallow: /xmlrpc.php

Host: site.com

User-agent: Googlebot

Disallow: /cgi-bin/

Disallow: /wp-admin/

Disallow: /wp-includes/

Disallow: /wp-content/cache

Disallow: */trackback

Disallow: */feed

Disallow: /author/

Disallow: /transfers.js

Disallow: /go.php

Disallow: /xmlrpc.php

Disallow: /*?

Allow: *.css

Allow: *.js

Allow: /wp-includes/*.js

Sitemap: http://site.com/sitemap.xml