Text File

1#

2# robots.txt for https://www.w3.org/

3#

4# $Id: robots.txt,v 1.89 2024/03/13 18:11:44 gerald Exp $

5#

6

7# For use by search.w3.org

8User-agent: W3C-gsa

9Disallow: /Out-Of-Date

10

11User-agent: W3T_SE

12Disallow: /Out-Of-Date

13

14User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)

15Disallow: /

16

17# W3C Link checker

18User-agent: W3C-checklink

19Disallow:

20

21# Applebot continues to make hundreds of thousands of reqs/day for this area

22# even though it has been returning permanent redirects for years

23User-agent: Applebot

24Disallow: /People/domain/

25

26# the following settings apply to all bots

27User-agent: *

28# Blogs - WordPress

29# https://codex.wordpress.org/Search_Engine_Optimization_for_WordPress#Robots.txt_Optimization

30Disallow: /*/wp-admin/

31Disallow: /*/wp-includes/

32Disallow: /*/wp-content/plugins/

33Disallow: /*/wp-content/cache/

34Disallow: /*/wp-content/themes/

35Disallow: /blog/*/trackback/

36Disallow: /blog/*/feed/

37Disallow: /blog/*/comments/

38Disallow: /blog/*/category/*/*

39Disallow: /blog/*/*/trackback/

40Disallow: /blog/*/*/feed/

...