Text File

1#

2# robots.txt for https://www.w3.org/

3#

4# $Id: robots.txt,v 1.101 2025/12/16 04:23:23 gerald Exp $

5#

6

7# the following settings apply to all bots

8User-agent: *

9# Blogs - WordPress

10# https://wordpress.org/documentation/article/search-engine-optimization/#robots-txt-optimization

11Disallow: /*/wp-admin/

12Disallow: /*/wp-includes/

13Disallow: /*/wp-content/plugins/

14Disallow: /*/wp-content/cache/

15Disallow: /*/wp-content/themes/

16Disallow: /blog/?

17Disallow: /blog/*/trackback/

18Disallow: /blog/*/feed/

19Disallow: /blog/*/comments/

20Disallow: /blog/*/category/*/*

21Disallow: /blog/*/*/trackback/

22Disallow: /blog/*/*/feed/

23Disallow: /blog/*/*/comments/

24Disallow: /blog/*/*?

25Disallow: /community/trackback/

26Disallow: /community/feed/

27Disallow: /community/comments/

28Disallow: /community/category/*/*

29Disallow: /community/*/trackback/

30Disallow: /community/*/feed/

31Disallow: /community/*/comments/

32Disallow: /community/*/category/*/*

33Disallow: /community/*?

34Disallow: /Consortium/Offices/trackback/

35Disallow: /Consortium/Offices/feed/

36Disallow: /Consortium/Offices/comments/

37Disallow: /Consortium/Offices/category/*/*

38Disallow: /Consortium/Offices/*/trackback/

39Disallow: /Consortium/Offices/*/feed/

40Disallow: /Consortium/Offices/*/comments/

...