Text File

1#

2# robots.txt for https://www.w3.org/

3#

4# $Id: robots.txt,v 1.94 2024/10/24 05:23:37 gerald Exp $

5#

6

7# the following settings apply to all bots

8User-agent: *

9# Blogs - WordPress

10# https://codex.wordpress.org/Search_Engine_Optimization_for_WordPress#Robots.txt_Optimization

11Disallow: /*/wp-admin/

12Disallow: /*/wp-includes/

13Disallow: /*/wp-content/plugins/

14Disallow: /*/wp-content/cache/

15Disallow: /*/wp-content/themes/

16Disallow: /blog/*/trackback/

17Disallow: /blog/*/feed/

18Disallow: /blog/*/comments/

19Disallow: /blog/*/category/*/*

20Disallow: /blog/*/*/trackback/

21Disallow: /blog/*/*/feed/

22Disallow: /blog/*/*/comments/

23Disallow: /blog/*/*?

24Disallow: /community/trackback/

25Disallow: /community/feed/

26Disallow: /community/comments/

27Disallow: /community/category/*/*

28Disallow: /community/*/trackback/

29Disallow: /community/*/feed/

30Disallow: /community/*/comments/

31Disallow: /community/*/category/*/*

32Disallow: /community/*?

33Disallow: /Consortium/Offices/trackback/

34Disallow: /Consortium/Offices/feed/

35Disallow: /Consortium/Offices/comments/

36Disallow: /Consortium/Offices/category/*/*

37Disallow: /Consortium/Offices/*/trackback/

38Disallow: /Consortium/Offices/*/feed/

39Disallow: /Consortium/Offices/*/comments/

40Disallow: /Consortium/Offices/*?

...