############################### # robots.txt for USF.edu ############################### User-agent: * # list folders robots are not allowed to index Disallow: /images Disallow: /css Disallow: /aspnet_client Disallow: /javascripts Disallow: /snapshots Disallow: /web-templates Disallow: /inc Disallow: /utilities Disallow: /calendar Disallow: /it/archive Disallow: /test-bob Disallow: /student-affairs/tedx # # list specific files robots are not allowed to index # Disallow: /Atest.html Disallow: /jstest.asp Disallow: /About-USF/contact-send.asp Disallow: /About-USF/page-not-found.asp # # # User-agent: Twitterbot # allow twitter to fetch images for news Allow: /images # # # End of robots.txt file # ###############################