###############################
# robots.txt for USF.edu
###############################

User-agent: *
# list folders robots are not allowed to index
Disallow: /images
Disallow: /css
Disallow: /aspnet_client
Disallow: /javascripts
Disallow: /snapshots
Disallow: /web-templates
Disallow: /inc
Disallow: /utilities
Disallow: /calendar
Disallow: /it/archive
Disallow: /test-bob
Disallow: /student-affairs/tedx
Disallow: /*.pdf$
Disallow: /*.doc$
Disallow: /*.docx$
Disallow: /*.ppt$
Disallow: /*.pptx$
Disallow: /*.xls$
Disallow: /*.xlsx$
Disallow: /*.rtf$
Disallow: /*.txt$
#
# list specific files robots are not allowed to index
#
Disallow: /Atest.html
Disallow: /jstest.asp
Disallow: /About-USF/contact-send.asp
Disallow: /About-USF/page-not-found.asp
#

# Ahrefs for Admissions
User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
Allow: /admissions

#
#
User-agent: Twitterbot
# allow twitter to fetch images for news
Allow: /images
#
#
# End of robots.txt file
#
###############################