User-agent: Googlebot-News
Allow: /news/

# Explicitly allow Google’s general web crawler
User-agent: Googlebot
Allow: /
Disallow: /marketing-communications/web-services/resources/website-demo/
Disallow: /childrens-heart-institute-test/

# Screaming Frog crawler
User-agent: Screaming Frog SEO Spider
Allow: /sarofim/
Allow: /marketing-communications/
Allow: /marketing-communications/web-services/resources/website-demo/
Allow: /investiture/
Allow: /president/investiture

User-agent: SiteimproveBot
Allow: /investiture/
Allow: /president/investiture

User-agent: SiteimproveBot-Crawler
Allow: /investiture/
Allow: /president/investiture

# All crawlers
User-agent: *
Disallow: /age-inclusively/
Disallow: /contentAsset/
Disallow: /dotAdmin/
Disallow: /dotAsset/
Disallow: /emergency/
Disallow: /global/youtubemodal/
Disallow: /international-affairs/document/
Disallow: /temp/
Disallow: /test-folders/
Disallow: /training/
Disallow: /marketing-communications/web-services/resources/website-demo/
Disallow: /childrens-heart-institute-test/
Disallow: /investiture/
Disallow: /president/investiture

Sitemap: https://www.uth.edu/uth-public-sitemap.xml
Sitemap: https://www.uth.edu/news-sitemap.xml