User-agent: AhrefsBot
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: Bytedance
User-agent: Bytespider
User-agent: CCBot
User-agent: cohere-ai
User-agent: DataForSeoBot
User-agent: Diffbot
User-agent: FacebookBot
User-agent: SemrushBot
User-agent: FriendlyCrawler
User-agent: GPTBot
User-agent: img2dataset
User-agent: ImagesiftBot
User-agent: magpie-crawler
User-agent: Meltwater
User-agent: omgili
User-agent: omgilibot
User-agent: peer39_crawler
User-agent: peer39_crawler/1.0
User-agent: PerplexityBot
User-agent: PiplBot
User-agent: scoop.it
User-agent: Seekr
User-agent: YandexBot
User-agent: YouBot
Disallow: /

User-agent: *
Disallow: /admin/stats/
Disallow: /goldenPath/certificate.html
Disallow: /goldenPath/certificates/
Disallow: /cgi-bin/hgTracks*.customText*.
Disallow: /cgi-bin/hgBlat*output=json*
Disallow: /cgi-bin/hgGeneGraph*
