User-agent: *
Disallow: /page/*/?s=
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-content/uploads/wpforms/
Disallow: /wp-admin/ #block access to admin section
Disallow: /wp-login.php #block access to backend login section
Disallow: /search/ #block access to internal search result pages
Disallow: *?s=* #block access to internal search result pages
Disallow: *?p=* #block access to pages for which permalinks fails
Disallow: *&p=* #block access to pages for which permalinks fails
Disallow: *&preview=* #block access to preview pages
# Disallow search query url's
Disallow: *?jobtitel=* #block access to internal search result pages
Disallow: *?locatie=* #block access to internal search result pages
Disallow: *?diploma=* #block access to internal search result pages
Disallow: *?dienst=* #block access to internal search result pages
Disallow: *?contracttype=* #block access to internal search result pages
Disallow: *&jobtitel=* #block access to internal search result pages
Disallow: *&locatie=* #block access to internal search result pages
Disallow: *&diploma=* #block access to internal search result pages
Disallow: *&dienst=* #block access to internal search result pages
Disallow: *&contracttype=* #block access to internal search result pages

Sitemap: https://www.jobfin.be/sitemap_index.xml

#Trying to prevent access from AI bots

User-agent: Amazonbot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

#User-agent: ChatGPT-User
#Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FriendlyCrawler
Disallow: /

#User-agent: GPTBot
#Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: peer39_crawler
Disallow: /

User-agent: peer39_crawler/1.0
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: YouBot
Disallow: /

#Trying to avoid other bots
User-Agent: Baiduspider
Disallow: /

User-Agent: CazoodleBot
Disallow: /

User-Agent: Fasterfox
Disallow: /

User-Agent: Jyxobot
Disallow: /

User-Agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: EtaoSpider
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: YandexBot
Disallow: /

Crawl-delay:5
                        
#            *********    
#        ***************  
#      ********   ******  
#    *******      ******  
#   ******                
#    ***((((      ######  
#      ((((((((   ######  
#        (((((((((((((((  
#            (((((((((