# Robots.txt file for AGR Technology
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html
#     ____________________________
#     !\_________________________/!\
#     !!                         !! \
#     !!                         !!  \
#     !!  C:\AGR_Technology.exe  !!  !
#     !!                         !!  !
#     !!                         !!  !
#     !!                         !!  !
#     !!                         !!  !
#     !!                         !!  /
#     !!_________________________!! /
#     !/_________________________\!/
#        __\_________________/__/!_
#       !_______________________!/ )
#     ________________________    (__
#    /oooo  oooo  oooo  oooo /!   _  )_
#   /ooooooooooooooooooooooo/ /  (_)_(_)
#  /ooooooooooooooooooooooo/ /    (o o)
# /_______________________/_/    ==\o/==

User-agent: *
Allow: /wp-admin/admin-ajax.php
Disallow: /xmlrpc.php
Disallow: /cgi-bin
Disallow: /wp-admin$
Disallow: /wp-admin*/
Allow: /author/admin/
Disallow: /category/uncategorized$
Disallow: /category/uncategorized/
Disallow: */trackback/
Disallow: /print$
Disallow: /print/
Disallow: /search$
Disallow: /search/
Allow: /wp-admin/admin-ajax.php
Allow: /*.js
Allow: /*.css
Disallow: /xmlrpc.php$
Disallow: /recommends/
Disallow: /assets
Allow: /assets/brand

#Sitemaps
Sitemap: https://agrtech.com.au/sitemap_index.xml
Sitemap: https://agrtech.com.au/sitemap.xml
Sitemap: https://agrtech.com.au/post-sitemap.xml
Sitemap: https://agrtech.com.au/page-sitemap.xml

User-agent: Googlebot-Image
Allow: /wp-content/uploads/

User-agent: Twitterbot
Allow: /

User-agent: Baiduspider-image
Allow: /wp-content/uploads/

User-agent: NinjaBot
Allow: /

User-agent: rogerbot
Disallow: /

#80legs
User-agent: 008
Disallow: /

User-agent: voltron
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

# disallow stress test
user-agent: stress-agent
Disallow: /

# advertising-related bots:
User-agent: Mediapartners-Google*
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: larbin
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: SiteExplorer
Disallow: /

User-agent: SearchmetricsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: archive.org_bot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: WeSEE
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: dotbot
Disallow: /

# Misbehaving: requests much too fast:
User-agent: fast
Disallow: /