# # robots.txt for Inner Banks Living (http://www.innerbanksliving.com) # # Colnect has a lot of pages so misbehaved spiders out may be blocked! # If your bot is rejected by this file or getting a permission denied # you may contact us directly through the site. # # Happy collecting :) # Sitemap: http://www.innerbanksliving.com/sitemap.xml # advertising-related bots: #User-agent: Mediapartners-Google* #Disallow: # Wikipedia work bots: #User-agent: IsraBot #Disallow: #User-agent: Orthogaffe #Disallow: # Crawlers that are kind enough to obey, but which we'd rather not have # unless they're feeding search engines. User-agent: UbiCrawler Disallow: / User-agent: DOC Disallow: / User-agent: Zao Disallow: / User-agent: Java/1.6.0_10 Disallow: / # Some bots are known to be trouble, particularly those designed to copy # entire sites. Please obey robots.txt. User-agent: sitecheck.internetseer.com Disallow: / User-agent: Zealbot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: linko Disallow: / User-agent: HTTrack Disallow: / User-agent: Microsoft.URL.Control Disallow: / User-agent: Xenu Disallow: / User-agent: larbin Disallow: / User-agent: libwww Disallow: / User-agent: ZyBORG Disallow: / User-agent: Download Ninja Disallow: / # # Sorry, wget in its recursive mode is a frequent problem. # Please read the man page and use it properly; there is a # --wait option you can use to set the delay between hits, # for instance. # User-agent: wget Disallow: / # # The 'grub' distributed client has been *very* poorly behaved. # User-agent: grub-client Disallow: / # # Doesn't follow robots.txt anyway, but... # User-agent: k2spider Disallow: / # # Hits many times per second, not acceptable # http://www.nameprotect.com/botinfo.html User-agent: NPBot Disallow: / # A capture bot, downloads gazillions of pages with no public benefit # http://www.webreaper.net/ User-agent: WebReaper Disallow: / # # Friendly, low-speed bots are welcome viewing article pages, but not # dynamically-generated pages please. # # Inktomi's "Slurp" can read a minimum delay between hits; if your # bot supports such a thing using the 'Crawl-delay' or another # instruction, please let us know. # User-agent: Slurp Crawl-delay: 2 Disallow: /SlurpConfirm404/ # Alexa User-agent: ia_archiver Disallow: / # Ask Fasterfox plugin not to prefetch User-agent: Fasterfox Disallow: / # All the rest User-agent: * Disallow: /fld/ Disallow: /teletalk/ Disallow: /downloads/ Disallow: /main/redir/ Disallow: /ar/main/redir/ Disallow: /bg/main/redir/ Disallow: /bn/main/redir/ Disallow: /br/main/redir/ Disallow: /cs/main/redir/ Disallow: /de/main/redir/ Disallow: /el/main/redir/ Disallow: /es/main/redir/ Disallow: /fa/main/redir/ Disallow: /fi/main/redir/ Disallow: /fr/main/redir/ Disallow: /he/main/redir/ Disallow: /hr/main/redir/ Disallow: /hu/main/redir/ Disallow: /it/main/redir/ Disallow: /ko/main/redir/ Disallow: /lv/main/redir/ Disallow: /mk/main/redir/ Disallow: /ms/main/redir/ Disallow: /nl/main/redir/ Disallow: /pl/main/redir/ Disallow: /pt/main/redir/ Disallow: /ro/main/redir/ Disallow: /ru/main/redir/ Disallow: /sk/main/redir/ Disallow: /sl/main/redir/ Disallow: /sv/main/redir/ Disallow: /th/main/redir/ Disallow: /tr/main/redir/ Disallow: /uk/main/redir/ Disallow: /ur/main/redir/ Disallow: /zh/main/redir/ Disallow: /zt/main/redir/ Disallow: /ja/main/redir/ Disallow: /lt/main/redir/ Disallow: /item/view_collectors/ Disallow: /ar/item/view_collectors/ Disallow: /bg/item/view_collectors/ Disallow: /bn/item/view_collectors/ Disallow: /br/item/view_collectors/ Disallow: /cs/item/view_collectors/ Disallow: /de/item/view_collectors/ Disallow: /el/item/view_collectors/ Disallow: /es/item/view_collectors/ Disallow: /fa/item/view_collectors/ Disallow: /fi/item/view_collectors/ Disallow: /fr/item/view_collectors/ Disallow: /he/item/view_collectors/ Disallow: /hr/item/view_collectors/ Disallow: /hu/item/view_collectors/ Disallow: /it/item/view_collectors/ Disallow: /ko/item/view_collectors/ Disallow: /lv/item/view_collectors/ Disallow: /mk/item/view_collectors/ Disallow: /ms/item/view_collectors/ Disallow: /nl/item/view_collectors/ Disallow: /pl/item/view_collectors/ Disallow: /pt/item/view_collectors/ Disallow: /ro/item/view_collectors/ Disallow: /ru/item/view_collectors/ Disallow: /sk/item/view_collectors/ Disallow: /sl/item/view_collectors/ Disallow: /sv/item/view_collectors/ Disallow: /th/item/view_collectors/ Disallow: /tr/item/view_collectors/ Disallow: /uk/item/view_collectors/ Disallow: /ur/item/view_collectors/ Disallow: /zh/item/view_collectors/ Disallow: /zt/item/view_collectors/ Disallow: /ja/item/view_collectors/ Disallow: /lt/item/view_collectors/ Disallow: /collectors/log/ Disallow: /ar/collectors/log/ Disallow: /bg/collectors/log/ Disallow: /bn/collectors/log/ Disallow: /br/collectors/log/ Disallow: /cs/collectors/log/ Disallow: /de/collectors/log/ Disallow: /el/collectors/log/ Disallow: /es/collectors/log/ Disallow: /fa/collectors/log/ Disallow: /fi/collectors/log/ Disallow: /fr/collectors/log/ Disallow: /he/collectors/log/ Disallow: /hr/collectors/log/ Disallow: /hu/collectors/log/ Disallow: /it/collectors/log/ Disallow: /ko/collectors/log/ Disallow: /lv/collectors/log/ Disallow: /mk/collectors/log/ Disallow: /ms/collectors/log/ Disallow: /nl/collectors/log/ Disallow: /pl/collectors/log/ Disallow: /pt/collectors/log/ Disallow: /ro/collectors/log/ Disallow: /ru/collectors/log/ Disallow: /sk/collectors/log/ Disallow: /sl/collectors/log/ Disallow: /sv/collectors/log/ Disallow: /th/collectors/log/ Disallow: /tr/collectors/log/ Disallow: /uk/collectors/log/ Disallow: /ur/collectors/log/ Disallow: /zh/collectors/log/ Disallow: /zt/collectors/log/ Disallow: /ja/collectors/log/ Disallow: /lt/collectors/log/ Disallow: /collectors/rate/ Disallow: /ar/collectors/rate/ Disallow: /bg/collectors/rate/ Disallow: /bn/collectors/rate/ Disallow: /br/collectors/rate/ Disallow: /cs/collectors/rate/ Disallow: /de/collectors/rate/ Disallow: /el/collectors/rate/ Disallow: /es/collectors/rate/ Disallow: /fa/collectors/rate/ Disallow: /fi/collectors/rate/ Disallow: /fr/collectors/rate/ Disallow: /he/collectors/rate/ Disallow: /hr/collectors/rate/ Disallow: /hu/collectors/rate/ Disallow: /it/collectors/rate/ Disallow: /ko/collectors/rate/ Disallow: /lv/collectors/rate/ Disallow: /mk/collectors/rate/ Disallow: /ms/collectors/rate/ Disallow: /nl/collectors/rate/ Disallow: /pl/collectors/rate/ Disallow: /pt/collectors/rate/ Disallow: /ro/collectors/rate/ Disallow: /ru/collectors/rate/ Disallow: /sk/collectors/rate/ Disallow: /sl/collectors/rate/ Disallow: /sv/collectors/rate/ Disallow: /th/collectors/rate/ Disallow: /tr/collectors/rate/ Disallow: /uk/collectors/rate/ Disallow: /ur/collectors/rate/ Disallow: /zh/collectors/rate/ Disallow: /zt/collectors/rate/ Disallow: /ja/collectors/rate/ Disallow: /lt/collectors/rate/ Disallow: /account/ Disallow: /ar/account/ Disallow: /bg/account/ Disallow: /bn/account/ Disallow: /br/account/ Disallow: /cs/account/ Disallow: /de/account/ Disallow: /el/account/ Disallow: /es/account/ Disallow: /fa/account/ Disallow: /fi/account/ Disallow: /fr/account/ Disallow: /he/account/ Disallow: /hr/account/ Disallow: /hu/account/ Disallow: /it/account/ Disallow: /ko/account/ Disallow: /lv/account/ Disallow: /mk/account/ Disallow: /ms/account/ Disallow: /nl/account/ Disallow: /pl/account/ Disallow: /pt/account/ Disallow: /ro/account/ Disallow: /ru/account/ Disallow: /sk/account/ Disallow: /sl/account/ Disallow: /sv/account/ Disallow: /th/account/ Disallow: /tr/account/ Disallow: /uk/account/ Disallow: /ur/account/ Disallow: /zh/account/ Disallow: /zt/account/ Disallow: /ja/account/ Disallow: /lt/account/ Disallow: /directory/ Disallow: /ar/directory/ Disallow: /bg/directory/ Disallow: /bn/directory/ Disallow: /br/directory/ Disallow: /cs/directory/ Disallow: /de/directory/ Disallow: /el/directory/ Disallow: /es/directory/ Disallow: /fa/directory/ Disallow: /fi/directory/ Disallow: /fr/directory/ Disallow: /he/directory/ Disallow: /hr/directory/ Disallow: /hu/directory/ Disallow: /it/directory/ Disallow: /ko/directory/ Disallow: /lv/directory/ Disallow: /mk/directory/ Disallow: /ms/directory/ Disallow: /nl/directory/ Disallow: /pl/directory/ Disallow: /pt/directory/ Disallow: /ro/directory/ Disallow: /ru/directory/ Disallow: /sk/directory/ Disallow: /sl/directory/ Disallow: /sv/directory/ Disallow: /th/directory/ Disallow: /tr/directory/ Disallow: /uk/directory/ Disallow: /ur/directory/ Disallow: /zh/directory/ Disallow: /zt/directory/ Disallow: /ja/directory/ Disallow: /lt/directory/