#****************************************************************************
# PrettyLittleThing.se robots.txt
#****************************************************************************

# # Bots to block 
User-agent: python-grcrawler
Disallow: /

User-agent: proximic
Disallow: /

User-agent: grapeshot
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: Wget
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

# # AdsBot + Storebot
#****************************************************************************
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google
User-agent: Storebot-Google

# Do not crawl legacy system settings
Disallow: */rb_bf12093eab
Disallow: */ruxitagentjs
Disallow: */akam
Disallow: */nugg

# Do not crawl system settings
Disallow: */api/*
Disallow: */_api/*
Disallow: /metrics/
Disallow: */metrics/*

# # All other bots
#****************************************************************************
User-agent: *

# # Legacy
# Do not crawl legacy pages
Disallow: */catalogsearch/
Disallow: */catalog/product/
Disallow: */catalog/category/
Disallow: */pltcatalog/stock/getStock
Disallow: */onestepcheckout
Disallow: */customer/
Disallow: */sales/order
Disallow: */tracking/?order
Disallow: */directory/currency/switch/currency/
Disallow: */nugg
Disallow: */ajaxbanners
Disallow: */merchandiser
Disallow: */bms/
Disallow: */nms/
Disallow: */cms/
Disallow: */worldpay/

# Do not crawl legacy parameters
Disallow: *?navigation
Disallow: *brands=
Disallow: *colour_group=
Disallow: *clothes_size_mapped=
Disallow: *price_slider=
Disallow: *product_category=
Disallow: *style=
Disallow: *?fit=
Disallow: *?sleeves=
Disallow: *clothes_size=
Disallow: *?brand=
Disallow: *?price=
Disallow: *?itemsperpage=

# Do not crawl legacy system settings
Disallow: */rb_bf12093eab
Disallow: */ruxitagentjs
Disallow: */akam

# # New
# Do not crawl checkout and customer account section 
Disallow: */account
Disallow: */wishlist
Disallow: */search
Disallow: */basket
Disallow: */checkout
Disallow: */login
Disallow: */reset-password
Disallow: */sign-in-link

# Do not crawl system settings
Disallow: */wordsworth
Disallow: */api/*
Disallow: */_api/*

# Do not crawl facets
Disallow: /*/facet/collection
Disallow: /*/facet/categoryTaxonomy
Disallow: /*/facet/brand
Disallow: /*/facet/sleeveLength
Disallow: /*/facet/styleTaxonomy
Disallow: /*/facet/classification
Disallow: /*/facet/discountBands
Disallow: /*/facet/discountRange
Disallow: /*/facet/gender
Disallow: /*/facet/priceBands
Disallow: /*/facet/priceRange
Disallow: /*/facet/occasion
Disallow: /*/facet/department
Disallow: /*/facet/isNextDay
Disallow: /*/facet/colourFacet
Disallow: /*/facet/sizes

# Do not crawl any broken URL formats
Disallow: *amp;amp;amp
Disallow: .html.
Disallow: .html/1000
Disallow: *Partner.URL
Disallow: *US_CENSUS_NAME

# Do not crawl these parameters
Disallow: *?SID=
Disallow: *?sid=
Disallow: *=affiliate
Disallow: *=awin
Disallow: *=email
Disallow: *?awc
Disallow: /categories/*?colour=
Disallow: *select_item_id=*
Disallow: *?homepage-pillar*
Disallow: *?Homepage-Pillar*

# Other sections to block
Disallow: /clicktale
Disallow: /orasso
Disallow: /itm/
Disallow: /br/pathways/
Disallow: /metrics/
Disallow: */metrics/*

# Spam Bots
Disallow: /member/
Disallow: /browse/
Disallow: /redirect
Disallow: /banner_conta.php
Disallow: /go?*
Disallow: /go.php
Disallow: /goto
Disallow: /*.ru
Disallow: /open_is_url

# Sitemaps
Sitemap: https://www.prettylittlething.se/sitemap/prettylittlethingse/sitemap.xml

Sitemap: https://www.prettylittlething.se/sitemap/prettylittlethingse/products-0.xml
Sitemap: https://www.prettylittlething.se/sitemap/prettylittlethingse/products-1.xml