# =================================== # Robots.txt for Tdawi (Magento 2) # Last updated: 2026-05-10 (tightened: only user-driven AI fetches allowed; block training/index crawlers) # ============================= # ============================= # Crawl Management # ============================= User-agent: SemrushBot Crawl-delay: 20 # RFC 9309: a UA-specific group is used ALONE (never merged with `*`), # and a group with no rules allows everything - so Crawl-delay on its own # granted full access. Rules below mirror the `*` group verbatim. Disallow: /lib/ Disallow: /*.php$ Disallow: /pkginfo/ Disallow: /report/ Disallow: /var/ Disallow: /sendfriend/ Disallow: /review/ Disallow: /*SID= Disallow: /onestepcheckout/ Disallow: /app/ Disallow: /bin/ Disallow: /dev/ Disallow: /phpserver/ Disallow: /*?dir* Disallow: /*?dir=desc Disallow: /*?dir=asc Disallow: /*?limit=all Disallow: /*?mode* Disallow: /*?price= Disallow: /*&price= Disallow: */review/ Disallow: */sendfriend/ Disallow: */onestepcheckout/ Disallow: */report/ Disallow: */catalogsearch/advanced/ Disallow: */checkout/ Disallow: */customer/ Allow: */customer/account/login Disallow: */wishlist/ Disallow: /*?manufacturer= Disallow: /*&manufacturer= Disallow: /*?cat= Disallow: /*&cat= Disallow: /*%2C Disallow: /*%2c Disallow: /*, # CF image-resizing URLs carry comma-separated params # (/cdn-cgi/image/width=767,quality=82,format=auto/media/...), so the # comma rules above blocked them. Longer pattern wins per RFC 9309. Allow: /cdn-cgi/image/ Disallow: /*product_list_order=* Disallow: /*?is_in_stock=* Disallow: /*rating=* Disallow: /customer/* Allow: /customer/account/login Disallow: /checkout/* Disallow: /*add-to-cart=* Disallow: /*wishlist/* Disallow: /catalogsearch/ Disallow: */search/* Disallow: /*/catalogsearch/ Disallow: /*?q=* Disallow: /*&q=* Disallow: /catalog/product/view/ Disallow: /*/catalog/product/view/ Disallow: /catalog/product_compare/ Disallow: /rest/ Disallow: /graphql Disallow: /paypal/ Disallow: /newsletter/ Allow: /pub/static/ Allow: /pub/media/ User-agent: AhrefsBot Crawl-delay: 15 # RFC 9309: a UA-specific group is used ALONE (never merged with `*`), # and a group with no rules allows everything - so Crawl-delay on its own # granted full access. Rules below mirror the `*` group verbatim. Disallow: /lib/ Disallow: /*.php$ Disallow: /pkginfo/ Disallow: /report/ Disallow: /var/ Disallow: /sendfriend/ Disallow: /review/ Disallow: /*SID= Disallow: /onestepcheckout/ Disallow: /app/ Disallow: /bin/ Disallow: /dev/ Disallow: /phpserver/ Disallow: /*?dir* Disallow: /*?dir=desc Disallow: /*?dir=asc Disallow: /*?limit=all Disallow: /*?mode* Disallow: /*?price= Disallow: /*&price= Disallow: */review/ Disallow: */sendfriend/ Disallow: */onestepcheckout/ Disallow: */report/ Disallow: */catalogsearch/advanced/ Disallow: */checkout/ Disallow: */customer/ Allow: */customer/account/login Disallow: */wishlist/ Disallow: /*?manufacturer= Disallow: /*&manufacturer= Disallow: /*?cat= Disallow: /*&cat= Disallow: /*%2C Disallow: /*%2c Disallow: /*, # CF image-resizing URLs carry comma-separated params # (/cdn-cgi/image/width=767,quality=82,format=auto/media/...), so the # comma rules above blocked them. Longer pattern wins per RFC 9309. Allow: /cdn-cgi/image/ Disallow: /*product_list_order=* Disallow: /*?is_in_stock=* Disallow: /*rating=* Disallow: /customer/* Allow: /customer/account/login Disallow: /checkout/* Disallow: /*add-to-cart=* Disallow: /*wishlist/* Disallow: /catalogsearch/ Disallow: */search/* Disallow: /*/catalogsearch/ Disallow: /*?q=* Disallow: /*&q=* Disallow: /catalog/product/view/ Disallow: /*/catalog/product/view/ Disallow: /catalog/product_compare/ Disallow: /rest/ Disallow: /graphql Disallow: /paypal/ Disallow: /newsletter/ Allow: /pub/static/ Allow: /pub/media/ User-agent: MJ12bot Crawl-delay: 15 # RFC 9309: a UA-specific group is used ALONE (never merged with `*`), # and a group with no rules allows everything - so Crawl-delay on its own # granted full access. Rules below mirror the `*` group verbatim. Disallow: /lib/ Disallow: /*.php$ Disallow: /pkginfo/ Disallow: /report/ Disallow: /var/ Disallow: /sendfriend/ Disallow: /review/ Disallow: /*SID= Disallow: /onestepcheckout/ Disallow: /app/ Disallow: /bin/ Disallow: /dev/ Disallow: /phpserver/ Disallow: /*?dir* Disallow: /*?dir=desc Disallow: /*?dir=asc Disallow: /*?limit=all Disallow: /*?mode* Disallow: /*?price= Disallow: /*&price= Disallow: */review/ Disallow: */sendfriend/ Disallow: */onestepcheckout/ Disallow: */report/ Disallow: */catalogsearch/advanced/ Disallow: */checkout/ Disallow: */customer/ Allow: */customer/account/login Disallow: */wishlist/ Disallow: /*?manufacturer= Disallow: /*&manufacturer= Disallow: /*?cat= Disallow: /*&cat= Disallow: /*%2C Disallow: /*%2c Disallow: /*, # CF image-resizing URLs carry comma-separated params # (/cdn-cgi/image/width=767,quality=82,format=auto/media/...), so the # comma rules above blocked them. Longer pattern wins per RFC 9309. Allow: /cdn-cgi/image/ Disallow: /*product_list_order=* Disallow: /*?is_in_stock=* Disallow: /*rating=* Disallow: /customer/* Allow: /customer/account/login Disallow: /checkout/* Disallow: /*add-to-cart=* Disallow: /*wishlist/* Disallow: /catalogsearch/ Disallow: */search/* Disallow: /*/catalogsearch/ Disallow: /*?q=* Disallow: /*&q=* Disallow: /catalog/product/view/ Disallow: /*/catalog/product/view/ Disallow: /catalog/product_compare/ Disallow: /rest/ Disallow: /graphql Disallow: /paypal/ Disallow: /newsletter/ Allow: /pub/static/ Allow: /pub/media/ # ============================= # General Restrictions # ============================= User-agent:* Disallow: /lib/ Disallow: /*.php$ Disallow: /pkginfo/ Disallow: /report/ Disallow: /var/ Disallow: /sendfriend/ Disallow: /review/ Disallow: /*SID= Disallow: /onestepcheckout/ Disallow: /app/ Disallow: /bin/ Disallow: /dev/ Disallow: /phpserver/ Disallow: /*?dir* Disallow: /*?dir=desc Disallow: /*?dir=asc Disallow: /*?limit=all Disallow: /*?mode* Disallow: /*?price= Disallow: /*&price= # 2026-07-28: every storefront URL is store-code prefixed (/en_eg/…, /en-qa/…), # so the root-relative rules above never match the real URLs. Googlebot was # crawling and 404ing ~1,000/day of /{locale}/review/product/listAjax/*. Disallow: */review/ Disallow: */sendfriend/ Disallow: */onestepcheckout/ Disallow: */report/ Disallow: */catalogsearch/advanced/ Disallow: */checkout/ Disallow: */customer/ Allow: */customer/account/login Disallow: */wishlist/ # 2026-10-28: every product page links to its own login URL # (/{store}/customer/account/login/referer//) = ~300K unique NOINDEX # pages Google keeps crawling. Google's crawl-budget guide: block, don't noindex. # Longer than the Allow above, so it wins; the plain login page stays crawlable. Disallow: /*/customer/account/login/referer/ # Analytics pings fired while a page renders (counters only, no page content). Disallow: /*/tdawi_fpc/reports/ Disallow: /*/tdawirp/analytics/ Disallow: /*/reports/report_product/ # 2026-10-28: store-prefixed twins of Magento's standard blocks (the root-relative # /catalog/product_compare/ rule never matches /{store}/... URLs). ESI only: browsers # do use /page_cache/block/render/, so that stays open. Disallow: /*/catalog/product_compare/ Disallow: /*/sales/guest/ Disallow: /*/page_cache/block/esi/ # 2026-10-28: per-product "notify me" links (/{store}/productalert/add/.../uenc/..., # 302 to login) and JS-only endpoints with no page content: Meta pixel (/fbe/, # also served at the root) and search-autocomplete JSON. Disallow: /productalert/ Disallow: /*/productalert/ Disallow: /fbe/ Disallow: /*/fbe/ Disallow: /*/mageworx_searchsuiteautocomplete/ Disallow: /*?manufacturer= Disallow: /*&manufacturer= Disallow: /*?cat= Disallow: /*&cat= Disallow: /*%2C Disallow: /*%2c Disallow: /*, # CF image-resizing URLs carry comma-separated params # (/cdn-cgi/image/width=767,quality=82,format=auto/media/...), so the # comma rules above blocked them. Longer pattern wins per RFC 9309. Allow: /cdn-cgi/image/ Disallow: /*product_list_order=* Disallow: /*?is_in_stock=* Disallow: /*rating=* Disallow: /customer/* Allow: /customer/account/login Disallow: /checkout/* Disallow: /*add-to-cart=* Disallow: /*wishlist/* Disallow: /catalogsearch/ Disallow: */search/* # store-code prefixed search URLs (/en_eg/catalogsearch/...) — the bare # /catalogsearch/ rule is a prefix match from root and matches NONE of them Disallow: /*/catalogsearch/ Disallow: /*?q=* Disallow: /*&q=* Disallow: /catalog/product/view/ Disallow: /*/catalog/product/view/ Disallow: /catalog/product_compare/ Disallow: /rest/ Disallow: /graphql Disallow: /paypal/ Disallow: /newsletter/ # Allow assets for full rendering Allow: /pub/static/ Allow: /pub/media/ # ============================= # Explicit Bot Rules # ============================= User-agent: Bingbot Crawl-delay: 10 Disallow: /catalogsearch/ # 2026-11-10: store-prefixed search pages (/{store}/catalogsearch/result/?...). # Bing reads only this group, and the root-relative rule above never matches them. Disallow: /*/catalogsearch/ Disallow: */search/* Disallow: /*?q=* # 2026-07-28: storefront URLs are store-code prefixed (/en_eg/…), so the # root-relative rules never match them. Googlebot was 404ing ~1k/day on # /{locale}/review/product/listAjax/*. Disallow: */review/ Disallow: */sendfriend/ Disallow: */onestepcheckout/ Disallow: */checkout/ Disallow: */customer/ Allow: */customer/account/login Disallow: */wishlist/ # 2026-10-28: every product page links to its own login URL # (/{store}/customer/account/login/referer//) = ~300K unique NOINDEX # pages Google keeps crawling. Google's crawl-budget guide: block, don't noindex. # Longer than the Allow above, so it wins; the plain login page stays crawlable. Disallow: /*/customer/account/login/referer/ # Analytics pings fired while a page renders (counters only, no page content). Disallow: /*/tdawi_fpc/reports/ Disallow: /*/tdawirp/analytics/ Disallow: /*/reports/report_product/ # 2026-10-28: store-prefixed twins of Magento's standard blocks (the root-relative # /catalog/product_compare/ rule never matches /{store}/... URLs). ESI only: browsers # do use /page_cache/block/render/, so that stays open. Disallow: /*/catalog/product_compare/ Disallow: /*/sales/guest/ Disallow: /*/page_cache/block/esi/ # 2026-10-28: per-product "notify me" links (/{store}/productalert/add/.../uenc/..., # 302 to login) and JS-only endpoints with no page content: Meta pixel (/fbe/, # also served at the root) and search-autocomplete JSON. Disallow: /productalert/ Disallow: /*/productalert/ Disallow: /fbe/ Disallow: /*/fbe/ Disallow: /*/mageworx_searchsuiteautocomplete/ Allow: / # ============================= # AI Bots — Allow OpenAI / Anthropic / Google (real referral traffic), # block the rest. Network-level CF ai_bots_protection=block must also be # adjusted in dashboard if it's blocking the allowed UAs. # ============================= # Allow — these drive real user citation traffic User-agent: ChatGPT-User Allow: / User-agent: Claude-User Allow: / User-agent: Claude-Web Allow: / User-agent: anthropic-ai Allow: / User-agent: Google-Extended Allow: / # Allow — OpenAI Search index. Powers ChatGPT Search citations. # Distinct from GPTBot (model-training) which remains blocked below. User-agent: OAI-SearchBot Allow: / # Block — training crawlers, scrapers User-agent: GPTBot Disallow: / User-agent: ClaudeBot Disallow: / User-agent: Amazonbot Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: Bytespider Disallow: / User-agent: CCBot Disallow: / User-agent: Diffbot Disallow: / User-agent: FacebookBot Disallow: / User-agent: Meta-ExternalAgent Disallow: / User-agent: Meta-ExternalFetcher Disallow: / User-agent: cohere-ai Disallow: / User-agent: Omgilibot Disallow: / User-agent: Omgili Disallow: / User-agent: YouBot Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: Timpibot Disallow: / User-agent: Webzio-Extended Disallow: / User-agent: AI2Bot Disallow: / User-agent: DuckAssistBot Disallow: / User-agent: PetalBot Disallow: / # ============================= # Sitemaps # ============================= Sitemap: https://tdawi.com/sitemap.xml Sitemap: https://tdawi.com/sitemap_ar.xml Sitemap: https://tdawi.com/sitemap_qatar_en.xml Sitemap: https://tdawi.com/sitemap_qatar_ar.xml Sitemap: https://tdawi.com/sitemap_saudi_en.xml Sitemap: https://tdawi.com/sitemap_saudi_ar.xml Sitemap: https://tdawi.com/sitemap_usa_en.xml Sitemap: https://tdawi.com/sitemap_germany_en.xml Sitemap: https://tdawi.com/sitemap_germany_de.xml Sitemap: https://tdawi.com/sitemap_uk_en.xml Sitemap: https://tdawi.com/sitemap_videos.xml