# robots.txt for https://www.teads.com/
# Goal: allow indexing of useful content while reducing the server load
# generated by crawlers.
#
# Served languages (verified on the site): English (root), /fr/, /ja/, /de/,
# /es/, /it/, /zh/ (Simplified Chinese), plus Hebrew and Brazilian Portuguese.
# The site runs on WordPress.
#
# NOTE: This file only governs www.teads.com. Subdomains
#   (investors., academy., engineering.teads.com, privacy-policy.teads.com,
#   login.teads.tv, ...) are NOT covered: each one needs its own robots.txt.
# NOTE: robots.txt is public and only honored by "well-behaved" crawlers.
#   Complement it with CDN/server-side rate limiting for abusive bots.

# =========================================================================
# 1) General rules: all crawlers without a specific group (including Googlebot).
#    NB: Googlebot's crawl rate is controlled in Google Search Console
#    (Googlebot ignores Crawl-delay).
# =========================================================================
User-agent: *
Crawl-delay: 10

# --- Internal search (near-infinite URL space) ---
Disallow: /search
Disallow: /*?s=
Disallow: /*?q=
Disallow: /*&s=
Disallow: /*&q=

# --- Faceted navigation / sorting / filters ---
Disallow: /*?*filter=
Disallow: /*?*sort=
Disallow: /*?*orderby=
Disallow: /*?*order=
Disallow: /*?*view=

# --- Tracking / campaign parameters ---
Disallow: /*?*utm_
Disallow: /*?*gclid=
Disallow: /*?*fbclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Disallow: /*?*ref=
Disallow: /*?*referrer=

# --- Comments / trackbacks ---
Disallow: /*?replytocom=
Disallow: /*/trackback/
Disallow: /trackback/

# --- Feeds / duplicate formats ---
Disallow: /*/feed/
Disallow: /feed/
Disallow: /*.atom$
Disallow: /*.rss$

# --- Printable / preview versions ---
Disallow: /*?print=
Disallow: /*?preview=

# --- WordPress: admin / API ---
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-json/
Disallow: /xmlrpc.php
Disallow: /?p=

# =========================================================================
# 2) Bing: same hygiene rules + rate limiting.
#    A crawler applies only ONE group, so the rules above do NOT
#    automatically apply to bingbot -> they are redeclared here.
# =========================================================================
User-agent: bingbot
Crawl-delay: 10
Disallow: /search
Disallow: /*?s=
Disallow: /*?q=
Disallow: /*&s=
Disallow: /*&q=
Disallow: /*?*filter=
Disallow: /*?*sort=
Disallow: /*?*orderby=
Disallow: /*?*order=
Disallow: /*?*view=
Disallow: /*?*utm_
Disallow: /*?*gclid=
Disallow: /*?*fbclid=
Disallow: /*?*mc_cid=
Disallow: /*?*mc_eid=
Disallow: /*?*ref=
Disallow: /*?*referrer=
Disallow: /*?replytocom=
Disallow: /*/trackback/
Disallow: /trackback/
Disallow: /*/feed/
Disallow: /feed/
Disallow: /*.atom$
Disallow: /*.rss$
Disallow: /*?print=
Disallow: /*?preview=
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-json/
Disallow: /xmlrpc.php
Disallow: /?p=

# =========================================================================
# 3) CHINESE regional search engines: Chinese IS served -> restrict to /zh/.
#    (The more specific "Allow: /zh/" overrides "Disallow: /".)
# =========================================================================

# --- Baidu (China) ---
User-agent: Baiduspider
Disallow: /
Allow: /zh/
Crawl-delay: 10

# --- Sogou (China) ---
User-agent: Sogou web spider
User-agent: Sogou inst spider
User-agent: Sogou spider2
Disallow: /
Allow: /zh/
Crawl-delay: 10

# --- 360 / Haosou (China) ---
User-agent: HaoSouSpider
User-agent: 360Spider
Disallow: /
Allow: /zh/
Crawl-delay: 10

# =========================================================================
# 4) Regional search engines whose language is NOT served -> fully blocked.
#    (Reduces load: they no longer crawl anything.)
# =========================================================================

# --- Yandex (Russia): no Russian content ---
User-agent: Yandex
Disallow: /

# --- Seznam (Czech Republic): no Czech content ---
User-agent: SeznamBot
Disallow: /

# --- Naver (South Korea): no Korean content ---
User-agent: Yeti
Disallow: /

# =========================================================================
# (OPTIONAL) AI / data-collection bots. Editorial/legal decision:
# uncomment to block them and further reduce hits.
# =========================================================================
# User-agent: GPTBot
# Disallow: /
#
# User-agent: ClaudeBot
# Disallow: /
#
# User-agent: CCBot
# Disallow: /
#
# User-agent: Google-Extended
# Disallow: /
#
# User-agent: Bytespider
# Disallow: /

# =========================================================================
# Sitemap: helps crawlers target useful URLs => less wasted crawling.
# =========================================================================
Sitemap: https://www.teads.com/sitemaps.xml