# Apply rules to all web crawlers
User-agent: *
Disallow: /newsletter/email-adresse-bestaetigen/
Disallow: /newsletter/anmeldung-erfolgreich/
Disallow: /401/
Disallow: /search/
Disallow: /404/


# Block specific bad bots
User-agent: 008
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SEMrushBot
Disallow: /

User-agent: sentibot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: SiteExplorer
Disallow: /

User-agent: spbot
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: UbiCrawler
Disallow: /

User-agent: WBSearchBot
Disallow: /

User-agent: YandexBot
Disallow: /

# Allow Ahrefs

User-agent: AhrefsSiteAudit
Allow: /

User-agent: AhrefsBot
Allow: /

# Allow IndexNow
User-agent: IndexNow ZipTie Bot
Allow: /

Sitemap: https://www.steuernsteuern.de/sitemap.xml