User-agent: *
Allow: /
User-agent: Adsbot-Google
Allow: /
User-agent: Mediapartners-Google
Allow: /
User-agent: Baiduspider
Disallow: /
User-agent: R6_CommentReader
Disallow: /
User-agent: SISTRIX Crawler
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: FlipboardProxy
Disallow: /
User-agent: Q_BlogBot
Disallow: /
User-agent: QT_BlogBot
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: NKBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: MegaIndex.ru
Disallow: /
# for Google GDN (Not Crawl)
User-agent: Mediapartners-Google
Allow: /