diff options
Diffstat (limited to 'themes/CodeIT/layouts/robots.txt')
| -rw-r--r-- | themes/CodeIT/layouts/robots.txt | 64 |
1 files changed, 64 insertions, 0 deletions
diff --git a/themes/CodeIT/layouts/robots.txt b/themes/CodeIT/layouts/robots.txt new file mode 100644 index 0000000..cefec37 --- /dev/null +++ b/themes/CodeIT/layouts/robots.txt @@ -0,0 +1,64 @@ +User-agent: * + +Disallow: /images/ +Disallow: /js/ +Disallow: /css/ + +User-agent: MJ12bot +Disallow: / + +User-agent: AhrefsBot +Disallow: / + +User-agent: BLEXBot +Disallow: / + +# Block SISTRIX +User-agent: SISTRIX Crawler +Disallow: / +User-agent: sistrix +Disallow: / +User-agent: 007ac9 +Disallow: / +User-agent: 007ac9 Crawler +Disallow: / + +# Block Uptime robot +User-agent: UptimeRobot/2.0 +Disallow: / + +# Block Ezooms Robot +User-agent: Ezooms Robot +Disallow: / + +# Block Perl LWP +User-agent: Perl LWP +Disallow: / + +# Block netEstate NE Crawler (+http://www.website-datenbank.de/) +User-agent: netEstate NE Crawler (+http://www.website-datenbank.de/) +Disallow: / + +# Block WiseGuys Robot +User-agent: WiseGuys Robot +Disallow: / + +# Block Turnitin Robot +User-agent: Turnitin Robot +Disallow: / + +# Block Heritrix +User-agent: Heritrix +Disallow: / + +# Block pricepi +User-agent: pimonster +Disallow: / + +User-agent: SurdotlyBot +Disallow: / + +User-agent: ZoominfoBot +Disallow: / + +Sitemap: {{ "/sitemap.xml" | absLangURL }} |