diff options
| author | uPagge <git@upagge.ru> | 2021-02-26 11:10:07 +0300 |
|---|---|---|
| committer | uPagge <git@upagge.ru> | 2021-02-26 11:10:07 +0300 |
| commit | d6e8d14c4721bf5f60d221972ec4e7a1eafc6644 (patch) | |
| tree | f038920dd61aca4792ebb50dd445c0dc0e7e9a6e /layouts | |
| parent | 8d8204749533579c58a36234c77c92def73c35d3 (diff) | |
fix robots
Diffstat (limited to 'layouts')
| -rw-r--r-- | layouts/robots.txt | 48 |
1 files changed, 48 insertions, 0 deletions
diff --git a/layouts/robots.txt b/layouts/robots.txt index 622743f..0252a9a 100644 --- a/layouts/robots.txt +++ b/layouts/robots.txt @@ -21,13 +21,61 @@ Disallow: / # Block SISTRIX User-agent: SISTRIX Crawler Disallow: / + User-agent: sistrix Disallow: / + User-agent: 007ac9 Disallow: / + User-agent: 007ac9 Crawler Disallow: / +User-agent: Nutch +Disallow: / + +User-agent: ia_archiver +Disallow: / + +User-agent: WebCopier +Disallow: / + +User-agent: WebStripper +Disallow: / + +User-agent: Teleport +Disallow: / + +User-agent: Offline Explorer +Disallow: / + +User-agent: SiteSnagger +Disallow: / + +User-agent: psbot +Disallow: / + +User-agent: TeleportPro +Disallow: / + +User-agent: EmailSiphon +Disallow: / + +User-agent: EmailCollector +Disallow: / + +User-agent: larbin +Disallow: / + +User-agent: SemrushBot +Disallow: / + +User-agent: MSIECrawler +Disallow: / + +User-agent: moget +Disallow: / + # Block Uptime robot User-agent: UptimeRobot/2.0 Disallow: / |