klogproc/bots.default.json at main · czcorpus/klogproc · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
{
    "bots": [
        {
            "title": "Ahrefs Backlink Research Bot",
            "match": ["ahrefsbot/"],
            "example": "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)"
        },
        {
            "title": "Applebot",
            "match": ["applebot/"],
            "example": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)"
        },
        {
            "title": "Baidu Spider",
            "match": ["baiduspider/"],
            "example": "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
        },
        {
            "title": "BingBot",
            "match": ["bingbot/"],
            "example": "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"
        },
        {
            "title": "BlexBot",
            "match": ["blexbot/"],
            "example": "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)"
        },
        {
            "title": "DotNetDotComDotOrg Crawler",
            "match": ["dotbot/"],
            "example": "Mozilla/5.0 (compatible; DotBot/1.1; http://www.dotnetdotcom.org/, crawler@dotnetdotcom.org)"
        },
        {
            "title": "DuckDuckBot (old)",
            "match": ["duckduckbot"],
            "example": null
        },
        {
            "title": "DuckDuckBot/1.1",
            "match": ["duckduckgo-favicons-bot/"],
            "example": "Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)"
        },
        {
            "title": "Exabot",
            "match": ["exabot/"],
            "example": "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)"
        },
        {
            "title": "Googlebot",
            "match": ["googlebot/"],
            "example": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
        },
        {
            "title": "Alexa crawler",
            "match": ["ia_archiver"],
            "example": "ia_archiver"
        },
        {
            "title": "Mail.Ru Bot",
            "match": ["mail.ru_bot/"],
            "example": "Mozilla/5.0 (compatible; Mail.RU/2.0); Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)"
        },
        {
            "title": "Mauibot",
            "match": ["mauibot "],
            "example": "MauiBot (crawler.feedback+wc@gmail.com)"
        },
        {
            "title": "Mediatoolkitbot",
            "match": ["mediatoolkitbot "],
            "example": "Mediatoolkitbot (complaints@mediatoolkit.com)"
        },
        {
            "title": "MegaIndex.ru",
            "match": ["megaindex.ru/"],
            "example": "Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)"
        },
        {
            "title": "MJ12bot/v1.3.3",
            "match": ["mj12bot"],
            "example": "Mozilla/5.0 (compatible; MJ12bot/v1.3.3; http://www.majestic12.co.uk/bot.php?+)"
        },
        {
            "title": "Semantic Scholar Bot",
            "match": ["semanticscholarbot "],
            "example": "Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)"
        },
        {
            "title": "SEMrushBot",
            "match": ["semrushbot-"],
            "example": "SemrushBot-SA"
        },
        {
            "title": "SEO Kicks Robot",
            "match": ["seokicks-robot"],
            "example": "Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)"
        },
        {
            "title": "Seznam Bot",
            "match": ["seznambot/"],
            "example": "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)"
        },
        {
            "title": "YaCy-Bot",
            "match": ["yacybot "],
            "example": "yacybot (-global; amd64 Linux 3.16.0-4-amd64; java 1.7.0_121; Europe/de) http://yacy.net/bot.html"
        },
        {
            "title": "Yahoo! Slurp",
            "match": ["yahoo", "slurp"],
            "example": "Mozilla/5.0 (compatible; Yahoo!-AdCrawler; http://help.yahoo.com/yahoo_adcrawler); Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
        }

    ],
    "monitors": [
        {
            "title": "CNC python 2.x scripts",
            "match": ["python-urllib/2.7"],
            "example": null
        },
        {
            "title": "CNC zabbix",
            "match": ["zabbix-test"],
            "example": null
        },
        {
            "title": "Influx Telegraf",
            "match": ["telegraf"],
            "example": null
        },
        {
            "title": "Default Go http client",
            "match": ["Go-http-client/"],
            "example": "Go-http-client/1.1"
        }
    ]
}