Skip to content

Commit 95485f7

Browse files
faisalmanfaisalman
committed
[extensions][helpers] Add new bots: cohere-training-data-crawler, Gemini-Deep-Research, kakaotalk-scrap, TikTokSpider
1 parent 74ef71c commit 95485f7

File tree

4 files changed

+50
-6
lines changed

4 files changed

+50
-6
lines changed

‎src/extensions/ua-parser-extensions.js‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ const Crawlers = Object.freeze({
5858
// OpenAI's SearchGPT - https://platform.openai.com/docs/bots
5959
// PerplexityBot - https://perplexity.ai/perplexitybot
6060
// SeznamBot - http://napoveda.seznam.cz/seznambot-intro
61-
/((?:adidx|ahrefs|amazon|bing|cc|coveo|criteo|dot|duckduck|exa|facebook|gpt|iask|linkedin|mj12|mojeek|oai-search|onespot-scraper|perplexity|semrush|seznam)bot)\/([\w\.-]+)/i,
61+
/((?:adidx|ahrefs|amazon|bing|cc|coveo|criteo|dot|duckduck(?:go-favicons-)?|exa|facebook|gpt|iask|linkedin|mj12|mojeek|oai-search|onespot-scraper|perplexity|semrush|seznam)bot)\/([\w\.-]+)/i,
6262

6363
// Applebot - http://apple.com/go/applebot
6464
/(applebot(?:-extended)?)\/?([\w\.]*)/i,
@@ -116,12 +116,13 @@ const Crawlers = Object.freeze({
116116
// AI2Bot - https://allenai.org/crawler
117117
// Bytespider
118118
// DataForSeoBot - https://dataforseo.com/dataforseo-bot
119+
// DeepSeekBot
119120
// Huawei AspiegelBot / PetalBot https://aspiegel.com/petalbot
120121
// ImagesiftBot - https://imagesift.com/about
121122
// Qihoo 360Spider
122123
// TurnitinBot - https://www.turnitin.com/robot/crawlerinfo.html
123124
// Yahoo! Slurp - http://help.yahoo.com/help/us/ysearch/slurp
124-
/\b(360spider-?(?:image|video)?|bytespider|(?:ai2|aspiegel|dataforseo|imagesift|petal|turnitin)bot|teoma|yahoo! slurp)/i
125+
/\b(360spider-?(?:image|video)?|bytespider|cohere-training-data-crawler|elastic(?=\/s)|(?:ai2|aspiegel|dataforseo|deepseek|imagesift|petal|turnitin)bot|teoma|yahoo! slurp)/i
125126
],
126127
[NAME, [TYPE, CRAWLER]]
127128
]
@@ -236,17 +237,17 @@ const Emails = Object.freeze({
236237
const Fetchers = Object.freeze({
237238
browser : [
238239
[
240+
// Asana / Bitlybot / Better Uptime / BingPreview / Blueno / kakaotalk-scrap / Mastodon / MicrosoftPreview / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot
239241
// AhrefsSiteAudit - https://ahrefs.com/robot/site-audit
240242
// Buffer Link Preview Bot - https://scraper.buffer.com/about/bots/link-preview-bot
241243
// ChatGPT-User - https://platform.openai.com/docs/plugins/bot
242244
// DuckAssistBot - https://duckduckgo.com/duckassistbot/
243-
// Better Uptime / BingPreview / Blueno / Mastodon / MicrosoftPreview / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot
244245
// Google Site Verifier / Meta / Yahoo! Japan
245246
// Iframely - https://iframely.com/docs/about
246247
// Perplexity-User - https://docs.perplexity.ai/guides/bots
247248
// MistralAI-User - https://docs.mistral.ai/robots/
248249
// Yandex Bots - https://yandex.com/bots
249-
/(ahrefssiteaudit|(?:bing|microsoft)preview|blueno|(?:chatgpt|claude|mistralai|perplexity)-user|mastodon|(?:bufferlinkpreview|discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptimero)bot|google-site-verification|iframely|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i,
250+
/(asana|ahrefssiteaudit|(?:bing|microsoft)preview|blueno|(?:chatgpt|claude|mistralai|perplexity)-user|mastodon|(?:bitly|bufferlinkpreview|discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptimero|zoom)bot|google-site-verification|iframely|kakaotalk-scrap|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i,
250251

251252
// Bluesky
252253
/(bluesky) cardyb\/([\w\.]+)/i,
@@ -263,8 +264,8 @@ const Fetchers = Object.freeze({
263264
[NAME, VERSION, [TYPE, FETCHER]],
264265

265266
[
266-
// Google Bots / Cohere / Snapchat / Vercelbot / Yandex Bots
267-
/((?:better uptime |telegram|vercel)bot|cohere-ai|feedfetcher-google|google(?:imageproxy|-read-aloud|-pagerenderer|producer)|snap url preview|yandex(?:sitelinks|userproxy))/i
267+
// Google Bots / Chrome-Lighthouse / Cohere / Gemini-Deep-Research / Snapchat / TikTokSpider / Vercelbot / Yandex Bots
268+
/((?:better uptime |telegram|vercel)bot|chrome-lighthouse|cohere-ai|feedfetcher-google|gemini-deep-research|google(?:imageproxy|-read-aloud|-pagerenderer|producer)|snap url preview|tiktokspider|yandex(?:sitelinks|userproxy))/i
268269
],
269270
[NAME, [TYPE, FETCHER]],
270271
],

‎src/helpers/ua-parser-helpers.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ const isAIBot = (resultOrUA) => [
6262
// ByteDance
6363
'bytespider',
6464

65+
// Cohere
66+
'cohere-training-data-crawler',
67+
6568
// Common Crawl
6669
'ccbot',
6770

‎test/data/ua/extension/crawler.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,16 @@
299299
"type" : "crawler"
300300
}
301301
},
302+
{
303+
"desc" : "cohere-training-data-crawler",
304+
"ua" : "cohere-training-data-crawler (+crawler@cohere.ai)",
305+
"expect" :
306+
{
307+
"name" : "cohere-training-data-crawler",
308+
"version" : "undefined",
309+
"type" : "crawler"
310+
}
311+
},
302312
{
303313
"desc" : "Coveobot",
304314
"ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) (compatible; Coveobot/2.0;+http://www.coveo.com/bot.html)",

‎test/data/ua/extension/fetcher.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,16 @@
109109
"type" : "fetcher"
110110
}
111111
},
112+
{
113+
"desc" : "Gemini-Deep-Research",
114+
"ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Gemini-Deep-Research; +https://gemini.google/overview/deep-research/) Chrome/135.0.0.0 Safari/537.36",
115+
"expect" :
116+
{
117+
"name" : "Gemini-Deep-Research",
118+
"version" : "undefined",
119+
"type" : "fetcher"
120+
}
121+
},
112122
{
113123
"desc" : "Google FeedFetcher",
114124
"ua" : "FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)",
@@ -189,6 +199,16 @@
189199
"type" : "fetcher"
190200
}
191201
},
202+
{
203+
"desc" : "kakaotalk-scrap",
204+
"ua" : "facebookexternalhit/1.1; kakaotalk-scrap/1.0; +https://devtalk.kakao.com/t/scrap/33984",
205+
"expect" :
206+
{
207+
"name" : "kakaotalk-scrap",
208+
"version" : "1.0",
209+
"type" : "fetcher"
210+
}
211+
},
192212
{
193213
"desc" : "Meta-ExternalFetcher",
194214
"ua" : "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
@@ -289,6 +309,16 @@
289309
"type" : "fetcher"
290310
}
291311
},
312+
{
313+
"desc" : "TikTokSpider",
314+
"ua" : "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; TikTokSpider; ttspider-feedback@tiktok.com)",
315+
"expect" :
316+
{
317+
"name" : "TikTokSpider",
318+
"version" : "undefined",
319+
"type" : "fetcher"
320+
}
321+
},
292322
{
293323
"desc" : "UptimeRobot",
294324
"ua" : "Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)",

0 commit comments

Comments
 (0)