loadkpi · loadkpi · Nov 4, 2024 · Oct 6, 2024 · Oct 6, 2024 · Oct 6, 2024
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,6 +1,6 @@
 name: build
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   lint:
@@ -9,7 +9,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '3.0'
+          ruby-version: ruby
           bundler-cache: true
       - run: bundle exec rubocop
   test:
@@ -30,3 +30,19 @@ jobs:
       - name: test
         run: |
           bundle exec parallel_rspec spec/ -n 4 --only-group ${{ matrix.test-group }} --group-by runtime --runtime-log spec/fixtures/parallel_runtime_rspec.log
+  push:
+    # Run on push to master branch
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    name: Push gem to RubyGems.org
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ruby
+          bundler-cache: true
+      - uses: rubygems/release-gem@v1
diff --git a/README.md b/README.md
@@ -11,7 +11,7 @@ Comparing with other popular bot-detection gems:
 |  | CrawlerDetect | Voight-Kampff | Browser  |
 |--|--|--|--|
 | Number of bot-patterns | >1000 | ~280 | ~280 |
-| Number of checked HTTP-headers | 10 | 1 | 1 |
+| Number of checked HTTP-headers | 11 | 1 | 1 |
 | Number of updates of bot-list *(1st half of 2018)* | 14 | 1 | 7 |
 
 In order to remain up-to-date, this gem does not accept any crawler data updates – any PRs to edit the crawler data should be offered to the original  [JayBizzle/CrawlerDetect](https://github.com/JayBizzle/Crawler-Detect) project.

diff --git a/bin/update_raw_files b/bin/update_raw_files
@@ -6,5 +6,5 @@ wget -O $DIR/../lib/crawler_detect/library/raw/Crawlers.json https://raw.githubu
 wget -O $DIR/../lib/crawler_detect/library/raw/Exclusions.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Exclusions.json
 wget -O $DIR/../lib/crawler_detect/library/raw/Headers.json https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Headers.json
 
-wget -O $DIR/../spec/fixtures/crawlers.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/crawlers.txt
-wget -O $DIR/../spec/fixtures/devices.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/devices.txt
+wget -O $DIR/../spec/fixtures/crawlers.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/data/user_agent/crawlers.txt
+wget -O $DIR/../spec/fixtures/devices.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/data/user_agent/devices.txt
diff --git a/lib/crawler_detect/library/raw/Crawlers.json b/lib/crawler_detect/library/raw/Crawlers.json
diff --git a/lib/crawler_detect/library/raw/Exclusions.json b/lib/crawler_detect/library/raw/Exclusions.json
@@ -1 +1 @@
-["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+"]
+["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+","\"Yandex\"","YandexModule2"]
diff --git a/lib/crawler_detect/library/raw/Headers.json b/lib/crawler_detect/library/raw/Headers.json
@@ -1 +1 @@
-["HTTP_USER_AGENT","HTTP_X_OPERAMINI_PHONE_UA","HTTP_X_DEVICE_USER_AGENT","HTTP_X_ORIGINAL_USER_AGENT","HTTP_X_SKYFIRE_PHONE","HTTP_X_BOLT_PHONE_UA","HTTP_DEVICE_STOCK_UA","HTTP_X_UCBROWSER_DEVICE_UA","HTTP_FROM","HTTP_X_SCANNER"]
+["HTTP_USER_AGENT","HTTP_X_OPERAMINI_PHONE_UA","HTTP_X_DEVICE_USER_AGENT","HTTP_X_ORIGINAL_USER_AGENT","HTTP_X_SKYFIRE_PHONE","HTTP_X_BOLT_PHONE_UA","HTTP_DEVICE_STOCK_UA","HTTP_X_UCBROWSER_DEVICE_UA","HTTP_FROM","HTTP_X_SCANNER","HTTP_SEC_CH_UA"]
diff --git a/spec/fixtures/crawlers.txt b/spec/fixtures/crawlers.txt
@@ -1116,7 +1116,6 @@ ShowyouBot (http://showyou.com/crawler)
 Yoleo Consumer v0.2
 Mozilla/5.0 (compatible; IstellaBot/1.10.2 +http://www.tiscali.it/)
 Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server; +http://ws.daum.net/aboutWebSearch.html) Daumoa/3.0
-Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 YaBrowser/15.12.0.6151 Safari/537.36 Yandex.Translate
 Mozilla/5.0 (compatible; TeeRaidBot; +https://teeraid.com/bot/)
 Mozilla/5.0 (compatible; Scopia Crawler 1.1; +http://www.scopia.co)
 CopperEgg/RevealUptime/DublinIE(aws)
@@ -2874,6 +2873,8 @@ COMODO DCV
 CATExplorador/1.0beta (sistemes at domini dot cat; http://domini.cat/catexplorador.html)
 ZmEu
 RSSOwl/2.2.1.201312301314 (Windows; U; en)
+RuxitSynthetic/
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.99 Safari/537.36 RuxitSynthetic/1.0 v119975163693 t1975984516666483988 ath87850d94 altpub cvcv=2 smf=0
 WebClient/1.0
 Microsoft-WebDAV-MiniRedir/10.0.10586
 Microsoft-WebDAV-MiniRedir/10.0.14393
@@ -3669,3 +3670,9 @@ Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MicrosoftPreview/
 Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GeedoProductSearch; +http://www.geedo.com/product-search.html) Chrome/79.0.3945.88 Safari/537.36
 Mozilla/5.0 (compatible; +probely.com/sos) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36 ProbelySPDR/0.2.0
 facebookcatalog/1.0
+Verity/1.1 (gumgum.com/verity; verity-support@gumgum.com)
+Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GoogleOther) Chrome/117.0.5938.132 Safari/537.36
+Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Mobile Safari/537.36 (compatible; GoogleOther)
+Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.94 Mobile Safari/537.36 (compatible; GoogleOther)
+Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)
diff --git a/spec/fixtures/devices.txt b/spec/fixtures/devices.txt
@@ -165632,4 +165632,6 @@ Mozilla/5.0 (iPhone; CPU iPhone OS 13_3_1 like Mac OS X) AppleWebKit/605.1.15 (K
 Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148/mvmkbtaqqdhshwcbsygjnuqcqmpndbrt
 Mozilla/5.0 (Linux; Android 5.0; SM-N9100 Build/LRX21V) > AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 > Chrome/37.0.0.0 Mobile Safari/537.36 > MicroMessenger/6.0.2.56_r958800.520 NetType/WIFI
 Mozilla/5.0 (Linux; U; Android 5.0; en-US; SM-N900S;iPhone: Build/LRX21V)
-Mozilla/5.0 (Linux; Android 9; OCTOPUS-CORE T8 Build/PPR1.181005.003) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36 YaApp_Android/9.50/apad YaSearchBrowser/9.50
+Mozilla/5.0 (Linux; Android 9; OCTOPUS-CORE T8 Build/PPR1.181005.003) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36 YaApp_Android/9.50/apad YaSearchBrowser/9.50
+Mozilla/5.0 (Linux; Android 9; YandexModule2-00001 Build/PI; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/113.0.5672.163 YaBrowser/24.1.2.86 (lite) TV Safari/537.36
+Mozilla/5.0 (Linux; Android 9; YandexModule2-00001 Build/PI; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/113.0.5672.163 Mobile Safari/537.36
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		["Safari.[\\d\\.]","Firefox.[\\d\\.]"," Chrome.[\\d\\.]","Chromium.[\\d\\.]","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]","Mozilla.[\\d\\.]","AppleWebKit.[\\d\\.]","Trident.[\\d\\.]","Windows NT.[\\d\\.]","Android [\\d\\.]","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]","(like )?Gecko(.[\\d\\.])?","KHTML,","CriOS.[\\d\\.]","CPU iPhone OS ([0-9_]) like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]","Version.[\\d\\.]","WOW64","Win64","Dalvik.[\\d\\.]"," \\.NET CLR [\\d\\.]","Presto.[\\d\\.]","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+"]
		["Safari.[\\d\\.]","Firefox.[\\d\\.]"," Chrome.[\\d\\.]","Chromium.[\\d\\.]","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]","Mozilla.[\\d\\.]","AppleWebKit.[\\d\\.]","Trident.[\\d\\.]","Windows NT.[\\d\\.]","Android [\\d\\.]","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]","(like )?Gecko(.[\\d\\.])?","KHTML,","CriOS.[\\d\\.]","CPU iPhone OS ([0-9_]) like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]","Version.[\\d\\.]","WOW64","Win64","Dalvik.[\\d\\.]"," \\.NET CLR [\\d\\.]","Presto.[\\d\\.]","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+","\"Yandex\"","YandexModule2"]
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		["HTTP_USER_AGENT","HTTP_X_OPERAMINI_PHONE_UA","HTTP_X_DEVICE_USER_AGENT","HTTP_X_ORIGINAL_USER_AGENT","HTTP_X_SKYFIRE_PHONE","HTTP_X_BOLT_PHONE_UA","HTTP_DEVICE_STOCK_UA","HTTP_X_UCBROWSER_DEVICE_UA","HTTP_FROM","HTTP_X_SCANNER"]
		["HTTP_USER_AGENT","HTTP_X_OPERAMINI_PHONE_UA","HTTP_X_DEVICE_USER_AGENT","HTTP_X_ORIGINAL_USER_AGENT","HTTP_X_SKYFIRE_PHONE","HTTP_X_BOLT_PHONE_UA","HTTP_DEVICE_STOCK_UA","HTTP_X_UCBROWSER_DEVICE_UA","HTTP_FROM","HTTP_X_SCANNER","HTTP_SEC_CH_UA"]