Add documentation on how to measure model perplexity

Author: countzero

.gitignore

@@ -4,6 +4,3 @@
 
 # Ignore Sublime Text files.
 *.sublime-workspace
-
-# Ignore OpenBLAS directories and files
-vendor/OpenBLAS/*

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.1.0] - 2023-07-03
+
+### Added
+- Add dataset "wikitext-2-raw-v1"
+- Add documentation on how to measure model perplexity
+
 ## [1.0.0] - 2023-06-28
 
 ### Added

README.md

@@ -103,6 +103,20 @@ You can now chat with the model:
     --interactive
 ```
 
+### Measure model perplexity
+
+Execute the following to measure the perplexity of the GGML formatted model:
+
+```PowerShell
+./vendor/llama.cpp/build/bin/Release/perplexity `
+    --model "./vendor/llama.cpp/models/open-llama-7B-open-instruct.ggmlv3.q4_K_M.bin" `
+    --ctx-size 2048 `
+    --n-predict 2048 `
+    --threads 16 `
+    --n-gpu-layers 32 `
+    --file "./vendor/wikitext-2-raw-v1/wikitext-2-raw/wiki.test.raw"
+```
+
 ### Rebuild llama.cpp
 
 Every time there is a new release of [llama.cpp](https://github.com/ggerganov/llama.cpp) you can simply execute the script to automatically rebuild everything:

rebuild_llama.cpp.ps1

@@ -37,6 +37,18 @@ if (-not(Test-Path -Path "./vendor/OpenBLAS/OpenBLAS-${openBLASVersion}-x64.zip"
         -Force
 }
 
+if (-not(Test-Path -Path "./vendor/wikitext-2-raw-v1/wikitext-2-raw-v1.zip")) {
+
+    Invoke-WebRequest `
+        -Uri "https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-raw-v1.zip" `
+        -OutFile "./vendor/wikitext-2-raw-v1/wikitext-2-raw-v1.zip"
+
+    Expand-Archive `
+        -Path "./vendor/wikitext-2-raw-v1/wikitext-2-raw-v1.zip" `
+        -DestinationPath "./vendor/wikitext-2-raw-v1" `
+        -Force
+}
+
 function Resolve-UnixPath {
     Param ([String] $path)
     Write-Output ((Resolve-Path "$path").Path -replace '\\','/')
@@ -67,7 +79,7 @@ if (!(Select-String -Path "./vendor/llama.cpp/CMakeLists.txt" -Pattern $lines[0]
     Set-Content "./vendor/llama.cpp/CMakeLists.txt"
 }
 
-Remove-Item  -Path "./vendor/llama.cpp/build" -Force -Recurse
+Remove-Item -Path "./vendor/llama.cpp/build" -Force -Recurse
 
 New-Item -Path "./vendor/llama.cpp" -Name "build" -ItemType "directory"
 

vendor/llama.cpp

@@ -0,0 +1,3 @@
+# Ignore everything in this directory except this file.
+*
+!.gitignore