Skip to content

Commit 845a05e

Browse files
committed
Add examples for falcon and extended Llama 2 context
1 parent d2382cc commit 845a05e

File tree

4 files changed

+39
-4
lines changed

4 files changed

+39
-4
lines changed

examples/server_falcon_40b.ps1

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Start-Process "http://127.0.0.1:8080"
2+
3+
../vendor/llama.cpp/build/bin/Release/server `
4+
--model "../vendor/llama.cpp/models/falcon-40b/model-quantized-q4_K_M.gguf" `
5+
--ctx-size 4096 `
6+
--threads 16 `
7+
--n-gpu-layers 10
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
Start-Process "http://127.0.0.1:8080"
2+
3+
# We are increasing the context size of a Llama 2 model from 4096 token
4+
# to 16384 token, which is a ctx_scale of 4.0. The paramters formula is:
5+
#
6+
# --rope-freq-scale = 1 / ctx_scale
7+
# --rope-freq-base = 10000 * ctx_scale
8+
#
9+
../vendor/llama.cpp/build/bin/Release/server `
10+
--model "../vendor/llama.cpp/models/Phind-CodeLlama-34B-v2/model-quantized-q4_K_M.gguf" `
11+
--ctx-size 16384 `
12+
--rope-freq-scale 0.25 `
13+
--rope-freq-base 40000 `
14+
--threads 16 `
15+
--n-gpu-layers 10

rebuild_llama.cpp.ps1

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -127,15 +127,28 @@ Set-Location -Path "./vendor/llama.cpp/build"
127127
switch ($blasAccelerator) {
128128

129129
"OpenBLAS" {
130-
cmake -DLLAMA_BUILD_SERVER=ON -DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS ..
130+
cmake `
131+
-DLLAMA_BUILD_SERVER=ON `
132+
-DLLAMA_DISABLE_LOGS=ON `
133+
-DLLAMA_BLAS=ON `
134+
-DLLAMA_BLAS_VENDOR=OpenBLAS `
135+
..
131136
}
132137

133138
"cuBLAS" {
134-
cmake -DLLAMA_BUILD_SERVER=ON -DLLAMA_CUBLAS=ON ..
139+
cmake `
140+
-DLLAMA_BUILD_SERVER=ON `
141+
-DLLAMA_DISABLE_LOGS=ON `
142+
-DLLAMA_CUBLAS=ON `
143+
..
135144
}
136145

137146
default {
138-
cmake -DLLAMA_BUILD_SERVER=ON ..
147+
cmake `
148+
-DLLAMA_BUILD_SERVER=ON `
149+
-DLLAMA_DISABLE_LOGS=ON `
150+
-DLLAMA_SERVER_VERBOSE=OFF `
151+
..
139152
}
140153
}
141154

vendor/llama.cpp

0 commit comments

Comments
 (0)