From 65a5ca301375d6b0e6a6cced14cd87a12a6a2733 Mon Sep 17 00:00:00 2001 From: adienes <51664769+adienes@users.noreply.github.com> Date: Tue, 13 May 2025 17:17:17 -0400 Subject: [PATCH 1/2] simple_hash faster by lifting length(str) --- src/tokenize.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tokenize.jl b/src/tokenize.jl index 6eb76954..e0d3770e 100644 --- a/src/tokenize.jl +++ b/src/tokenize.jl @@ -1339,7 +1339,8 @@ end function simple_hash(str) ind = 1 h = UInt64(0) - while ind <= length(str) + L = length(str) + while ind <= L h = simple_hash(str[ind], h) ind = nextind(str, ind) end From 542dcd76dc85048204115940e3609e863dc8a2bb Mon Sep 17 00:00:00 2001 From: adienes <51664769+adienes@users.noreply.github.com> Date: Wed, 14 May 2025 12:38:11 -0400 Subject: [PATCH 2/2] Update src/tokenize.jl Co-authored-by: Sebastian Pfitzner --- src/tokenize.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tokenize.jl b/src/tokenize.jl index e0d3770e..0ea9be19 100644 --- a/src/tokenize.jl +++ b/src/tokenize.jl @@ -1339,7 +1339,7 @@ end function simple_hash(str) ind = 1 h = UInt64(0) - L = length(str) + L = min(lastindex(str), MAX_KW_LENGTH) while ind <= L h = simple_hash(str[ind], h) ind = nextind(str, ind)