From 5a32bb5285b59022529293458462043b8d7c26b1 Mon Sep 17 00:00:00 2001
From: Gonen <gonengazit@gmail.com>
Date: Thu, 7 Apr 2022 21:13:20 +0300
Subject: [PATCH 1/2] don't count ; and , as tokens

---
 pico8/lua/lua.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pico8/lua/lua.py b/pico8/lua/lua.py
index cd10be5..b9c3de3 100644
--- a/pico8/lua/lua.py
+++ b/pico8/lua/lua.py
@@ -351,6 +351,8 @@ def get_token_count(self):
                     t.matches(lexer.TokSymbol(b')')) or
                     t.matches(lexer.TokSymbol(b']')) or
                     t.matches(lexer.TokSymbol(b'}')) or
+                    t.matches(lexer.TokSymbol(b',')) or
+                    t.matches(lexer.TokSymbol(b';')) or
                     t.matches(lexer.TokKeyword(b'local')) or
                     t.matches(lexer.TokKeyword(b'end'))):
                 # PICO-8 generously does not count these as tokens.

From 0a3e715b96fd3cf9a03de7a6dc5a79b0698fe10f Mon Sep 17 00:00:00 2001
From: Gonen <gonengazit@gmail.com>
Date: Fri, 8 Apr 2022 01:03:16 +0300
Subject: [PATCH 2/2] don't token count unary ~ and - for constants

---
 pico8/lua/lua.py | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/pico8/lua/lua.py b/pico8/lua/lua.py
index b9c3de3..9947d2c 100644
--- a/pico8/lua/lua.py
+++ b/pico8/lua/lua.py
@@ -344,8 +344,19 @@ def get_char_count(self):
 
     def get_token_count(self):
         c = 0
-        for t in self._lexer._tokens:
+        prev_op=False #whether previous non whitespace token was an operator (for unary -)
+        unary_minus_ops=parser.BINOP_PATS + parser.UNOP_PATS+tuple(lexer.TokSymbol(x) for x in
+          (b'&=', b'|=', b'^^=', b'<<=', b'>>=', b'>>>=', b'<<>=', b'>><=', b'\\=',
+          b'..=', b'+=', b'-=', b'*=', b'/=', b'%=', b'^=',
+          b'(', b',', b'{', b'[' ,b'=') ) # these ops are not 100% accurate to how pico8 does it (pico8's list is slightly different)
+          #but all the edge cases left are pretty niche
+        for i,t in enumerate(self._lexer._tokens):
             # TODO: As of 0.1.8, "1 .. 5" is three tokens, "1..5" is one token
+            if (isinstance(t, lexer.TokSpace) or
+                  isinstance(t, lexer.TokNewline) or
+                  isinstance(t, lexer.TokComment)):
+              continue
+
             if (t.matches(lexer.TokSymbol(b':')) or
                     t.matches(lexer.TokSymbol(b'.')) or
                     t.matches(lexer.TokSymbol(b')')) or
@@ -357,13 +368,25 @@ def get_token_count(self):
                     t.matches(lexer.TokKeyword(b'end'))):
                 # PICO-8 generously does not count these as tokens.
                 pass
+            elif ((t.matches(lexer.TokSymbol(b'-')) or t.matches(lexer.TokSymbol(b'~'))) and
+                i+1<len(self._lexer._tokens) and
+                self._lexer._tokens[i+1].matches(lexer.TokNumber) and
+                prev_op):
+                #unary - and ~
+                pass
+
+
+
             elif t.matches(lexer.TokNumber) and t._data.find(b'e') != -1:
                 # PICO-8 counts 'e' part of number as a separate token.
                 c += 2
-            elif (not isinstance(t, lexer.TokSpace) and
-                  not isinstance(t, lexer.TokNewline) and
-                  not isinstance(t, lexer.TokComment)):
+            else:
                 c += 1
+            prev_op=False
+            for op in unary_minus_ops:
+                if t.matches(op):
+                    prev_op=True
+
         return c
 
     def get_line_count(self):