New lexer

.gitmodules

@@ -13,3 +13,6 @@
 	path = external/throwing_ptr
 	url = https://github.com/rockdreamer/throwing_ptr
 	ignore = dirty
+[submodule "external/lexertl17"]
+	path = external/lexertl17
+	url = https://github.com/BenHanson/lexertl17.git

lib/CMakeLists.txt

@@ -11,6 +11,22 @@ else ()
     message(STATUS "libpl static library is being created")
 endif ()
 
+add_subdirectory(lexer_gen)
+
+set(PL_STATICLEXERPATH "${CMAKE_CURRENT_SOURCE_DIR}/include/pl/core/generated/lexer_static.hpp")
+
+add_custom_command(
+    OUTPUT ${PL_STATICLEXERPATH}
+    COMMAND $<TARGET_FILE:lexer_gen> ${PL_STATICLEXERPATH}
+    DEPENDS lexer_gen
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMENT "Generating lexertl17 static lexer"
+)
+
+add_custom_target(run_lexer_gen
+    DEPENDS ${PL_STATICLEXERPATH}
+)
+
 add_library(libpl ${LIBRARY_TYPE}
         source/pl/helpers/utils.cpp
 
@@ -52,7 +68,11 @@ add_library(libpl ${LIBRARY_TYPE}
 
         source/pl/core/token.cpp
         source/pl/core/evaluator.cpp
+
         source/pl/core/lexer.cpp
+        $<$<NOT:$<CONFIG:Debug>>:${PL_STATICLEXERPATH}>
+        $<$<CONFIG:Debug>:source/pl/core/lexer_sm.cpp>
+
         source/pl/core/parser.cpp
         source/pl/core/preprocessor.cpp
         source/pl/core/validator.cpp
@@ -102,6 +122,7 @@ endif ()
 
 target_include_directories(libpl PUBLIC include ../external/throwing_ptr/include)
 target_include_directories(libpl_includes INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include ../external/throwing_ptr/include)
+target_include_directories(libpl PRIVATE include ../external/lexertl17/include)
 target_link_libraries(libpl PRIVATE ${FMT_LIBRARIES})
 target_link_libraries(libpl PUBLIC wolv::types wolv::io wolv::utils wolv::hash wolv::containers)
 

lib/include/pl/core/.gitignore

@@ -0,0 +1,2 @@
+# Ignore generated code folder
+generated/

lib/include/pl/core/errors/error.hpp

@@ -130,33 +130,27 @@ namespace pl::core::err {
         std::vector<Location> m_trace;
     };
 
-    class ErrorCollector {
+    class ErrorCollectorExplicitLocation
+    {
     public:
-
-        virtual ~ErrorCollector() = default;
-
-        virtual Location location() = 0;
+        virtual ~ErrorCollectorExplicitLocation() = default;
 
         template <typename... Args>
-        void error(const fmt::format_string<Args...>& fmt, Args&&... args) {
-            this->m_errors.emplace_back(fmt::format(fmt, std::forward<Args>(args)...), location());
+        void error(const Location &location, const fmt::format_string<Args...> &fmt, Args&&... args) {
+            this->m_errors.emplace_back(fmt::format(fmt, std::forward<Args>(args)...), location);
         }
 
-        void error(const std::string &message) {
-            this->m_errors.emplace_back(message, location());
-        }
-
-        void errorDesc(const std::string &message, const std::string &description) {
-            this->m_errors.emplace_back(message, description, location());
+        void errorDesc(const Location &location, const std::string &message, const std::string &description) {
+            this->m_errors.emplace_back(message, description, location);
         }
 
         template<typename... Args>
-        void errorDesc(const fmt::format_string<Args...>& message, const std::string &description, Args&&... args) {
-            this->m_errors.emplace_back(fmt::format(message, std::forward<Args>(args)...), description, location());
+        void errorDesc(const Location &location, const fmt::format_string<Args...>& message, const std::string &description, Args&&... args) {
+            this->m_errors.emplace_back(fmt::format(message, std::forward<Args>(args)...), description, location);
         }
 
-        void error(CompileError& error) {
-            error.getTrace().push_back(location());
+        void error(const Location &location, CompileError& error) {
+            error.getTrace().push_back(location);
             this->m_errors.push_back(std::move(error));
         }
 
@@ -188,8 +182,39 @@ namespace pl::core::err {
         void clear() {
             this->m_errors.clear();
         }
+
     private:
         std::vector<CompileError> m_errors;
     };
 
+    class ErrorCollector : public ErrorCollectorExplicitLocation {
+    public:
+
+        virtual ~ErrorCollector() = default;
+
+        virtual Location location() = 0;
+
+        template <typename... Args>
+        void error(const fmt::format_string<Args...> &fmt, Args&&... args) {
+            this->ErrorCollectorExplicitLocation::error(location(), fmt, std::forward<Args>(args)...);
+        }
+
+        void error(const std::string &message) {
+            this->errorAt(location(), message);
+        }
+
+        void errorDesc(const std::string &message, const std::string &description) {
+            this->ErrorCollectorExplicitLocation::errorDesc(location(), message, description);
+        }
+
+        template<typename... Args>
+        void errorDesc(const fmt::format_string<Args...>& message, const std::string &description, Args&&... args) {
+            this->ErrorCollectorExplicitLocation::errorDesc(location(), message, description, std::forward<Args>(args)...);
+        }
+
+        void error(CompileError& error) {
+            this->ErrorCollectorExplicitLocation::error(location(), error);
+        }
+    };
+
 }

lib/include/pl/core/lexer.hpp

@@ -1,72 +1,34 @@
+// "Lexer.hpp"
 #pragma once
 
-#include <pl/helpers/types.hpp>
-#include <pl/core/errors/error.hpp>
-
-#include <pl/core/token.hpp>
-
-#include <fmt/core.h>
-
+#include <cstddef>
 #include <optional>
-#include <string>
 #include <vector>
-
+#include <pl/core/token.hpp>
+#include <pl/core/errors/error.hpp>
 #include <pl/core/errors/result.hpp>
+// Debugging
+#include <string>
 
 namespace pl::core {
 
-    class Lexer : err::ErrorCollector {
+    class Lexer : err::ErrorCollectorExplicitLocation {
     public:
-        Lexer() = default;
+        Lexer();
+
+        void reset();
 
         hlp::CompileResult<std::vector<Token>> lex(const api::Source *source);
         size_t getLongestLineLength() const { return m_longestLineLength; }
-        void reset();
 
     private:
-        [[nodiscard]] char peek(size_t p = 1) const;
-        bool processToken(auto parserFunction, const std::string_view& identifier);
-        Location location() override;
-
-        std::optional<char> parseCharacter();
-        std::optional<Token> parseOperator();
-        std::optional<Token> parseSeparator();
-        std::optional<Token> parseOneLineComment();
-        std::optional<Token> parseOneLineDocComment();
-        std::optional<Token> parseMultiLineComment();
-        std::optional<Token> parseMultiLineDocComment();
-        std::optional<Token> parseKeyword(const std::string_view &identifier);
-        std::optional<Token> parseType(const std::string_view &identifier);
-        std::optional<Token> parseDirectiveName(const std::string_view &identifier);
-        std::optional<Token> parseNamedOperator(const std::string_view &identifier);
-        std::optional<Token> parseConstant(const std::string_view &identifier);
-        std::optional<Token> parseStringLiteral();
-        std::optional<Token> parseDirectiveArgument();
-        std::optional<Token> parseDirectiveValue();
-        std::optional<Token::Literal> parseIntegerLiteral(std::string_view literal);
+        std::optional<Token::Literal> parseInteger(std::string_view literal, const auto &location);
+        std::optional<double> parseFloatingPoint(std::string_view literal, const char suffix, const auto &location);
+        std::optional<char> parseCharacter(const char* &pchar, const char* e, const auto &location);
+        std::optional<Token> parseStringLiteral(std::string_view literal, const auto &location);
 
-        std::optional<double> parseFloatingPoint(std::string_view literal, char suffix);
-        std::optional<u128> parseInteger(std::string_view literal);
-
-        Token makeToken(const Token& token, size_t length = 1);
-        static Token makeTokenAt(const Token& token, Location& location, size_t length = 1);
-        void addToken(const Token& token);
-        bool hasTheLineEnded(const char &ch) {
-            if(ch == '\n') {
-                m_longestLineLength = std::max(m_longestLineLength, m_cursor - m_lineBegin);
-                m_line++;
-                m_lineBegin = m_cursor;
-                return true;
-            }
-            return false;
-        }
-        std::string m_sourceCode;
-        const api::Source* m_source = nullptr;
         std::vector<Token> m_tokens;
-        size_t m_cursor = 0;
-        u32 m_line = 0;
-        u32 m_lineBegin = 0;
-        size_t m_longestLineLength = 0;
-        u32 m_errorLength = 0;
+        std::size_t m_longestLineLength = 0;
     };
-}
+
+} // namespace pl::core

lib/include/pl/core/lexer_sm.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <lexertl/state_machine.hpp>
+
+namespace pl::core {
+
+    namespace {
+
+        namespace LexerToken {
+
+            enum {
+                EndOfFile, NewLine, KWNamedOpTypeConstIdent, SingleLineComment,
+                MultiLineCommentOpen,  MultiLineCommentClose, String, Separator,
+                Directive, DirectiveType, DirectiveParam, Operator, Char,
+                Integer, FPNumber
+            };
+
+        }
+    }
+
+    void newLexerBuild(lexertl::state_machine &sm);
+
+}

lib/lexer_gen/CMakeLists.txt

@@ -0,0 +1,12 @@
+set(SOURCES
+    ../source/pl/core/lexer_sm.cpp
+    main.cpp
+)
+
+add_executable(lexer_gen ${SOURCES})
+
+target_include_directories(lexer_gen PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../include
+)
+
+target_include_directories(lexer_gen PRIVATE include ../../external/lexertl17/include)

lib/lexer_gen/main.cpp

@@ -0,0 +1,39 @@
+/*
+Build a "static" lexer in release builds.
+
+Static in the sense that the state machine is built in a pre-build
+step to optimize application start-up time. 
+*/
+#include <pl/core/lexer_sm.hpp>
+
+#include <lexertl/state_machine.hpp>
+#include <lexertl/generate_cpp.hpp>
+
+#include <fstream>
+#include <filesystem>
+
+int main(int argc, char *argv[])
+{
+    if (argc!=2)
+        return 1;
+
+    try
+    {
+        std::filesystem::path argPath(argv[1]);
+        std::filesystem::path genDir = argPath.parent_path();
+        std::filesystem::create_directory(genDir);
+    }
+    catch (const std::filesystem::filesystem_error &fse)
+    {
+        return 1;
+    }
+
+    lexertl::state_machine sm;
+    pl::core::newLexerBuild(sm);
+    sm.minimise();
+
+    std::ofstream ofs(argv[1]);
+    lexertl::table_based_cpp::generate("lookup", sm, false, ofs);
+
+    return 0;
+}