Merge pull request #642 from Devsh-Graphics-Programming/nahim_ndt

Nabla Shader Compiler Tool

Author: AnastaZIuk

.gitignore

@@ -29,3 +29,7 @@ android-sample/bin/*
 preprocessed.hlsl
 compiled.spv
 .vs/*
+tools/ndt/__main__.py
+tools/ndt/.vscode/settings.json
+tools/ndt/.vscode/launch.json
+tools/ndt/.profiles/0.json

CMakeLists.txt

@@ -352,10 +352,8 @@ add_subdirectory(${THIRD_PARTY_SOURCE_DIR})
 
 add_subdirectory(src/nbl)
 add_subdirectory("${NBL_PYTHON_MODULE_ROOT_PATH}" tests) # Python Framework
-
-if(NBL_BUILD_EXAMPLES)
-	add_subdirectory(examples_tests)
-endif()
+add_subdirectory(examples_tests)
+add_subdirectory(tools)
 
 if(NBL_BUILD_DOCS)
 	add_subdirectory(docs)

include/nbl/asset/utils/CHLSLCompiler.h

@@ -8,6 +8,8 @@
 #include "nbl/asset/utils/ISPIRVOptimizer.h"
 #include "nbl/asset/utils/IShaderCompiler.h"
 
+
+
 #ifdef _NBL_PLATFORM_WINDOWS_
 
 namespace nbl::asset::impl
@@ -21,6 +23,8 @@ namespace nbl::asset
 class NBL_API2 CHLSLCompiler final : public IShaderCompiler
 {
 	public:
+		
+
 		IShader::E_CONTENT_TYPE getCodeContentType() const override { return IShader::E_CONTENT_TYPE::ECT_HLSL; };
 
 		CHLSLCompiler(core::smart_refctd_ptr<system::ISystem>&& system);
@@ -47,9 +51,17 @@ class NBL_API2 CHLSLCompiler final : public IShaderCompiler
 		//}
 
 		std::string preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions) const override;
-
+		std::string preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, std::vector<std::string>& dxc_compile_flags_override, const SPreprocessorOptions& preprocessOptions) const;
+							
 		void insertIntoStart(std::string& code, std::ostringstream&& ins) const override;
+
+		struct SdxcCompileResult {
+			uint8_t *begin;
+			size_t size;
+		};
+		SdxcCompileResult dxcCompile(std::string& source, LPCWSTR* args, uint32_t argCount, const CHLSLCompiler::SOptions& options) const;
 	protected:
+		
 
 		// This can't be a unique_ptr due to it being an undefined type 
 		// when Nabla is used as a lib

include/nbl/system/ISystem.h

@@ -149,6 +149,8 @@ class NBL_API2 ISystem : public core::IReferenceCounted
                 m_cachedArchiveFiles.removeObject(dummy,pathAlias);
         }
 
+        void unmountBuiltins();
+
         //
         struct SystemInfo
         {

src/nbl/asset/utils/CHLSLCompiler.cpp

@@ -3,23 +3,20 @@
 // For conditions of distribution and use, see copyright notice in nabla.h
 #include "nbl/asset/utils/CHLSLCompiler.h"
 #include "nbl/asset/utils/shadercUtils.h"
-// TODO: review
 #ifdef NBL_EMBED_BUILTIN_RESOURCES
 #include "nbl/builtin/CArchive.h"
 #include "spirv/builtin/CArchive.h"
 #endif // NBL_EMBED_BUILTIN_RESOURCES
 
 #ifdef _NBL_PLATFORM_WINDOWS_
 
-#include <wrl.h>
-#include <combaseapi.h>
-
-#include <dxc/dxcapi.h>
-
-#include <sstream>
 #include <regex>
 #include <iterator>
 #include <codecvt>
+#include <wrl.h>
+#include <combaseapi.h>
+#include <sstream>
+#include <dxc/dxcapi.h>
 
 
 using namespace nbl;
@@ -38,6 +35,19 @@ struct DXC
 };
 }
 
+struct DxcCompilationResult
+{
+    Microsoft::WRL::ComPtr<IDxcBlobEncoding> errorMessages;
+    Microsoft::WRL::ComPtr<IDxcBlob> objectBlob;
+    Microsoft::WRL::ComPtr<IDxcResult> compileResult;
+
+    std::string GetErrorMessagesString()
+    {
+        return std::string(reinterpret_cast<char*>(errorMessages->GetBufferPointer()), errorMessages->GetBufferSize());
+    }
+};
+
+
 CHLSLCompiler::CHLSLCompiler(core::smart_refctd_ptr<system::ISystem>&& system)
     : IShaderCompiler(std::move(system))
 {
@@ -60,20 +70,7 @@ CHLSLCompiler::~CHLSLCompiler()
     delete m_dxcCompilerTypes;
 }
 
-
-struct DxcCompilationResult
-{
-    ComPtr<IDxcBlobEncoding> errorMessages;
-    ComPtr<IDxcBlob> objectBlob;
-    ComPtr<IDxcResult> compileResult;
-
-    std::string GetErrorMessagesString()
-    {
-        return std::string(reinterpret_cast<char*>(errorMessages->GetBufferPointer()), errorMessages->GetBufferSize());
-    }
-};
-
-DxcCompilationResult dxcCompile(const CHLSLCompiler* compiler, nbl::asset::impl::DXC* dxc, std::string& source, LPCWSTR* args, uint32_t argCount, const CHLSLCompiler::SOptions& options)
+CHLSLCompiler::SdxcCompileResult CHLSLCompiler::dxcCompile(std::string& source, LPCWSTR* args, uint32_t argCount, const CHLSLCompiler::SOptions& options) const
 {
     // Append Commandline options into source only if debugInfoFlags will emit source
     auto sourceEmittingFlags =
@@ -83,7 +80,7 @@ DxcCompilationResult dxcCompile(const CHLSLCompiler* compiler, nbl::asset::impl:
     if ((options.debugInfoFlags.value & sourceEmittingFlags) != CHLSLCompiler::E_DEBUG_INFO_FLAGS::EDIF_NONE)
     {
         std::ostringstream insertion;
-        insertion << "//#pragma compile_flags ";
+        insertion << "#pragma wave dxc_compile_flags( ";
 
         std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> conv;
         for (uint32_t arg = 0; arg < argCount; arg ++)
@@ -92,10 +89,10 @@ DxcCompilationResult dxcCompile(const CHLSLCompiler* compiler, nbl::asset::impl:
             insertion << str.c_str() << " ";
         }
 
-        insertion << "\n";
-        compiler->insertIntoStart(source, std::move(insertion));
+        insertion << ")\n";
+        insertIntoStart(source, std::move(insertion));
     }
-    
+    nbl::asset::impl::DXC* dxc = m_dxcCompilerTypes;
     ComPtr<IDxcBlobEncoding> src;
     auto res = dxc->m_dxcUtils->CreateBlob(reinterpret_cast<const void*>(source.data()), source.size(), CP_UTF8, &src);
     assert(SUCCEEDED(res));
@@ -134,7 +131,7 @@ DxcCompilationResult dxcCompile(const CHLSLCompiler* compiler, nbl::asset::impl:
     else
     {
         options.preprocessorOptions.logger.log("DXC Compilation Failed:\n%s", system::ILogger::ELL_ERROR, errorMessagesString.c_str());
-        return result;
+        return { nullptr, 0 };
     }
 
     ComPtr<IDxcBlob> resultingBlob;
@@ -143,18 +140,19 @@ DxcCompilationResult dxcCompile(const CHLSLCompiler* compiler, nbl::asset::impl:
 
     result.objectBlob = resultingBlob;
 
-    return result;
+    return { (uint8_t*)result.objectBlob->GetBufferPointer(), result.objectBlob->GetBufferSize() };
 }
 
 
 #include "nbl/asset/utils/waveContext.h"
 
-std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions) const
+
+std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, std::vector<std::string>& dxc_compile_flags_override, const SPreprocessorOptions& preprocessOptions) const
 {
     nbl::wave::context context(code.begin(),code.end(),preprocessOptions.sourceIdentifier.data(),{preprocessOptions});
     context.add_macro_definition("__HLSL_VERSION");
 
-     // instead of defining extraDefines as "NBL_GLSL_LIMIT_MAX_IMAGE_DIMENSION_1D 32768", 
+    // instead of defining extraDefines as "NBL_GLSL_LIMIT_MAX_IMAGE_DIMENSION_1D 32768", 
     // now define them as "NBL_GLSL_LIMIT_MAX_IMAGE_DIMENSION_1D=32768" 
     // to match boost wave syntax
     // https://www.boost.org/doc/libs/1_82_0/libs/wave/doc/class_reference_context.html#:~:text=Maintain%20defined%20macros-,add_macro_definition,-bool%20add_macro_definition
@@ -192,12 +190,22 @@ std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADE
         }
     }
 
+    if (context.get_hooks().m_dxc_compile_flags_override.size() != 0)
+        dxc_compile_flags_override = context.get_hooks().m_dxc_compile_flags_override;
+
     if(context.get_hooks().m_pragmaStage != IShader::ESS_UNKNOWN)
         stage = context.get_hooks().m_pragmaStage;
 
+
+
     return resolvedString;
 }
 
+std::string CHLSLCompiler::preprocessShader(std::string&& code, IShader::E_SHADER_STAGE& stage, const SPreprocessorOptions& preprocessOptions) const
+{
+    std::vector<std::string> extra_dxc_compile_flags = {};
+    return preprocessShader(std::move(code), stage, extra_dxc_compile_flags, preprocessOptions);
+}
 
 core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV(const char* code, const IShaderCompiler::SCompilerOptions& options) const
 {
@@ -208,9 +216,9 @@ core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV(const char* cod
         hlslOptions.preprocessorOptions.logger.log("code is nullptr", system::ILogger::ELL_ERROR);
         return nullptr;
     }
-
+    std::vector<std::string> dxc_compile_flags = {};
     auto stage = hlslOptions.stage;
-    auto newCode = preprocessShader(code, stage, hlslOptions.preprocessorOptions);
+    auto newCode = preprocessShader(code, stage, dxc_compile_flags, hlslOptions.preprocessorOptions);
 
     // Suffix is the shader model version
     // TODO: Figure out a way to get the shader model version automatically
@@ -256,7 +264,21 @@ core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV(const char* cod
             return nullptr;
     };
 
-    std::vector<LPCWSTR> arguments = {
+    std::wstring* arg_storage = NULL;
+    std::vector<LPCWSTR> arguments;
+
+    if (dxc_compile_flags.size()) { // #pragma wave overrides compile flags
+        size_t arg_size = dxc_compile_flags.size();
+        arguments = {};
+        arguments.reserve(arg_size);
+        arg_storage = new std::wstring[arg_size]; // prevent deallocation before shader compilation
+        for (size_t i = 0; i < dxc_compile_flags.size(); i++) {
+            arg_storage[i] = std::wstring(dxc_compile_flags[i].begin(), dxc_compile_flags[i].end());
+            arguments.push_back(arg_storage[i].c_str());
+        }
+    }
+    else {
+        arguments = {
         L"-spirv",
         L"-HV", L"202x",
         L"-T", targetProfile.c_str(),
@@ -267,47 +289,48 @@ core::smart_refctd_ptr<ICPUShader> CHLSLCompiler::compileToSPIRV(const char* cod
         L"-Wno-c++1z-extensions",
         L"-Wno-gnu-static-float-init",
         L"-fspv-target-env=vulkan1.3"
-    };
+        };
+        // If a custom SPIR-V optimizer is specified, use that instead of DXC's spirv-opt.
+        // This is how we can get more optimizer options.
+        // 
+        // Optimization is also delegated to SPIRV-Tools. Right now there are no difference between 
+        // optimization levels greater than zero; they will all invoke the same optimization recipe. 
+        // https://github.com/Microsoft/DirectXShaderCompiler/blob/main/docs/SPIR-V.rst#optimization
+        if (hlslOptions.spirvOptimizer)
+        {
+            arguments.push_back(L"-O0");
+        }
 
-    // If a custom SPIR-V optimizer is specified, use that instead of DXC's spirv-opt.
-    // This is how we can get more optimizer options.
-    // 
-    // Optimization is also delegated to SPIRV-Tools. Right now there are no difference between 
-    // optimization levels greater than zero; they will all invoke the same optimization recipe. 
-    // https://github.com/Microsoft/DirectXShaderCompiler/blob/main/docs/SPIR-V.rst#optimization
-    if (hlslOptions.spirvOptimizer)
-    {
-        arguments.push_back(L"-O0");
+        // Debug only values
+        if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_FILE_BIT))
+            arguments.push_back(L"-fspv-debug=file");
+        if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_SOURCE_BIT))
+            arguments.push_back(L"-fspv-debug=source");
+        if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_LINE_BIT))
+            arguments.push_back(L"-fspv-debug=line");
+        if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_TOOL_BIT))
+            arguments.push_back(L"-fspv-debug=tool");
+        if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_NON_SEMANTIC_BIT))
+            arguments.push_back(L"-fspv-debug=vulkan-with-source");
     }
 
-    // Debug only values
-    if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_FILE_BIT))
-        arguments.push_back(L"-fspv-debug=file");
-    if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_SOURCE_BIT))
-        arguments.push_back(L"-fspv-debug=source");
-    if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_LINE_BIT))
-        arguments.push_back(L"-fspv-debug=line");
-    if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_TOOL_BIT))
-        arguments.push_back(L"-fspv-debug=tool");
-    if (hlslOptions.debugInfoFlags.hasFlags(E_DEBUG_INFO_FLAGS::EDIF_NON_SEMANTIC_BIT))
-        arguments.push_back(L"-fspv-debug=vulkan-with-source");
-
-    auto compileResult = dxcCompile(
-        this, 
-        m_dxcCompilerTypes, 
+    auto compileResult = dxcCompile( 
         newCode,
-        &arguments[0],
+        arguments.data(),
         arguments.size(),
         hlslOptions
     );
 
-    if (!compileResult.objectBlob)
+    if (arg_storage)
+        delete[] arg_storage;
+
+    if (!compileResult.begin)
     {
         return nullptr;
     }
 
-    auto outSpirv = core::make_smart_refctd_ptr<ICPUBuffer>(compileResult.objectBlob->GetBufferSize());
-    memcpy(outSpirv->getPointer(), compileResult.objectBlob->GetBufferPointer(), compileResult.objectBlob->GetBufferSize());
+    auto outSpirv = core::make_smart_refctd_ptr<ICPUBuffer>(compileResult.size);
+    memcpy(outSpirv->getPointer(), compileResult.begin, compileResult.size);
 
     // Optimizer step
     if (hlslOptions.spirvOptimizer)
@@ -322,4 +345,6 @@ void CHLSLCompiler::insertIntoStart(std::string& code, std::ostringstream&& ins)
 {
     code.insert(0u, ins.str());
 }
+
+
 #endif