diff --git a/manul/CMakeLists.txt b/manul/CMakeLists.txt index 66e3d38a..3a866562 100644 --- a/manul/CMakeLists.txt +++ b/manul/CMakeLists.txt @@ -22,6 +22,9 @@ add_subdirectory("thirdparty/fmt") add_subdirectory("thirdparty/entt") add_subdirectory("thirdparty/fsr2") +add_subdirectory("mashadercompiler") +add_subdirectory("shaders") + # Source code file(GLOB_RECURSE src "renderer/*.cpp" @@ -103,3 +106,4 @@ endif () target_sources(${LIBMANUL_NAME} INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/eu07_source/register.cpp") target_link_libraries(${LIBMANUL_NAME} ${LIBMANUL_NAME}_fsr2 nvrhi yaml-cpp fmt EnTT glfw) +add_dependencies(${LIBMANUL_NAME} ${LIBMANUL_NAME}_shaders) diff --git a/manul/mashadercompiler/CMakeLists.txt b/manul/mashadercompiler/CMakeLists.txt new file mode 100644 index 00000000..53ee2af2 --- /dev/null +++ b/manul/mashadercompiler/CMakeLists.txt @@ -0,0 +1,23 @@ +cmake_minimum_required(VERSION 3.24) + +set(CMAKE_CXX_STANDARD 17) + +project(MaShaderCompiler) + +file(GLOB_RECURSE src "src/*.cpp" "src/*.hpp") + +find_package(directx-dxc CONFIG REQUIRED) + +add_executable(MaShaderCompiler ${src}) + +target_link_libraries(MaShaderCompiler nvrhi yaml-cpp Microsoft::DirectXShaderCompiler Microsoft::DXIL) + +if (WIN32) + target_link_libraries(MaShaderCompiler delayimp) + target_link_options(MaShaderCompiler PRIVATE "/DELAYLOAD:dxcompiler.dll") +endif () + +# We later pass the DXIL.dll parent directory as first argument to MaShaderCompiler so that it can sign DX12 shaders +get_target_property(dxil_dll_path Microsoft::DXIL IMPORTED_LOCATION) +get_filename_component(dxil_dll_path ${dxil_dll_path} DIRECTORY) +set_target_properties(MaShaderCompiler PROPERTIES dxil_path "${dxil_dll_path}") diff --git a/manul/mashadercompiler/src/main.cpp b/manul/mashadercompiler/src/main.cpp new file mode 100644 index 00000000..113fbdcd --- /dev/null +++ b/manul/mashadercompiler/src/main.cpp @@ -0,0 +1,18 @@ +#include + +#include "shader_compiler.hpp" + +int main(int argc, const char** argv) { + if (argc != 4) { + return -1; + } +#ifdef WIN32 + std::cout << "DXIL path: " << argv[1] << std::endl; + SetDllDirectoryA(argv[1]); +#endif + setlocale(LC_ALL, ".UTF8"); + MaShaderCompiler compiler{}; + compiler.m_project_path = argv[2]; + compiler.m_output_path = argv[3]; + return compiler.Run(); +} \ No newline at end of file diff --git a/manul/mashadercompiler/src/shader_compiler.cpp b/manul/mashadercompiler/src/shader_compiler.cpp new file mode 100644 index 00000000..3424cf24 --- /dev/null +++ b/manul/mashadercompiler/src/shader_compiler.cpp @@ -0,0 +1,312 @@ +#include "shader_compiler.hpp" + +#include +#include + +#include "utils.hpp" + +int MaShaderCompiler::Run() { + ParseOptions(); + Init(); + m_shader_path = m_project_path.parent_path(); + YAML::Node src = YAML::LoadFile((m_project_path).generic_string()); + + create_directories(m_output_path); + + { + YAML::Node dest; + if (!CompileProject(dest, src["shaders"], src["templates"], + ShaderPlatform::D3D12)) { + return E_FAIL; + } + std::ofstream fout(m_output_path / "shaders_dxil.manul"); + fout << dest; + } + + { + YAML::Node dest; + if (!CompileProject(dest, src["shaders"], src["templates"], + ShaderPlatform::Vulkan)) { + return E_FAIL; + } + std::ofstream fout(m_output_path / "shaders_spirv.manul"); + fout << dest; + } + + { + std::ofstream fout(m_output_path / "project.manul"); + fout << src; + } + + return S_OK; +} + +void MaShaderCompiler::ParseOptions() {} + +void MaShaderCompiler::Init() { + const char *shader_path = getenv("shader_path"); + if (shader_path) { + m_shader_path = shader_path; + std::cout << "shader path: " << m_shader_path.generic_string() << std::endl; + } else { + m_shader_path = ""; + std::cout << "no custom shader path specified" << std::endl; + } + DxcCreateInstance(CLSID_DxcUtils, IID_PPV_ARGS(&m_dxc_utils)); + DxcCreateInstance(CLSID_DxcCompiler, IID_PPV_ARGS(&m_dxc_compiler)); + m_dxc_utils->CreateDefaultIncludeHandler(&m_dxc_include_handler); +} + +YAML::Binary MaShaderCompiler::CompileShaderToBlob( + std::filesystem::path file_name, std::wstring entry_name, + std::vector> defines, + std::string target, ShaderPlatform platform) { + file_name.replace_extension(".hlsl"); + static std::unordered_map targets{ + {"compute", L"cs_6_0"}, {"vertex", L"vs_6_0"}, {"hull", L"hs_6_0"}, + {"domain", L"ds_6_0"}, {"geometry", L"gs_6_0"}, {"pixel", L"ps_6_0"}}; + + const wchar_t *target_str; + if (auto target_found = targets.find(target); target_found != targets.end()) { + target_str = target_found->second.c_str(); + } else { + return {}; + } + + RefCountPtr source_blob; + if (FAILED(m_dxc_utils->LoadFile(file_name.generic_wstring().c_str(), nullptr, + &source_blob))) { + return {}; + } + DxcBuffer Source; + Source.Ptr = source_blob->GetBufferPointer(); + Source.Size = source_blob->GetBufferSize(); + Source.Encoding = DXC_CP_ACP; + + std::vector include_paths{ + m_shader_path.generic_wstring(), + file_name.parent_path().generic_wstring()}; + + std::vector args; + + args.emplace_back(L"-HV"); + args.emplace_back(L"2021"); + args.emplace_back(L"-Zpc"); + + for (const auto &include_path : include_paths) { + args.emplace_back(L"-I"); + args.emplace_back(include_path.c_str()); + } + + // if (m_generate_debug) { + // args.emplace_back(L"-Od"); + // args.emplace_back(L"-Zi"); + // } else { + // //args.emplace_back(L"-O3"); + // } + + args.emplace_back(L"-Zi"); + args.emplace_back(L"-Qembed_debug"); + + std::vector reg_shifts; + + // Gather SPIRV register shifts once + static const wchar_t *regShiftArgs[] = { + L"-fvk-s-shift", + L"-fvk-t-shift", + L"-fvk-b-shift", + L"-fvk-u-shift", + }; + + uint32_t regShifts[] = {128, 0, 256, 384}; + + switch (platform) { + case ShaderPlatform::D3D12: + args.emplace_back(L"-Gis"); + break; + case ShaderPlatform::Vulkan: + args.emplace_back(L"-spirv"); + args.emplace_back(L"-fspv-target-env=vulkan1.2"); + args.emplace_back(L"-fvk-use-dx-layout"); + defines.emplace_back(L"SPIRV", L"1"); + + for (uint32_t reg = 0; reg < 4; reg++) { + for (uint32_t space = 0; space < 8; space++) { + wchar_t buf[64]; + + reg_shifts.emplace_back(regShiftArgs[reg]); + + swprintf(buf, std::size(buf), L"%u", regShifts[reg]); + reg_shifts.emplace_back(buf); + + swprintf(buf, std::size(buf), L"%u", space); + reg_shifts.emplace_back(buf); + } + } + + for (const std::wstring &arg : reg_shifts) { + args.emplace_back(arg.c_str()); + } + break; + } + + std::vector dxc_defines{}; + dxc_defines.reserve(defines.size()); + for (const auto &[name, definition] : defines) { + auto &define = dxc_defines.emplace_back(); + define.Name = name.c_str(); + define.Value = definition.c_str(); + } + + RefCountPtr compiler_args; + m_dxc_utils->BuildArguments(file_name.stem().generic_wstring().c_str(), + entry_name.c_str(), target_str, args.data(), + args.size(), dxc_defines.data(), + dxc_defines.size(), &compiler_args); + + std::vector wargs( + compiler_args->GetArguments(), + compiler_args->GetArguments() + compiler_args->GetCount()); + + RefCountPtr result; + HRESULT hr = m_dxc_compiler->Compile( + &Source, // Source buffer. + compiler_args->GetArguments(), // Array of pointers to arguments. + compiler_args->GetCount(), // Number of arguments. + m_dxc_include_handler, // User-provided interface to handle + // #include + // directives (optional). + IID_PPV_ARGS(&result) // Compiler output status, buffer, and errors. + ); + + RefCountPtr errors; + result->GetOutput(DXC_OUT_ERRORS, IID_PPV_ARGS(&errors), nullptr); + auto errors_length = errors->GetStringLength(); + + if (errors && errors_length) { + std::string message(errors->GetStringPointer()); + std::cout << "Shader compile log: " << file_name.generic_string() + << std::endl; + std::cout << message; + // WriteLog("Shader compile log: " + fileName.generic_string()); + // WriteLog(message); + } + + if (FAILED(hr)) { + std::cout << "Shader compile failed: " << file_name.generic_string() + << std::endl; + return {}; + } + + RefCountPtr binary_blob; + result->GetOutput(DXC_OUT_OBJECT, IID_PPV_ARGS(&binary_blob), nullptr); + + auto buf_start = + static_cast(binary_blob->GetBufferPointer()); + auto buf_end = buf_start + binary_blob->GetBufferSize(); + std::vector buf{buf_start, buf_end}; + YAML::Binary binary{}; + binary.swap(buf); + return binary; +} + +bool MaShaderCompiler::CompileUtilityShader(YAML::Node &dest, + const YAML::Node &src, + const YAML::Node &templates, + ShaderPlatform platform) { + std::vector> definitions{}; + for (const auto definition : + TemplateOverride("definitions", src, templates)) { + auto &[name, define] = definitions.emplace_back(); + name = ToWide(definition.first.as()); + define = ToWide(definition.second.as()); + } + const std::filesystem::path file_name = + m_shader_path / + TemplateOverride("source", src, templates).as(); + const auto entry_name = + TemplateOverride("entrypoint", src, templates).as(); + const auto target = + TemplateOverride("target", src, templates).as(); + const auto blob = CompileShaderToBlob(file_name, ToWide(entry_name), + definitions, target, platform); + + if (!blob.size()) { + return false; + } + + dest["binary"] = blob; + dest["entrypoint"] = entry_name; + return true; +} + +bool MaShaderCompiler::CompileMaterial(YAML::Node &dest, const YAML::Node &src, + const YAML::Node &templates, + ShaderPlatform platform) { + std::vector> definitions{}; + for (const auto definition : + TemplateOverride("definitions", src, templates)) { + auto &[name, define] = definitions.emplace_back(); + name = ToWide(definition.first.as()); + define = ToWide(definition.second.as()); + } + const std::filesystem::path file_name = + m_shader_path / + TemplateOverride("source", src, templates).as(); + const std::string entry_name = "main"; + const std::string target = "pixel"; + + for (int i = 0; i < static_cast(MaterialRenderPass::Count); ++i) { + auto pass = static_cast(i); + const static std::unordered_map pass_names{ + {MaterialRenderPass::Deferred, "deferred"}, + {MaterialRenderPass::Forward, "forward"}, + {MaterialRenderPass::CubeMap, "cubemap"}, + }; + static std::unordered_map + pass_definitions{ + {MaterialRenderPass::Deferred, L"PASS_GBUFFER"}, + {MaterialRenderPass::Forward, L"PASS_FORWARD"}, + {MaterialRenderPass::CubeMap, L"PASS_CUBEMAP"}, + }; + std::vector> local_definitions{}; + local_definitions.emplace_back(L"PASS", pass_definitions.at(pass)); + local_definitions.insert(local_definitions.end(), definitions.begin(), + definitions.end()); + + auto blob = CompileShaderToBlob(file_name, ToWide(entry_name), + local_definitions, target, platform); + + if (!blob.size()) { + return false; + } + + dest[pass_names.at(pass)]["binary"] = blob; + dest[pass_names.at(pass)]["entrypoint"] = entry_name; + } + return true; +} + +bool MaShaderCompiler::CompileProject(YAML::Node &dest, const YAML::Node &src, + const YAML::Node &templates, + ShaderPlatform platform) { + { + YAML::Node dest_materials = dest["materials"]; + for (const auto material : src["materials"]) { + YAML::Node dest_material; + if (!CompileMaterial(dest_material, material.second, templates, platform)) + return false; + dest_materials[material.first] = dest_material; + } + } + { + for (const auto material : src["utility"]) { + YAML::Node dest_shader; + if (!CompileUtilityShader(dest_shader, material.second, templates, + platform)) + return false; + dest["utility"][material.first] = dest_shader; + } + } + return true; +} \ No newline at end of file diff --git a/manul/mashadercompiler/src/shader_compiler.hpp b/manul/mashadercompiler/src/shader_compiler.hpp new file mode 100644 index 00000000..45f00976 --- /dev/null +++ b/manul/mashadercompiler/src/shader_compiler.hpp @@ -0,0 +1,47 @@ +#pragma once + +#include +#include + +#include + +#ifndef _WIN32 +#include +#else +#include +#endif +#include + +template +using RefCountPtr = nvrhi::RefCountPtr; + +enum class MaterialRenderPass : int { Deferred, Forward, CubeMap, Count }; + +enum class ShaderPlatform { D3D12, Vulkan }; + +class MaShaderCompiler { + public: + int Run(); + std::filesystem::path m_project_path; + std::filesystem::path m_output_path; + + private: + std::filesystem::path m_shader_path; + bool m_generate_debug = false; + void ParseOptions(); + void Init(); + YAML::Binary CompileShaderToBlob( + std::filesystem::path file_name, std::wstring entry_name, + std::vector> defines, + std::string target, ShaderPlatform platform); + bool CompileUtilityShader(YAML::Node& dest, const YAML::Node& src, + const YAML::Node& templates, + ShaderPlatform platform); + bool CompileMaterial(YAML::Node& dest, const YAML::Node& src, + const YAML::Node& templates, ShaderPlatform platform); + bool CompileProject(YAML::Node& dest, const YAML::Node& src, + const YAML::Node& templates, ShaderPlatform platform); + RefCountPtr m_dxc_utils; + RefCountPtr m_dxc_compiler; + RefCountPtr m_dxc_include_handler; +}; diff --git a/manul/mashadercompiler/src/utils.hpp b/manul/mashadercompiler/src/utils.hpp new file mode 100644 index 00000000..0e647d32 --- /dev/null +++ b/manul/mashadercompiler/src/utils.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include + +inline std::wstring ToWide(const std::string& str) { + std::wstring wstr{}; + wstr.resize(std::mbstowcs(nullptr, str.c_str(), static_cast(-1))); + std::mbstowcs(wstr.data(), str.c_str(), wstr.size()); + return wstr; +} + +inline std::string ToNarrow(const std::wstring& wstr) { + std::string str{}; + str.resize(std::wcstombs(nullptr, wstr.c_str(), static_cast(-1))); + std::wcstombs(str.data(), wstr.c_str(), str.size()); + return str; +} + +template +YAML::Node TemplateOverride(const KeyType& key, const YAML::Node& container, + const YAML::Node& templates) { + YAML::Node local = container[key]; + YAML::Node use_template = container["use_template"]; + if (!local.IsDefined() && use_template.IsDefined() && + templates[use_template.as()].IsDefined()) { + return templates[use_template.as()][key]; + } + return local; +} diff --git a/manul/shaders/.gitignore b/manul/shaders/.gitignore index ce59f358..4caba348 100644 --- a/manul/shaders/.gitignore +++ b/manul/shaders/.gitignore @@ -2,4 +2,5 @@ !*.hlsl* !*.h -!project.manul \ No newline at end of file +!project.manul +!CMakeLists.txt \ No newline at end of file diff --git a/manul/shaders/CMakeLists.txt b/manul/shaders/CMakeLists.txt new file mode 100644 index 00000000..50b79f88 --- /dev/null +++ b/manul/shaders/CMakeLists.txt @@ -0,0 +1,10 @@ + +file(GLOB_RECURSE src "*.hlsl*" "*.h") +list(APPEND src "project.manul") + +get_target_property(dxil_path MaShaderCompiler dxil_path) + +add_custom_target( + ${LIBMANUL_NAME}_shaders + MaShaderCompiler "${dxil_path}" "${CMAKE_CURRENT_SOURCE_DIR}/project.manul" "$/shaders" + DEPENDS ${src})