diff --git a/rendering/vulkan14/vulkan14commands.cpp b/rendering/vulkan14/vulkan14commands.cpp index 59a4d2ba..b4800667 100644 --- a/rendering/vulkan14/vulkan14commands.cpp +++ b/rendering/vulkan14/vulkan14commands.cpp @@ -65,6 +65,7 @@ enum call_type : std::uint32_t call_begin_label, call_end_label, call_build_acceleration_structures, + call_set_checkpoint, call_external }; @@ -175,6 +176,10 @@ struct build_call std::size_t geometries; std::size_t ranges; }; +struct checkpoint_call +{ + std::uint64_t marker; +}; struct external_call { command_stream::external_function function; @@ -948,6 +953,12 @@ void command_stream::end_debug_utils_label() add(call_end_label, 0); } +void command_stream::set_checkpoint(std::uint64_t const Marker) +{ + auto const call{add(call_set_checkpoint, sizeof(checkpoint_call))}; + payload(call) = checkpoint_call{Marker}; +} + void command_stream::build_acceleration_structures(std::uint32_t const Count, VkAccelerationStructureBuildGeometryInfoKHR const *Infos, VkAccelerationStructureBuildRangeInfoKHR const *const *Ranges) { if (Count == 0) @@ -1273,6 +1284,13 @@ void command_stream::execute(std::size_t const Call, VkCommandBuffer const Buffe vkCmdBuildAccelerationStructuresKHR(Buffer, data.count, infos.data(), ranges.data()); break; } + case call_set_checkpoint: + if (vkCmdSetCheckpointNV != nullptr) + { + // the marker is the number itself, not a pointer to anything + vkCmdSetCheckpointNV(Buffer, reinterpret_cast(static_cast(payload(Call).marker))); + } + break; case call_external: { auto const &data{payload(Call)}; diff --git a/rendering/vulkan14/vulkan14commands.h b/rendering/vulkan14/vulkan14commands.h index c37fdb9d..cc612ee3 100644 --- a/rendering/vulkan14/vulkan14commands.h +++ b/rendering/vulkan14/vulkan14commands.h @@ -78,6 +78,8 @@ class command_stream void end_query(VkQueryPool Pool, std::uint32_t Query); void begin_debug_utils_label(VkDebugUtilsLabelEXT const *Label); void end_debug_utils_label(); + // a checkpoint of VK_NV_device_diagnostic_checkpoints, with the number of the marker (vk14::pass_list) + void set_checkpoint(std::uint64_t Marker); // the builds of acceleration structures (VK_KHR_acceleration_structure, gfx.raytracing), outside of a rendering void build_acceleration_structures(std::uint32_t Count, VkAccelerationStructureBuildGeometryInfoKHR const *Infos, VkAccelerationStructureBuildRangeInfoKHR const *const *Ranges); // a call of a library recording its own commands (the evaluation of dlss), outside of a rendering: Function gets the command diff --git a/rendering/vulkan14/vulkan14context.h b/rendering/vulkan14/vulkan14context.h index 97703641..e77f2db2 100644 --- a/rendering/vulkan14/vulkan14context.h +++ b/rendering/vulkan14/vulkan14context.h @@ -687,6 +687,8 @@ struct vulkan14_renderer::context void save_scene_pipelines(); // the value of the timeline the gpu reached, 0 when it can't be read std::uint64_t completed() const; + // the value completed() read last: once the device is lost the timeline can't be read + mutable std::atomic lastcompleted{0}; // waits until the gpu reached the value of the timeline; false: the device is lost bool wait(std::uint64_t const Value) const; void destroy(); diff --git a/rendering/vulkan14/vulkan14device.cpp b/rendering/vulkan14/vulkan14device.cpp index fa3e6312..e91fb1ad 100644 --- a/rendering/vulkan14/vulkan14device.cpp +++ b/rendering/vulkan14/vulkan14device.cpp @@ -12,6 +12,8 @@ http://mozilla.org/MPL/2.0/. #include "rendering/vulkan14/vulkan14instance.h" #include "utilities/Globals.h" +#include +#include #include namespace vk14 @@ -36,6 +38,36 @@ std::string lowercase(std::string Text) return Text; } +std::string hex(std::uint64_t const Value) +{ + char text[24]; + std::snprintf(text, sizeof(text), "0x%" PRIx64, Value); + return text; +} + +char const *fault_address_type(VkDeviceFaultAddressTypeEXT const Type) +{ + switch (Type) + { + case VK_DEVICE_FAULT_ADDRESS_TYPE_NONE_EXT: + return "no address"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_READ_INVALID_EXT: + return "invalid read"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_WRITE_INVALID_EXT: + return "invalid write"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_EXECUTE_INVALID_EXT: + return "invalid execution"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_UNKNOWN_EXT: + return "instruction pointer (fault not known)"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_INVALID_EXT: + return "instruction pointer (invalid instruction)"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_FAULT_EXT: + return "instruction pointer (fault)"; + default: + return "address"; + } +} + } // namespace device::~device() @@ -385,6 +417,46 @@ bool device::create(instance const &Instance, std::string &Reason, std::vector available(count); + vkEnumerateDeviceExtensionProperties(m_physical, nullptr, &count, available.data()); + auto const has = [&](char const *Name) { return std::any_of(available.begin(), available.end(), [Name](VkExtensionProperties const &Entry) { return std::strcmp(Entry.extensionName, Name) == 0; }); }; + auto const add = [&](char const *Name) { + if (std::none_of(extensions.begin(), extensions.end(), [Name](char const *Entry) { return std::strcmp(Entry, Name) == 0; })) + { + extensions.push_back(Name); + } + }; + if (Global.gfx_vulkan_checkpoints && has(VK_NV_DEVICE_DIAGNOSTIC_CHECKPOINTS_EXTENSION_NAME)) + { + add(VK_NV_DEVICE_DIAGNOSTIC_CHECKPOINTS_EXTENSION_NAME); + m_checkpoints = true; + } + if (has(VK_EXT_DEVICE_FAULT_EXTENSION_NAME)) + { + VkPhysicalDeviceFeatures2 features{}; + features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; + features.pNext = &fault; + vkGetPhysicalDeviceFeatures2(m_physical, &features); + fault.pNext = nullptr; + // the binary of the vendor is for its own tools, only the description and the addresses go to the log + fault.deviceFaultVendorBinary = VK_FALSE; + if (fault.deviceFault == VK_TRUE) + { + add(VK_EXT_DEVICE_FAULT_EXTENSION_NAME); + m_device_fault = true; + } + } + } + VkDeviceCreateInfo create{}; create.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO; create.pNext = enabled.head(); @@ -394,6 +466,11 @@ bool device::create(instance const &Instance, std::string &Reason, std::vector(create.pNext); create.pNext = &presentid; } + if (m_device_fault) + { + fault.pNext = const_cast(create.pNext); + create.pNext = &fault; + } create.queueCreateInfoCount = static_cast(queues.size()); create.pQueueCreateInfos = queues.data(); create.enabledExtensionCount = static_cast(extensions.size()); @@ -426,9 +503,64 @@ bool device::create(instance const &Instance, std::string &Reason, std::vector addresses(counts.addressInfoCount); + std::vector vendor(counts.vendorInfoCount); + counts.vendorBinarySize = 0; + VkDeviceFaultInfoEXT info{}; + info.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_INFO_EXT; + info.pAddressInfos = (addresses.empty() ? nullptr : addresses.data()); + info.pVendorInfos = (vendor.empty() ? nullptr : vendor.data()); + result = vkGetDeviceFaultInfoEXT(m_device, &counts, &info); + if ((result != VK_SUCCESS) && (result != VK_INCOMPLETE)) + { + return std::string("the fault info of the driver: vkGetDeviceFaultInfoEXT failed with ") + result_string(result); + } + addresses.resize(std::min(addresses.size(), counts.addressInfoCount)); + vendor.resize(std::min(vendor.size(), counts.vendorInfoCount)); + std::string report{std::string("the fault info of the driver: \"") + info.description + "\""}; + for (auto const &address : addresses) + { + // the address is known to the precision (a power of two): the fault is within the aligned range of that size around it + auto const precision{std::max(address.addressPrecision, 1)}; + auto const lower{address.reportedAddress & ~(precision - 1)}; + report += std::string("; ") + fault_address_type(address.addressType) + " at " + hex(address.reportedAddress); + if (precision > 1) + { + report += " (" + hex(lower) + " to " + hex(lower + precision - 1) + ")"; + } + } + for (auto const &entry : vendor) + { + report += std::string("; vendor: \"") + entry.description + "\" code " + hex(entry.vendorFaultCode) + " data " + hex(entry.vendorFaultData); + } + if (addresses.empty() && vendor.empty()) + { + report += " (no addresses: more likely a gpu which hung, a shader which didn't end, than one which read or wrote where it shouldn't)"; + } + return report; +} + void device::destroy() { if (m_device != VK_NULL_HANDLE) diff --git a/rendering/vulkan14/vulkan14device.h b/rendering/vulkan14/vulkan14device.h index 4f9d1008..f0068a00 100644 --- a/rendering/vulkan14/vulkan14device.h +++ b/rendering/vulkan14/vulkan14device.h @@ -93,6 +93,15 @@ class device { return m_low_latency; } + // VK_NV_device_diagnostic_checkpoints is on (gfx.vulkan.checkpoints): the passes mark where the gpu got to, for the report of a lost + // device + bool checkpoints_enabled() const + { + return m_checkpoints; + } + // what the driver tells of the fault which lost the device (VK_EXT_device_fault, where the device has it): its description, the + // addresses the gpu went wrong at and what the vendor adds. empty without the extension + std::string fault_report() const; private: static queue_families find_queue_families(VkInstance Instance, VkPhysicalDevice Physical, VkSurfaceKHR Surface, std::uint32_t &TimestampBits); @@ -107,6 +116,8 @@ class device bool m_memory_budget{false}; bool m_raytracing{false}; bool m_low_latency{false}; + bool m_checkpoints{false}; + bool m_device_fault{false}; VkQueue m_graphics_queue{VK_NULL_HANDLE}; VkQueue m_compute_queue{VK_NULL_HANDLE}; diff --git a/rendering/vulkan14/vulkan14passes.cpp b/rendering/vulkan14/vulkan14passes.cpp index 85fe2972..3b8d5c31 100644 --- a/rendering/vulkan14/vulkan14passes.cpp +++ b/rendering/vulkan14/vulkan14passes.cpp @@ -15,6 +15,18 @@ http://mozilla.org/MPL/2.0/. namespace vk14 { +namespace +{ + +// the marker of a checkpoint: the number of the frame, the queue (bit 15: the compute one, of the async passes), the index of the pass +// among those of the frame on its queue and whether it's the end of the pass (bit 0) +std::uint64_t checkpoint_marker(std::uint64_t const Number, bool const Async, std::size_t const Index) +{ + return (Number << 16) | (Async ? 0x8000u : 0u) | (static_cast(std::min(Index, 0x3fff)) << 1); +} + +} // namespace + image_use use_of(image &Image, VkImageLayout Layout, VkPipelineStageFlags2 Stage, VkAccessFlags2 Access) { return image_use{Image.handle(), Image.aspect(), &Image.state, Layout, Stage, Access}; @@ -103,6 +115,7 @@ bool pass_list::create(device const &Device, std::uint32_t Frames, std::string & { m_device = Device.handle(); m_period = Device.caps().timestamp_period; + m_checkpoints = Device.checkpoints_enabled(); // the bits of the timestamps of the queue the frames go to std::uint32_t count{0}; vkGetPhysicalDeviceQueueFamilyProperties(Device.physical(), &count, nullptr); @@ -191,12 +204,20 @@ void pass_list::begin_pass(char const *Name, std::vector const &Uses) barriers(m_commands, Uses); begin_label(m_commands, Name); auto &frame{m_frames[m_frame]}; + // every pass is named, the ones past the timed ones too (passes_of(), the checkpoints); the names of the timed ones go first, so + // they're those of the queries + auto &names{m_async ? frame.async_names : frame.names}; + m_openmarker = checkpoint_marker(frame.number, m_async, names.size()); + names.emplace_back(Name); + if (m_checkpoints) + { + m_commands->set_checkpoint(m_openmarker); + } if (m_async) { if ((frame.async_used < max_async) && (m_asyncmask != 0)) { m_commands->write_timestamp2(VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, frame.async_pool, frame.async_used * 2); - frame.async_names.emplace_back(Name); m_open = static_cast(frame.async_used); ++frame.async_used; } @@ -210,7 +231,6 @@ void pass_list::begin_pass(char const *Name, std::vector const &Uses) if ((frame.used < max_passes) && (m_mask != 0)) { m_commands->write_timestamp2(VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, frame.pool, frame.used * 2); - frame.names.emplace_back(Name); m_open = static_cast(frame.used); ++frame.used; } @@ -230,6 +250,10 @@ void pass_list::end_pass() { m_commands->write_timestamp2(VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT, m_async ? m_frames[m_frame].async_pool : m_frames[m_frame].pool, static_cast(m_open) * 2 + 1); } + if (m_checkpoints) + { + m_commands->set_checkpoint(m_openmarker | 1u); + } end_label(m_commands); m_open = -1; } @@ -335,6 +359,50 @@ std::string pass_list::passes_of(std::uint64_t Number) const return (result.empty() ? std::string("(not known)") : result); } +std::string pass_list::describe_checkpoint(std::uint64_t const Marker) const +{ + auto const number{Marker >> 16}; + auto const async{(Marker & 0x8000u) != 0}; + auto const index{static_cast((Marker >> 1) & 0x3fff)}; + std::string name{"pass " + std::to_string(index)}; + for (auto const &frame : m_frames) + { + auto const &names{async ? frame.async_names : frame.names}; + if ((frame.number == number) && (index < names.size())) + { + name = names[index]; + } + } + return std::string((Marker & 1u) ? "the end of " : "the start of ") + name + (async ? " (async)" : "") + " of frame " + std::to_string(number); +} + +std::string pass_list::checkpoints(VkQueue const Queue) const +{ + if ((false == m_checkpoints) || (Queue == VK_NULL_HANDLE) || (vkGetQueueCheckpointData2NV == nullptr)) + { + return "(not known)"; + } + std::uint32_t count{0}; + vkGetQueueCheckpointData2NV(Queue, &count, nullptr); + std::vector data(count); + for (auto &entry : data) + { + entry.sType = VK_STRUCTURE_TYPE_CHECKPOINT_DATA_2_NV; + } + vkGetQueueCheckpointData2NV(Queue, &count, data.data()); + data.resize(std::min(data.size(), count)); + std::string result; + for (auto const &entry : data) + { + auto const marker{static_cast(reinterpret_cast(entry.pCheckpointMarker))}; + auto const stage{(entry.stage == VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT) ? std::string("taken in at the top of the pipe") + : (entry.stage == VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT) ? std::string("done at the bottom of the pipe") + : "reached in the stages " + std::to_string(entry.stage)}; + result += (result.empty() ? "" : ", ") + stage + ": " + describe_checkpoint(marker); + } + return (result.empty() ? std::string("(no checkpoint reached)") : result); +} + std::string pass_list::info() const { std::string text; diff --git a/rendering/vulkan14/vulkan14passes.h b/rendering/vulkan14/vulkan14passes.h index 168c07bf..62970984 100644 --- a/rendering/vulkan14/vulkan14passes.h +++ b/rendering/vulkan14/vulkan14passes.h @@ -70,6 +70,9 @@ class pass_list // the passes recorded in the frame done last, and the ones of the frame Number, for the report of a lost device std::string passes_of(std::uint64_t Number) const; + // the checkpoints the gpu got to on Queue (device::checkpoints_enabled()): the pass of the frame whose start or end it reached last + // at the top of the pipe (the work it took in) and at the bottom (the work done), for the report of a lost device + std::string checkpoints(VkQueue Queue) const; // the gpu times (ms) of the parts of the frame, averaged, as the opengl renderer gives them: "name: x.xx ms" lines std::string info() const; double frame_ms() const @@ -103,6 +106,10 @@ class pass_list bool m_async{false}; std::uint32_t m_frame{0}; int m_open{-1}; + bool m_checkpoints{false}; + std::uint64_t m_openmarker{0}; // the checkpoint of the start of the open pass + // the name of the pass of a checkpoint + std::string describe_checkpoint(std::uint64_t Marker) const; // averaged times of the parts, ms std::vector> m_times; std::vector> m_asynctimes; // of the compute queue diff --git a/rendering/vulkan14/vulkan14renderer.cpp b/rendering/vulkan14/vulkan14renderer.cpp index 9cd43e9b..0b6b7b1a 100644 --- a/rendering/vulkan14/vulkan14renderer.cpp +++ b/rendering/vulkan14/vulkan14renderer.cpp @@ -641,6 +641,7 @@ std::uint64_t vulkan14_renderer::context::completed() const { return 0; } + lastcompleted = value; return value; } @@ -1008,7 +1009,8 @@ void vulkan14_renderer::report_device_lost(char const *Where) return; } ctx.devicelost = true; - auto const done{ctx.completed()}; + // the timeline can't be read once the device is lost: the value read before then + auto const done{std::max(ctx.completed(), ctx.lastcompleted.load())}; std::string report{std::string("the device was lost (") + Where + "); the gpu was done with the timeline value " + std::to_string(done) + " of " + std::to_string(ctx.timelinevalue)}; for (auto const &frame : ctx.frames) { @@ -1017,6 +1019,19 @@ void vulkan14_renderer::report_device_lost(char const *Where) report += "; frame " + std::to_string(frame.number) + " (value " + std::to_string(frame.done) + ") wasn't done, its passes: " + ctx.passes.passes_of(frame.number); } } + // where the gpu got to (the pass it hung or failed in is after the one done last, up to the one taken in last) + if (ctx.device.checkpoints_enabled()) + { + report += "; the checkpoints of the graphics queue: " + ctx.passes.checkpoints(ctx.device.graphics_queue()); + if (ctx.async_compute && (ctx.device.compute_queue() != ctx.device.graphics_queue())) + { + report += "; of the compute queue: " + ctx.passes.checkpoints(ctx.device.compute_queue()); + } + } + if (auto const fault{ctx.device.fault_report()}; false == fault.empty()) + { + report += "; " + fault; + } vk14::log_error(report); crashreport_add_info("vulkan_device_lost", report); } diff --git a/utilities/Globals.cpp b/utilities/Globals.cpp index dce8b21c..8a8a125c 100644 --- a/utilities/Globals.cpp +++ b/utilities/Globals.cpp @@ -1850,6 +1850,11 @@ global_settings::ConfigParse_gfx( cParser &Parser, std::string_view const Token Parser.getTokens(1); Parser >> gfx_vulkan_asynccompute; } + else if (Token == "gfx.vulkan.checkpoints") + { + Parser.getTokens(1); + Parser >> gfx_vulkan_checkpoints; + } else if (ConfigParse_gfx_sky(Parser, Token)) { @@ -2391,6 +2396,7 @@ global_settings::export_as_text( std::ostream &Output ) const { export_as_text( Output, "gfx.vulkan.parallelpasses", gfx_vulkan_parallelpasses ); export_as_text( Output, "gfx.vulkan.parallelmodels", gfx_vulkan_parallelmodels ); export_as_text( Output, "gfx.vulkan.asynccompute", gfx_vulkan_asynccompute ); + export_as_text( Output, "gfx.vulkan.checkpoints", gfx_vulkan_checkpoints ); export_as_text( Output, "gfx.shadow.angle.min", gfx_shadow_angle_min ); export_as_text( Output, "gfx.shadow.rank.cutoff", gfx_shadow_rank_cutoff ); export_as_text( Output, "gfx.shadow.sdsm", gfx_shadow_sdsm ); diff --git a/utilities/Globals.h b/utilities/Globals.h index b91da60e..ed3f90a1 100644 --- a/utilities/Globals.h +++ b/utilities/Globals.h @@ -436,6 +436,9 @@ struct global_settings { // (vulkan) the passes of the sky and the weather run on the compute queue, at the same time as the sun shadows, where the gpu has a // compute family of its own; off until a measurement on such a gpu shows what it gains bool gfx_vulkan_asynccompute { false }; + // (vulkan) the passes leave checkpoints where the gpu has them (VK_NV_device_diagnostic_checkpoints): the report of a lost device + // in the log tells the pass the gpu got to + bool gfx_vulkan_checkpoints { true }; bool vr = false; std::string vr_backend;