Report where the GPU got to when the Vulkan device is lost

rendering/vulkan14/vulkan14commands.cpp

@@ -65,6 +65,7 @@ enum call_type : std::uint32_t
 	call_begin_label,
 	call_end_label,
 	call_build_acceleration_structures,
+	call_set_checkpoint,
 	call_external
 };
 
@@ -175,6 +176,10 @@ struct build_call
 	std::size_t geometries;
 	std::size_t ranges;
 };
+struct checkpoint_call
+{
+	std::uint64_t marker;
+};
 struct external_call
 {
 	command_stream::external_function function;
@@ -948,6 +953,12 @@ void command_stream::end_debug_utils_label()
 	add(call_end_label, 0);
 }
 
+void command_stream::set_checkpoint(std::uint64_t const Marker)
+{
+	auto const call{add(call_set_checkpoint, sizeof(checkpoint_call))};
+	payload<checkpoint_call>(call) = checkpoint_call{Marker};
+}
+
 void command_stream::build_acceleration_structures(std::uint32_t const Count, VkAccelerationStructureBuildGeometryInfoKHR const *Infos, VkAccelerationStructureBuildRangeInfoKHR const *const *Ranges)
 {
 	if (Count == 0)
@@ -1273,6 +1284,13 @@ void command_stream::execute(std::size_t const Call, VkCommandBuffer const Buffe
 		vkCmdBuildAccelerationStructuresKHR(Buffer, data.count, infos.data(), ranges.data());
 		break;
 	}
+	case call_set_checkpoint:
+		if (vkCmdSetCheckpointNV != nullptr)
+		{
+			// the marker is the number itself, not a pointer to anything
+			vkCmdSetCheckpointNV(Buffer, reinterpret_cast<void const *>(static_cast<std::uintptr_t>(payload<checkpoint_call>(Call).marker)));
+		}
+		break;
 	case call_external:
 	{
 		auto const &data{payload<external_call>(Call)};

rendering/vulkan14/vulkan14commands.h

@@ -78,6 +78,8 @@ class command_stream
 	void end_query(VkQueryPool Pool, std::uint32_t Query);
 	void begin_debug_utils_label(VkDebugUtilsLabelEXT const *Label);
 	void end_debug_utils_label();
+	// a checkpoint of VK_NV_device_diagnostic_checkpoints, with the number of the marker (vk14::pass_list)
+	void set_checkpoint(std::uint64_t Marker);
 	// the builds of acceleration structures (VK_KHR_acceleration_structure, gfx.raytracing), outside of a rendering
 	void build_acceleration_structures(std::uint32_t Count, VkAccelerationStructureBuildGeometryInfoKHR const *Infos, VkAccelerationStructureBuildRangeInfoKHR const *const *Ranges);
 	// a call of a library recording its own commands (the evaluation of dlss), outside of a rendering: Function gets the command

rendering/vulkan14/vulkan14context.h

@@ -687,6 +687,8 @@ struct vulkan14_renderer::context
 	void save_scene_pipelines();
 	// the value of the timeline the gpu reached, 0 when it can't be read
 	std::uint64_t completed() const;
+	// the value completed() read last: once the device is lost the timeline can't be read
+	mutable std::atomic<std::uint64_t> lastcompleted{0};
 	// waits until the gpu reached the value of the timeline; false: the device is lost
 	bool wait(std::uint64_t const Value) const;
 	void destroy();

rendering/vulkan14/vulkan14device.cpp

@@ -12,6 +12,8 @@ http://mozilla.org/MPL/2.0/.
 #include "rendering/vulkan14/vulkan14instance.h"
 #include "utilities/Globals.h"
 
+#include <cinttypes>
+#include <cstdio>
 #include <cstring>
 
 namespace vk14
@@ -36,6 +38,36 @@ std::string lowercase(std::string Text)
 	return Text;
 }
 
+std::string hex(std::uint64_t const Value)
+{
+	char text[24];
+	std::snprintf(text, sizeof(text), "0x%" PRIx64, Value);
+	return text;
+}
+
+char const *fault_address_type(VkDeviceFaultAddressTypeEXT const Type)
+{
+	switch (Type)
+	{
+	case VK_DEVICE_FAULT_ADDRESS_TYPE_NONE_EXT:
+		return "no address";
+	case VK_DEVICE_FAULT_ADDRESS_TYPE_READ_INVALID_EXT:
+		return "invalid read";
+	case VK_DEVICE_FAULT_ADDRESS_TYPE_WRITE_INVALID_EXT:
+		return "invalid write";
+	case VK_DEVICE_FAULT_ADDRESS_TYPE_EXECUTE_INVALID_EXT:
+		return "invalid execution";
+	case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_UNKNOWN_EXT:
+		return "instruction pointer (fault not known)";
+	case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_INVALID_EXT:
+		return "instruction pointer (invalid instruction)";
+	case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_FAULT_EXT:
+		return "instruction pointer (fault)";
+	default:
+		return "address";
+	}
+}
+
 } // namespace
 
 device::~device()
@@ -385,6 +417,46 @@ bool device::create(instance const &Instance, std::string &Reason, std::vector<s
 		log_info("gfx.raytracing: " + m_capabilities.device_name + " has no ray queries (or not what they need), the reflections keep to the screen and the environment map");
 	}
 
+	// what the report of a lost device can tell: the passes the gpu got to (checkpoints of nvidia, gfx.vulkan.checkpoints) and what the
+	// driver knows of the fault. neither costs anything until the device is lost, but for the few checkpoints of a frame
+	VkPhysicalDeviceFaultFeaturesEXT fault{};
+	fault.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FAULT_FEATURES_EXT;
+	m_checkpoints = false;
+	m_device_fault = false;
+	{
+		std::uint32_t count{0};
+		vkEnumerateDeviceExtensionProperties(m_physical, nullptr, &count, nullptr);
+		std::vector<VkExtensionProperties> available(count);
+		vkEnumerateDeviceExtensionProperties(m_physical, nullptr, &count, available.data());
+		auto const has = [&](char const *Name) { return std::any_of(available.begin(), available.end(), [Name](VkExtensionProperties const &Entry) { return std::strcmp(Entry.extensionName, Name) == 0; }); };
+		auto const add = [&](char const *Name) {
+			if (std::none_of(extensions.begin(), extensions.end(), [Name](char const *Entry) { return std::strcmp(Entry, Name) == 0; }))
+			{
+				extensions.push_back(Name);
+			}
+		};
+		if (Global.gfx_vulkan_checkpoints && has(VK_NV_DEVICE_DIAGNOSTIC_CHECKPOINTS_EXTENSION_NAME))
+		{
+			add(VK_NV_DEVICE_DIAGNOSTIC_CHECKPOINTS_EXTENSION_NAME);
+			m_checkpoints = true;
+		}
+		if (has(VK_EXT_DEVICE_FAULT_EXTENSION_NAME))
+		{
+			VkPhysicalDeviceFeatures2 features{};
+			features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;
+			features.pNext = &fault;
+			vkGetPhysicalDeviceFeatures2(m_physical, &features);
+			fault.pNext = nullptr;
+			// the binary of the vendor is for its own tools, only the description and the addresses go to the log
+			fault.deviceFaultVendorBinary = VK_FALSE;
+			if (fault.deviceFault == VK_TRUE)
+			{
+				add(VK_EXT_DEVICE_FAULT_EXTENSION_NAME);
+				m_device_fault = true;
+			}
+		}
+	}
+
 	VkDeviceCreateInfo create{};
 	create.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO;
 	create.pNext = enabled.head();
@@ -394,6 +466,11 @@ bool device::create(instance const &Instance, std::string &Reason, std::vector<s
 		presentid.pNext = const_cast<void *>(create.pNext);
 		create.pNext = &presentid;
 	}
+	if (m_device_fault)
+	{
+		fault.pNext = const_cast<void *>(create.pNext);
+		create.pNext = &fault;
+	}
 	create.queueCreateInfoCount = static_cast<std::uint32_t>(queues.size());
 	create.pQueueCreateInfos = queues.data();
 	create.enabledExtensionCount = static_cast<std::uint32_t>(extensions.size());
@@ -426,9 +503,64 @@ bool device::create(instance const &Instance, std::string &Reason, std::vector<s
 	{
 		log_info("VK_NV_low_latency2 on, for reflex (gfx.reflex)");
 	}
+	if (m_checkpoints || m_device_fault)
+	{
+		log_info(std::string("for the report of a lost device:") + (m_checkpoints ? " the checkpoints of the passes (gfx.vulkan.checkpoints)" : "") + (m_checkpoints && m_device_fault ? "," : "") +
+		         (m_device_fault ? " the fault info of the driver" : ""));
+	}
 	return true;
 }
 
+std::string device::fault_report() const
+{
+	if ((false == m_device_fault) || (m_device == VK_NULL_HANDLE) || (vkGetDeviceFaultInfoEXT == nullptr))
+	{
+		return {};
+	}
+	VkDeviceFaultCountsEXT counts{};
+	counts.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT;
+	auto result{vkGetDeviceFaultInfoEXT(m_device, &counts, nullptr)};
+	if (result != VK_SUCCESS)
+	{
+		return std::string("the fault info of the driver: vkGetDeviceFaultInfoEXT failed with ") + result_string(result);
+	}
+	std::vector<VkDeviceFaultAddressInfoEXT> addresses(counts.addressInfoCount);
+	std::vector<VkDeviceFaultVendorInfoEXT> vendor(counts.vendorInfoCount);
+	counts.vendorBinarySize = 0;
+	VkDeviceFaultInfoEXT info{};
+	info.sType = VK_STRUCTURE_TYPE_DEVICE_FAULT_INFO_EXT;
+	info.pAddressInfos = (addresses.empty() ? nullptr : addresses.data());
+	info.pVendorInfos = (vendor.empty() ? nullptr : vendor.data());
+	result = vkGetDeviceFaultInfoEXT(m_device, &counts, &info);
+	if ((result != VK_SUCCESS) && (result != VK_INCOMPLETE))
+	{
+		return std::string("the fault info of the driver: vkGetDeviceFaultInfoEXT failed with ") + result_string(result);
+	}
+	addresses.resize(std::min<std::size_t>(addresses.size(), counts.addressInfoCount));
+	vendor.resize(std::min<std::size_t>(vendor.size(), counts.vendorInfoCount));
+	std::string report{std::string("the fault info of the driver: \"") + info.description + "\""};
+	for (auto const &address : addresses)
+	{
+		// the address is known to the precision (a power of two): the fault is within the aligned range of that size around it
+		auto const precision{std::max<VkDeviceSize>(address.addressPrecision, 1)};
+		auto const lower{address.reportedAddress & ~(precision - 1)};
+		report += std::string("; ") + fault_address_type(address.addressType) + " at " + hex(address.reportedAddress);
+		if (precision > 1)
+		{
+			report += " (" + hex(lower) + " to " + hex(lower + precision - 1) + ")";
+		}
+	}
+	for (auto const &entry : vendor)
+	{
+		report += std::string("; vendor: \"") + entry.description + "\" code " + hex(entry.vendorFaultCode) + " data " + hex(entry.vendorFaultData);
+	}
+	if (addresses.empty() && vendor.empty())
+	{
+		report += " (no addresses: more likely a gpu which hung, a shader which didn't end, than one which read or wrote where it shouldn't)";
+	}
+	return report;
+}
+
 void device::destroy()
 {
 	if (m_device != VK_NULL_HANDLE)

rendering/vulkan14/vulkan14device.h

@@ -93,6 +93,15 @@ class device
 	{
 		return m_low_latency;
 	}
+	// VK_NV_device_diagnostic_checkpoints is on (gfx.vulkan.checkpoints): the passes mark where the gpu got to, for the report of a lost
+	// device
+	bool checkpoints_enabled() const
+	{
+		return m_checkpoints;
+	}
+	// what the driver tells of the fault which lost the device (VK_EXT_device_fault, where the device has it): its description, the
+	// addresses the gpu went wrong at and what the vendor adds. empty without the extension
+	std::string fault_report() const;
 
   private:
 	static queue_families find_queue_families(VkInstance Instance, VkPhysicalDevice Physical, VkSurfaceKHR Surface, std::uint32_t &TimestampBits);
@@ -107,6 +116,8 @@ class device
 	bool m_memory_budget{false};
 	bool m_raytracing{false};
 	bool m_low_latency{false};
+	bool m_checkpoints{false};
+	bool m_device_fault{false};
 
 	VkQueue m_graphics_queue{VK_NULL_HANDLE};
 	VkQueue m_compute_queue{VK_NULL_HANDLE};

rendering/vulkan14/vulkan14passes.cpp

@@ -15,6 +15,18 @@ http://mozilla.org/MPL/2.0/.
 namespace vk14
 {
 
+namespace
+{
+
+// the marker of a checkpoint: the number of the frame, the queue (bit 15: the compute one, of the async passes), the index of the pass
+// among those of the frame on its queue and whether it's the end of the pass (bit 0)
+std::uint64_t checkpoint_marker(std::uint64_t const Number, bool const Async, std::size_t const Index)
+{
+	return (Number << 16) | (Async ? 0x8000u : 0u) | (static_cast<std::uint64_t>(std::min<std::size_t>(Index, 0x3fff)) << 1);
+}
+
+} // namespace
+
 image_use use_of(image &Image, VkImageLayout Layout, VkPipelineStageFlags2 Stage, VkAccessFlags2 Access)
 {
 	return image_use{Image.handle(), Image.aspect(), &Image.state, Layout, Stage, Access};
@@ -103,6 +115,7 @@ bool pass_list::create(device const &Device, std::uint32_t Frames, std::string &
 {
 	m_device = Device.handle();
 	m_period = Device.caps().timestamp_period;
+	m_checkpoints = Device.checkpoints_enabled();
 	// the bits of the timestamps of the queue the frames go to
 	std::uint32_t count{0};
 	vkGetPhysicalDeviceQueueFamilyProperties(Device.physical(), &count, nullptr);
@@ -191,12 +204,20 @@ void pass_list::begin_pass(char const *Name, std::vector<image_use> const &Uses)
 	barriers(m_commands, Uses);
 	begin_label(m_commands, Name);
 	auto &frame{m_frames[m_frame]};
+	// every pass is named, the ones past the timed ones too (passes_of(), the checkpoints); the names of the timed ones go first, so
+	// they're those of the queries
+	auto &names{m_async ? frame.async_names : frame.names};
+	m_openmarker = checkpoint_marker(frame.number, m_async, names.size());
+	names.emplace_back(Name);
+	if (m_checkpoints)
+	{
+		m_commands->set_checkpoint(m_openmarker);
+	}
 	if (m_async)
 	{
 		if ((frame.async_used < max_async) && (m_asyncmask != 0))
 		{
 			m_commands->write_timestamp2(VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, frame.async_pool, frame.async_used * 2);
-			frame.async_names.emplace_back(Name);
 			m_open = static_cast<int>(frame.async_used);
 			++frame.async_used;
 		}
@@ -210,7 +231,6 @@ void pass_list::begin_pass(char const *Name, std::vector<image_use> const &Uses)
 	if ((frame.used < max_passes) && (m_mask != 0))
 	{
 		m_commands->write_timestamp2(VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, frame.pool, frame.used * 2);
-		frame.names.emplace_back(Name);
 		m_open = static_cast<int>(frame.used);
 		++frame.used;
 	}
@@ -230,6 +250,10 @@ void pass_list::end_pass()
 	{
 		m_commands->write_timestamp2(VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT, m_async ? m_frames[m_frame].async_pool : m_frames[m_frame].pool, static_cast<std::uint32_t>(m_open) * 2 + 1);
 	}
+	if (m_checkpoints)
+	{
+		m_commands->set_checkpoint(m_openmarker | 1u);
+	}
 	end_label(m_commands);
 	m_open = -1;
 }
@@ -335,6 +359,50 @@ std::string pass_list::passes_of(std::uint64_t Number) const
 	return (result.empty() ? std::string("(not known)") : result);
 }
 
+std::string pass_list::describe_checkpoint(std::uint64_t const Marker) const
+{
+	auto const number{Marker >> 16};
+	auto const async{(Marker & 0x8000u) != 0};
+	auto const index{static_cast<std::size_t>((Marker >> 1) & 0x3fff)};
+	std::string name{"pass " + std::to_string(index)};
+	for (auto const &frame : m_frames)
+	{
+		auto const &names{async ? frame.async_names : frame.names};
+		if ((frame.number == number) && (index < names.size()))
+		{
+			name = names[index];
+		}
+	}
+	return std::string((Marker & 1u) ? "the end of " : "the start of ") + name + (async ? " (async)" : "") + " of frame " + std::to_string(number);
+}
+
+std::string pass_list::checkpoints(VkQueue const Queue) const
+{
+	if ((false == m_checkpoints) || (Queue == VK_NULL_HANDLE) || (vkGetQueueCheckpointData2NV == nullptr))
+	{
+		return "(not known)";
+	}
+	std::uint32_t count{0};
+	vkGetQueueCheckpointData2NV(Queue, &count, nullptr);
+	std::vector<VkCheckpointData2NV> data(count);
+	for (auto &entry : data)
+	{
+		entry.sType = VK_STRUCTURE_TYPE_CHECKPOINT_DATA_2_NV;
+	}
+	vkGetQueueCheckpointData2NV(Queue, &count, data.data());
+	data.resize(std::min<std::size_t>(data.size(), count));
+	std::string result;
+	for (auto const &entry : data)
+	{
+		auto const marker{static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(entry.pCheckpointMarker))};
+		auto const stage{(entry.stage == VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT)      ? std::string("taken in at the top of the pipe")
+		                 : (entry.stage == VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT) ? std::string("done at the bottom of the pipe")
+		                                                                           : "reached in the stages " + std::to_string(entry.stage)};
+		result += (result.empty() ? "" : ", ") + stage + ": " + describe_checkpoint(marker);
+	}
+	return (result.empty() ? std::string("(no checkpoint reached)") : result);
+}
+
 std::string pass_list::info() const
 {
 	std::string text;

rendering/vulkan14/vulkan14passes.h

@@ -70,6 +70,9 @@ class pass_list
 
 	// the passes recorded in the frame done last, and the ones of the frame Number, for the report of a lost device
 	std::string passes_of(std::uint64_t Number) const;
+	// the checkpoints the gpu got to on Queue (device::checkpoints_enabled()): the pass of the frame whose start or end it reached last
+	// at the top of the pipe (the work it took in) and at the bottom (the work done), for the report of a lost device
+	std::string checkpoints(VkQueue Queue) const;
 	// the gpu times (ms) of the parts of the frame, averaged, as the opengl renderer gives them: "name: x.xx ms" lines
 	std::string info() const;
 	double frame_ms() const
@@ -103,6 +106,10 @@ class pass_list
 	bool m_async{false};
 	std::uint32_t m_frame{0};
 	int m_open{-1};
+	bool m_checkpoints{false};
+	std::uint64_t m_openmarker{0}; // the checkpoint of the start of the open pass
+	// the name of the pass of a checkpoint
+	std::string describe_checkpoint(std::uint64_t Marker) const;
 	// averaged times of the parts, ms
 	std::vector<std::pair<std::string, double>> m_times;
 	std::vector<std::pair<std::string, double>> m_asynctimes; // of the compute queue

rendering/vulkan14/vulkan14renderer.cpp

@@ -641,6 +641,7 @@ std::uint64_t vulkan14_renderer::context::completed() const
 	{
 		return 0;
 	}
+	lastcompleted = value;
 	return value;
 }
 
@@ -1008,7 +1009,8 @@ void vulkan14_renderer::report_device_lost(char const *Where)
 		return;
 	}
 	ctx.devicelost = true;
-	auto const done{ctx.completed()};
+	// the timeline can't be read once the device is lost: the value read before then
+	auto const done{std::max(ctx.completed(), ctx.lastcompleted.load())};
 	std::string report{std::string("the device was lost (") + Where + "); the gpu was done with the timeline value " + std::to_string(done) + " of " + std::to_string(ctx.timelinevalue)};
 	for (auto const &frame : ctx.frames)
 	{
@@ -1017,6 +1019,19 @@ void vulkan14_renderer::report_device_lost(char const *Where)
 			report += "; frame " + std::to_string(frame.number) + " (value " + std::to_string(frame.done) + ") wasn't done, its passes: " + ctx.passes.passes_of(frame.number);
 		}
 	}
+	// where the gpu got to (the pass it hung or failed in is after the one done last, up to the one taken in last)
+	if (ctx.device.checkpoints_enabled())
+	{
+		report += "; the checkpoints of the graphics queue: " + ctx.passes.checkpoints(ctx.device.graphics_queue());
+		if (ctx.async_compute && (ctx.device.compute_queue() != ctx.device.graphics_queue()))
+		{
+			report += "; of the compute queue: " + ctx.passes.checkpoints(ctx.device.compute_queue());
+		}
+	}
+	if (auto const fault{ctx.device.fault_report()}; false == fault.empty())
+	{
+		report += "; " + fault;
+	}
 	vk14::log_error(report);
 	crashreport_add_info("vulkan_device_lost", report);
 }

utilities/Globals.cpp

@@ -1850,6 +1850,11 @@ global_settings::ConfigParse_gfx( cParser &Parser, std::string_view const Token
         Parser.getTokens(1);
         Parser >> gfx_vulkan_asynccompute;
     }
+    else if (Token == "gfx.vulkan.checkpoints")
+    {
+        Parser.getTokens(1);
+        Parser >> gfx_vulkan_checkpoints;
+    }
 
     else if (ConfigParse_gfx_sky(Parser, Token))
     {
@@ -2391,6 +2396,7 @@ global_settings::export_as_text( std::ostream &Output ) const {
     export_as_text( Output, "gfx.vulkan.parallelpasses", gfx_vulkan_parallelpasses );
     export_as_text( Output, "gfx.vulkan.parallelmodels", gfx_vulkan_parallelmodels );
     export_as_text( Output, "gfx.vulkan.asynccompute", gfx_vulkan_asynccompute );
+    export_as_text( Output, "gfx.vulkan.checkpoints", gfx_vulkan_checkpoints );
     export_as_text( Output, "gfx.shadow.angle.min", gfx_shadow_angle_min );
     export_as_text( Output, "gfx.shadow.rank.cutoff", gfx_shadow_rank_cutoff );
     export_as_text( Output, "gfx.shadow.sdsm", gfx_shadow_sdsm );

utilities/Globals.h

@@ -436,6 +436,9 @@ struct global_settings {
     // (vulkan) the passes of the sky and the weather run on the compute queue, at the same time as the sun shadows, where the gpu has a
     // compute family of its own; off until a measurement on such a gpu shows what it gains
     bool gfx_vulkan_asynccompute { false };
+    // (vulkan) the passes leave checkpoints where the gpu has them (VK_NV_device_diagnostic_checkpoints): the report of a lost device
+    // in the log tells the pass the gpu got to
+    bool gfx_vulkan_checkpoints { true };
     bool vr = false;
     std::string vr_backend;