perf(models): e3d file read whole in one go and parsed from memory, vertices and indices taken straight from the buffer instead of value by value from the file stream

model/Model3d.cpp

@@ -1938,8 +1938,121 @@ void TSubModel::deserialize(std::istream &s)
 	m_rotation_init_done = true;
 }
 
+namespace
+{
+// read only stream buffer over the content of a whole e3d file kept in memory.
+// reading the geometry of a model value by value from a file stream costs several times more than the read of the file itself
+class e3d_memorybuffer : public std::streambuf
+{
+  public:
+	e3d_memorybuffer(char *Begin, char *End)
+	{
+		setg(Begin, Begin, End);
+	}
+	// data from the current read position on
+	unsigned char const *cursor() const
+	{
+		return reinterpret_cast<unsigned char const *>(gptr());
+	}
+	std::size_t available() const
+	{
+		return static_cast<std::size_t>(egptr() - gptr());
+	}
+
+  protected:
+	pos_type seekoff(off_type Offset, std::ios_base::seekdir Direction, std::ios_base::openmode) override
+	{
+		off_type const size{egptr() - eback()};
+		off_type const origin{Direction == std::ios_base::beg ? 0 : Direction == std::ios_base::cur ? (gptr() - eback()) + m_beyondend : size};
+		off_type const position{origin + Offset};
+		if (position < 0)
+		{
+			return pos_type(off_type(-1));
+		}
+		// like a file stream it can be placed past the end of the data, where nothing more can be read
+		m_beyondend = std::max<off_type>(position - size, 0);
+		setg(eback(), eback() + std::min(position, size), egptr());
+		return pos_type(position);
+	}
+	pos_type seekpos(pos_type Position, std::ios_base::openmode Mode) override
+	{
+		return seekoff(off_type(Position), std::ios_base::beg, Mode);
+	}
+
+  private:
+	off_type m_beyondend{0};
+};
+
+// little endian values taken from memory the same way the sn_utils functions read them from a stream
+std::uint16_t e3d_uint16(unsigned char const *&Data)
+{
+	std::uint16_t const value = static_cast<std::uint16_t>(Data[1] << 8 | Data[0]);
+	Data += 2;
+	return value;
+}
+std::uint32_t e3d_uint32(unsigned char const *&Data)
+{
+	std::uint32_t const value = std::uint32_t{Data[3]} << 24 | std::uint32_t{Data[2]} << 16 | std::uint32_t{Data[1]} << 8 | std::uint32_t{Data[0]};
+	Data += 4;
+	return value;
+}
+std::uint64_t e3d_uint64(unsigned char const *&Data)
+{
+	std::uint64_t const low{e3d_uint32(Data)};
+	std::uint64_t const high{e3d_uint32(Data)};
+	return high << 32 | low;
+}
+float e3d_float32(unsigned char const *&Data)
+{
+	return std::bit_cast<float>(e3d_uint32(Data));
+}
+glm::vec3 e3d_vec3(unsigned char const *&Data)
+{
+	glm::vec3 value;
+	value.x = e3d_float32(Data);
+	value.y = e3d_float32(Data);
+	value.z = e3d_float32(Data);
+	return value;
+}
+glm::vec4 e3d_vec4(unsigned char const *&Data)
+{
+	glm::vec4 value;
+	value.x = e3d_float32(Data);
+	value.y = e3d_float32(Data);
+	value.z = e3d_float32(Data);
+	value.w = e3d_float32(Data);
+	return value;
+}
+// counterparts of basic_vertex::deserialize() and basic_vertex::deserialize_packed()
+void e3d_vertex(unsigned char const *&Data, gfx::basic_vertex &Vertex, bool const Tangent)
+{
+	Vertex.position = e3d_vec3(Data);
+	Vertex.normal = e3d_vec3(Data);
+	Vertex.texture.x = e3d_float32(Data);
+	Vertex.texture.y = e3d_float32(Data);
+	if (Tangent)
+	{
+		Vertex.tangent = e3d_vec4(Data);
+	}
+}
+void e3d_vertex_packed(unsigned char const *&Data, gfx::basic_vertex &Vertex, bool const Tangent)
+{
+	Vertex.position = glm::unpackHalf4x16(e3d_uint64(Data));
+	Vertex.normal = glm::unpackSnorm3x10_1x2(e3d_uint32(Data));
+	Vertex.texture.x = glm::unpackHalf1x16(e3d_uint16(Data));
+	Vertex.texture.y = glm::unpackHalf1x16(e3d_uint16(Data));
+	if (Tangent)
+	{
+		Vertex.tangent = glm::unpackSnorm3x10_1x2(e3d_uint32(Data));
+	}
+}
+} // namespace
+
 void TModel3d::deserialize(std::istream &s, size_t size, bool dynamic)
 {
+	// a model loaded by LoadFromBinFile() comes from memory, its geometry is taken straight from there
+	auto *const memory{dynamic_cast<e3d_memorybuffer *>(s.rdbuf())};
+
 	Root = nullptr;
 	if (m_geometrybank == null_handle)
 	{
@@ -2026,6 +2139,67 @@ void TModel3d::deserialize(std::istream &s, size_t size, bool dynamic)
 				if (hasuserdata)
 					submodel.Userdata.resize(submodelgeometry.vertex_count);
 				m_vertexcount += submodelgeometry.vertex_count;
+				if (memory != nullptr)
+				{
+					// geometry past the end of the file is left as it was made
+					auto const datasize{vertex_size * sizeof(float)};
+					auto const vertexcount{std::min<std::size_t>(submodel.Vertices.size(), (datasize > 0 ? memory->available() / datasize : 0))};
+					if ((datasize > 0) && (vertexcount < submodel.Vertices.size()))
+					{
+						ErrorLog("Bad model: vertex data of sub-model " + std::to_string(submodeloffset.second) + " lies past the end of the file", logtype::model);
+					}
+					auto *data{memory->cursor()};
+					switch (vertextype & 3)
+					{
+					case 0:
+					{
+						// legacy vnt0 format
+						for (std::size_t i = 0; i < vertexcount; ++i)
+						{
+							e3d_vertex(data, submodel.Vertices[i], hastangents);
+							if (hasuserdata)
+								submodel.Userdata[i].data = e3d_vec4(data);
+							if (submodel.eType < TP_ROTATOR)
+							{
+								// normal vectors debug routine
+								if (false == submodel.m_normalizenormals && std::abs(glm::length2(submodel.Vertices[i].normal) - 1.0f) > 0.01f)
+								{
+									submodel.m_normalizenormals = TSubModel::normalize; // we don't know if uniform scaling would suffice
+									WriteLog("Bad model: non-unit normal vector(s) encountered during sub-model geometry deserialization", logtype::model);
+								}
+							}
+						}
+						break;
+					}
+					case 1:
+					{
+						// expanded chunk formats
+						for (std::size_t i = 0; i < vertexcount; ++i)
+						{
+							e3d_vertex_packed(data, submodel.Vertices[i], hastangents);
+							if (hasuserdata)
+								submodel.Userdata[i].data = glm::unpackHalf4x16(e3d_uint64(data));
+						}
+						break;
+					}
+					case 2:
+					{
+						// expanded chunk formats
+						for (std::size_t i = 0; i < vertexcount; ++i)
+						{
+							e3d_vertex(data, submodel.Vertices[i], hastangents);
+							if (hasuserdata)
+								submodel.Userdata[i].data = e3d_vec4(data);
+						}
+						break;
+					}
+					default:
+					{
+						break;
+					}
+					}
+					continue;
+				}
 				switch (vertextype & 3)
 				{
 				case 0:
@@ -2106,6 +2280,48 @@ void TModel3d::deserialize(std::istream &s, size_t size, bool dynamic)
 				auto const &submodelgeometry{submodel.m_geometry};
 				submodel.Indices.resize(submodelgeometry.index_count);
 				m_indexcount += submodelgeometry.index_count;
+				if (memory != nullptr)
+				{
+					// indices past the end of the file are left as zeroes
+					auto const indexcount{std::min<std::size_t>(submodel.Indices.size(), (indexsize > 0 ? memory->available() / indexsize : 0))};
+					if ((indexsize > 0) && (indexcount < submodel.Indices.size()))
+					{
+						ErrorLog("Bad model: index data of sub-model " + std::to_string(submodeloffset.second) + " lies past the end of the file", logtype::model);
+					}
+					auto *data{memory->cursor()};
+					switch (indexsize)
+					{
+					case 1:
+					{
+						for (std::size_t i = 0; i < indexcount; ++i)
+						{
+							submodel.Indices[i] = *data++;
+						}
+						break;
+					}
+					case 2:
+					{
+						for (std::size_t i = 0; i < indexcount; ++i)
+						{
+							submodel.Indices[i] = e3d_uint16(data);
+						}
+						break;
+					}
+					case 4:
+					{
+						for (std::size_t i = 0; i < indexcount; ++i)
+						{
+							submodel.Indices[i] = e3d_uint32(data);
+						}
+						break;
+					}
+					default:
+					{
+						break;
+					}
+					}
+					continue;
+				}
 				switch (indexsize)
 				{
 				case 1:
@@ -2377,7 +2593,22 @@ void TModel3d::LoadFromBinFile(std::string const &FileName, bool dynamic)
 { // wczytanie modelu z pliku binarnego
 	WriteLog("Loading binary format 3d model data from \"" + FileName + "\"...", logtype::model);
 
-	std::ifstream file(FileName, std::ios::binary);
+	// the file is read whole in one go and parsed from memory
+	std::unique_ptr<char[]> filedata;
+	std::size_t filesize{0};
+	{
+		std::ifstream input(FileName, std::ios::binary | std::ios::ate);
+		auto const length{input.tellg()};
+		if (length > 0)
+		{
+			filedata.reset(new char[static_cast<std::size_t>(length)]);
+			input.seekg(0);
+			input.read(filedata.get(), length);
+			filesize = static_cast<std::size_t>(input.gcount());
+		}
+	}
+	e3d_memorybuffer buffer(filedata.get(), filedata.get() + filesize);
+	std::istream file(&buffer);
 
 	uint32_t type = sn_utils::ld_uint32(file);
 	uint32_t size = sn_utils::ld_uint32(file) - 8;
@@ -2385,7 +2616,6 @@ void TModel3d::LoadFromBinFile(std::string const &FileName, bool dynamic)
 	if (type == MAKE_ID4('E', '3', 'D', '0'))
 	{
 		deserialize(file, size, dynamic);
-		file.close();
 
 		WriteLog("Finished loading 3d model data from \"" + FileName + "\"", logtype::model);
 	}
@@ -2393,7 +2623,6 @@ void TModel3d::LoadFromBinFile(std::string const &FileName, bool dynamic)
 	{
 		// throw std::runtime_error("e3d: unknown main chunk");
 		ErrorLog("Bad model: unknown main chunk in file \"" + FileName + "\"", logtype::model);
-		file.close();
 	}
 };