From 516aa19cb5cf8b57dac4bcd024275a440f42adaa Mon Sep 17 00:00:00 2001 From: Hirek193 Date: Thu, 8 Oct 2026 16:17:37 +0200 Subject: [PATCH] perf(models): e3d file read whole in one go and parsed from memory, vertices and indices taken straight from the buffer instead of value by value from the file stream --- model/Model3d.cpp | 235 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 232 insertions(+), 3 deletions(-) diff --git a/model/Model3d.cpp b/model/Model3d.cpp index cb9c0793..91860cdf 100644 --- a/model/Model3d.cpp +++ b/model/Model3d.cpp @@ -1938,8 +1938,121 @@ void TSubModel::deserialize(std::istream &s) m_rotation_init_done = true; } +namespace +{ +// read only stream buffer over the content of a whole e3d file kept in memory. +// reading the geometry of a model value by value from a file stream costs several times more than the read of the file itself +class e3d_memorybuffer : public std::streambuf +{ + public: + e3d_memorybuffer(char *Begin, char *End) + { + setg(Begin, Begin, End); + } + // data from the current read position on + unsigned char const *cursor() const + { + return reinterpret_cast(gptr()); + } + std::size_t available() const + { + return static_cast(egptr() - gptr()); + } + + protected: + pos_type seekoff(off_type Offset, std::ios_base::seekdir Direction, std::ios_base::openmode) override + { + off_type const size{egptr() - eback()}; + off_type const origin{Direction == std::ios_base::beg ? 0 : Direction == std::ios_base::cur ? (gptr() - eback()) + m_beyondend : size}; + off_type const position{origin + Offset}; + if (position < 0) + { + return pos_type(off_type(-1)); + } + // like a file stream it can be placed past the end of the data, where nothing more can be read + m_beyondend = std::max(position - size, 0); + setg(eback(), eback() + std::min(position, size), egptr()); + return pos_type(position); + } + pos_type seekpos(pos_type Position, std::ios_base::openmode Mode) override + { + return seekoff(off_type(Position), std::ios_base::beg, Mode); + } + + private: + off_type m_beyondend{0}; +}; + +// little endian values taken from memory the same way the sn_utils functions read them from a stream +std::uint16_t e3d_uint16(unsigned char const *&Data) +{ + std::uint16_t const value = static_cast(Data[1] << 8 | Data[0]); + Data += 2; + return value; +} +std::uint32_t e3d_uint32(unsigned char const *&Data) +{ + std::uint32_t const value = std::uint32_t{Data[3]} << 24 | std::uint32_t{Data[2]} << 16 | std::uint32_t{Data[1]} << 8 | std::uint32_t{Data[0]}; + Data += 4; + return value; +} +std::uint64_t e3d_uint64(unsigned char const *&Data) +{ + std::uint64_t const low{e3d_uint32(Data)}; + std::uint64_t const high{e3d_uint32(Data)}; + return high << 32 | low; +} +float e3d_float32(unsigned char const *&Data) +{ + return std::bit_cast(e3d_uint32(Data)); +} +glm::vec3 e3d_vec3(unsigned char const *&Data) +{ + glm::vec3 value; + value.x = e3d_float32(Data); + value.y = e3d_float32(Data); + value.z = e3d_float32(Data); + return value; +} +glm::vec4 e3d_vec4(unsigned char const *&Data) +{ + glm::vec4 value; + value.x = e3d_float32(Data); + value.y = e3d_float32(Data); + value.z = e3d_float32(Data); + value.w = e3d_float32(Data); + return value; +} +// counterparts of basic_vertex::deserialize() and basic_vertex::deserialize_packed() +void e3d_vertex(unsigned char const *&Data, gfx::basic_vertex &Vertex, bool const Tangent) +{ + Vertex.position = e3d_vec3(Data); + Vertex.normal = e3d_vec3(Data); + Vertex.texture.x = e3d_float32(Data); + Vertex.texture.y = e3d_float32(Data); + if (Tangent) + { + Vertex.tangent = e3d_vec4(Data); + } +} +void e3d_vertex_packed(unsigned char const *&Data, gfx::basic_vertex &Vertex, bool const Tangent) +{ + Vertex.position = glm::unpackHalf4x16(e3d_uint64(Data)); + Vertex.normal = glm::unpackSnorm3x10_1x2(e3d_uint32(Data)); + Vertex.texture.x = glm::unpackHalf1x16(e3d_uint16(Data)); + Vertex.texture.y = glm::unpackHalf1x16(e3d_uint16(Data)); + if (Tangent) + { + Vertex.tangent = glm::unpackSnorm3x10_1x2(e3d_uint32(Data)); + } +} +} // namespace + void TModel3d::deserialize(std::istream &s, size_t size, bool dynamic) { + // a model loaded by LoadFromBinFile() comes from memory, its geometry is taken straight from there + auto *const memory{dynamic_cast(s.rdbuf())}; + Root = nullptr; if (m_geometrybank == null_handle) { @@ -2026,6 +2139,67 @@ void TModel3d::deserialize(std::istream &s, size_t size, bool dynamic) if (hasuserdata) submodel.Userdata.resize(submodelgeometry.vertex_count); m_vertexcount += submodelgeometry.vertex_count; + if (memory != nullptr) + { + // geometry past the end of the file is left as it was made + auto const datasize{vertex_size * sizeof(float)}; + auto const vertexcount{std::min(submodel.Vertices.size(), (datasize > 0 ? memory->available() / datasize : 0))}; + if ((datasize > 0) && (vertexcount < submodel.Vertices.size())) + { + ErrorLog("Bad model: vertex data of sub-model " + std::to_string(submodeloffset.second) + " lies past the end of the file", logtype::model); + } + auto *data{memory->cursor()}; + switch (vertextype & 3) + { + case 0: + { + // legacy vnt0 format + for (std::size_t i = 0; i < vertexcount; ++i) + { + e3d_vertex(data, submodel.Vertices[i], hastangents); + if (hasuserdata) + submodel.Userdata[i].data = e3d_vec4(data); + if (submodel.eType < TP_ROTATOR) + { + // normal vectors debug routine + if (false == submodel.m_normalizenormals && std::abs(glm::length2(submodel.Vertices[i].normal) - 1.0f) > 0.01f) + { + submodel.m_normalizenormals = TSubModel::normalize; // we don't know if uniform scaling would suffice + WriteLog("Bad model: non-unit normal vector(s) encountered during sub-model geometry deserialization", logtype::model); + } + } + } + break; + } + case 1: + { + // expanded chunk formats + for (std::size_t i = 0; i < vertexcount; ++i) + { + e3d_vertex_packed(data, submodel.Vertices[i], hastangents); + if (hasuserdata) + submodel.Userdata[i].data = glm::unpackHalf4x16(e3d_uint64(data)); + } + break; + } + case 2: + { + // expanded chunk formats + for (std::size_t i = 0; i < vertexcount; ++i) + { + e3d_vertex(data, submodel.Vertices[i], hastangents); + if (hasuserdata) + submodel.Userdata[i].data = e3d_vec4(data); + } + break; + } + default: + { + break; + } + } + continue; + } switch (vertextype & 3) { case 0: @@ -2106,6 +2280,48 @@ void TModel3d::deserialize(std::istream &s, size_t size, bool dynamic) auto const &submodelgeometry{submodel.m_geometry}; submodel.Indices.resize(submodelgeometry.index_count); m_indexcount += submodelgeometry.index_count; + if (memory != nullptr) + { + // indices past the end of the file are left as zeroes + auto const indexcount{std::min(submodel.Indices.size(), (indexsize > 0 ? memory->available() / indexsize : 0))}; + if ((indexsize > 0) && (indexcount < submodel.Indices.size())) + { + ErrorLog("Bad model: index data of sub-model " + std::to_string(submodeloffset.second) + " lies past the end of the file", logtype::model); + } + auto *data{memory->cursor()}; + switch (indexsize) + { + case 1: + { + for (std::size_t i = 0; i < indexcount; ++i) + { + submodel.Indices[i] = *data++; + } + break; + } + case 2: + { + for (std::size_t i = 0; i < indexcount; ++i) + { + submodel.Indices[i] = e3d_uint16(data); + } + break; + } + case 4: + { + for (std::size_t i = 0; i < indexcount; ++i) + { + submodel.Indices[i] = e3d_uint32(data); + } + break; + } + default: + { + break; + } + } + continue; + } switch (indexsize) { case 1: @@ -2377,7 +2593,22 @@ void TModel3d::LoadFromBinFile(std::string const &FileName, bool dynamic) { // wczytanie modelu z pliku binarnego WriteLog("Loading binary format 3d model data from \"" + FileName + "\"...", logtype::model); - std::ifstream file(FileName, std::ios::binary); + // the file is read whole in one go and parsed from memory + std::unique_ptr filedata; + std::size_t filesize{0}; + { + std::ifstream input(FileName, std::ios::binary | std::ios::ate); + auto const length{input.tellg()}; + if (length > 0) + { + filedata.reset(new char[static_cast(length)]); + input.seekg(0); + input.read(filedata.get(), length); + filesize = static_cast(input.gcount()); + } + } + e3d_memorybuffer buffer(filedata.get(), filedata.get() + filesize); + std::istream file(&buffer); uint32_t type = sn_utils::ld_uint32(file); uint32_t size = sn_utils::ld_uint32(file) - 8; @@ -2385,7 +2616,6 @@ void TModel3d::LoadFromBinFile(std::string const &FileName, bool dynamic) if (type == MAKE_ID4('E', '3', 'D', '0')) { deserialize(file, size, dynamic); - file.close(); WriteLog("Finished loading 3d model data from \"" + FileName + "\"", logtype::model); } @@ -2393,7 +2623,6 @@ void TModel3d::LoadFromBinFile(std::string const &FileName, bool dynamic) { // throw std::runtime_error("e3d: unknown main chunk"); ErrorLog("Bad model: unknown main chunk in file \"" + FileName + "\"", logtype::model); - file.close(); } };