From 2e631dcbd6993327a03812aa3b66ca3d3a62eca1 Mon Sep 17 00:00:00 2001 From: Hirek193 Date: Tue, 6 Oct 2026 01:55:38 +0200 Subject: [PATCH] feat(terrain): binary terrain files (.btf) made out of text terrain files (.txtf) named by terrain directives, checked against length and checksum of the text and loaded section by section when needed; -btf command line conversion; SBT files no longer generated, only loaded --- CMakeLists.txt | 2 + EU07.cpp | 23 + application/editormode.cpp | 6 + export_btf_standalone.cpp | 41 ++ scene/scene.cpp | 29 + scene/scene.h | 19 + scene/scenenode.h | 1 + scene/sceneterrain.cpp | 858 +++++++++++++++++++++++ scene/sceneterrain.h | 57 ++ simulation/simulationstateserializer.cpp | 115 ++- 10 files changed, 1138 insertions(+), 13 deletions(-) create mode 100644 export_btf_standalone.cpp create mode 100644 scene/sceneterrain.cpp create mode 100644 scene/sceneterrain.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d66759c7..30f6396c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,6 +110,7 @@ set(SOURCES "vehicle/DynObj.cpp" "EU07.cpp" "export_e3d_standalone.cpp" +"export_btf_standalone.cpp" "world/Event.cpp" "world/EvLaunch.cpp" "utilities/Float3d.cpp" @@ -196,6 +197,7 @@ set(SOURCES "scene/scenenodegroups.cpp" "scene/scenelayers.cpp" "scene/scenelayerssave.cpp" +"scene/sceneterrain.cpp" "simulation/simulationenvironment.cpp" "simulation/simulationstateserializer.cpp" "rendering/precipitation.cpp" diff --git a/EU07.cpp b/EU07.cpp index e1be8584..4fd8b8b1 100644 --- a/EU07.cpp +++ b/EU07.cpp @@ -33,6 +33,7 @@ Stele, firleju, szociu, hunter, ZiomalCl, OLI_EU and others #endif void export_e3d_standalone(std::string in, std::string out, int flags, bool dynamic); +int export_btf_standalone(std::string in, std::string out); #include #include @@ -98,6 +99,28 @@ int main(int argc, char *argv[]) int dynamic = std::stoi(std::string(argv[5])); export_e3d_standalone(in, out, flags, dynamic); } + // standalone conversion of text terrain file to binary format: -btf -s [-o ] + else if (argc >= 4 && std::string(argv[1]) == "-btf") + { + std::string in; + std::string out; + for (int i = 2; i + 1 < argc; i += 2) + { + std::string const option(argv[i]); + if (option == "-s") + { + in = argv[i + 1]; + } + else if (option == "-o") + { + out = argv[i + 1]; + } + } + auto const result{export_btf_standalone(in, out)}; + fflush(stdout); + fflush(stderr); + std::_Exit(result); + } else { try diff --git a/application/editormode.cpp b/application/editormode.cpp index d684b13f..ef991c64 100644 --- a/application/editormode.cpp +++ b/application/editormode.cpp @@ -3202,6 +3202,12 @@ void editor_mode::enter() { m_statebackup = {Global.pCamera, FreeFlyModeFlag, Global.ControlPicking}; + if (simulation::Region != nullptr) + { + // the editor works with geometry of the whole scenery, including the parts of binary terrain which weren't needed so far + simulation::Region->load_terrain(); + } + Camera = Global.pCamera; if (!FreeFlyModeFlag) diff --git a/export_btf_standalone.cpp b/export_btf_standalone.cpp new file mode 100644 index 00000000..6d659242 --- /dev/null +++ b/export_btf_standalone.cpp @@ -0,0 +1,41 @@ +#include "stdafx.h" + +#include "scene/sceneterrain.h" +#include "utilities/Globals.h" +#include "utilities/utilities.h" + +// converts text terrain file to binary format. returns: 0 on success, 1 otherwise +int export_btf_standalone(std::string in, std::string out) +{ + if (in.empty()) + { + std::printf("usage: -btf -s [-o ]\n"); + return 1; + } + replace_slashes(in); + replace_slashes(out); + if (false == FileExists(in) && FileExists(Global.asCurrentSceneryPath + in)) + { + // the file can be given the way the sceneries refer to it + in = Global.asCurrentSceneryPath + in; + } + if (out.empty()) + { + // by default the binary file goes where the sceneries expect to find it, next to the text + out = in; + erase_extension(out); + out += ".btf"; + } + std::string message; + auto const result{scene::terrain_file::convert(in, out, &message)}; + // NOTE: the log service doesn't run in this mode, the outcome is reported directly + if (result) + { + std::printf("%s -> %s: %s\n", in.c_str(), out.c_str(), message.c_str()); + } + else + { + std::printf("%s: conversion failed, %s\n", in.c_str(), message.c_str()); + } + return (result ? 0 : 1); +} diff --git a/scene/scene.cpp b/scene/scene.cpp index d876a7ba..2667c730 100644 --- a/scene/scene.cpp +++ b/scene/scene.cpp @@ -19,6 +19,7 @@ http://mozilla.org/MPL/2.0/. #include "utilities/Timer.h" #include "utilities/Logs.h" #include "scene/sn_utils.h" +#include "scene/sceneterrain.h" #include "rendering/renderer.h" #include "widgets/map_objects.h" @@ -1007,6 +1008,12 @@ basic_section::center( glm::dvec3 Center ) { void basic_section::create_geometry() { + if( false == m_terrain.empty() ) { + // this is the first thing done with a section about to be drawn or modified, which makes it + // the moment for the geometry kept in binary terrain files to arrive + load_terrain(); + } + if( true == m_geometrycreated ) { return; } else { // mark it done for future checks @@ -1026,6 +1033,18 @@ basic_section::create_geometry() { } } +// loads geometry of the section kept in binary terrain files, if any is waiting +void +basic_section::load_terrain() { + + // the list is emptied first, as loading can lead back here + auto const blocks { std::move( m_terrain ) }; + m_terrain.clear(); + for( auto const &block : blocks ) { + terrain_file::load( *this, block ); + } +} + void basic_section::create_map_geometry(const gfx::geometrybank_handle handle) { std::vector lines; @@ -1777,6 +1796,16 @@ void basic_region::create_map_geometry() } } +// loads all geometry kept in binary terrain files which wasn't needed so far +void basic_region::load_terrain() +{ + for( auto *section : m_sections ) { + if( section != nullptr ) { + section->load_terrain(); + } + } +} + void basic_region::update_poi_geometry() { std::vector vertices; diff --git a/scene/scene.h b/scene/scene.h index 9e07982a..74b3ba4c 100644 --- a/scene/scene.h +++ b/scene/scene.h @@ -45,6 +45,8 @@ struct scratch_data { std::vector terrain_files; // files named by terrain directives; each one is loaded once std::size_t geometry_imported{ 0 }; // pieces of static geometry read from the text definitions so far std::size_t geometry_skipped{ 0 }; // pieces of static geometry left out so far, as provided by binary terrain + std::vector terrain_textfiles; // terrain files (.txtf with binary .btf version) named by terrain directives so far + std::vector terrain_binaryfiles; // binary versions of these, to be put to use once the scenery is initialized } binary; struct location_data { @@ -78,6 +80,16 @@ struct scratch_data { bool time_initialized { false }; }; +struct terrain_source; + +// piece of a binary terrain file holding geometry of a single section, waiting to be loaded +struct terrain_block { + + std::shared_ptr source; // the file and data shared by its pieces + std::uint64_t offset { 0 }; // location of the piece in the file + std::uint32_t size { 0 }; +}; + // basic element of rudimentary partitioning scheme for the section. fixed size, no further subdivision // TBD, TODO: replace with quadtree scheme? class basic_cell { @@ -346,6 +358,9 @@ public: // generates renderable version of held non-instanced geometry void create_geometry(); + // loads geometry of the section kept in binary terrain files, if any is waiting + void + load_terrain(); void create_map_geometry(const gfx::geometrybank_handle handle); void @@ -374,6 +389,7 @@ public: // content cell_array m_cells; // partitioning scheme shapenode_sequence m_shapes; // large pieces of opaque geometry and (legacy) terrain + std::vector m_terrain; // geometry in binary terrain files, loaded when the section comes into use // TODO: implement dedicated, higher fidelity, fixed resolution terrain mesh item // gfx renderer data gfx::geometrybank_handle m_geometrybank; @@ -481,6 +497,9 @@ public: create_map_geometry(); void update_poi_geometry(); + // loads all geometry kept in binary terrain files which wasn't needed so far + void + load_terrain(); basic_section* get_section(size_t section) { return m_sections[section]; } gfx::geometrybank_handle diff --git a/scene/scenenode.h b/scene/scenenode.h index 7d1db424..dd844a65 100644 --- a/scene/scenenode.h +++ b/scene/scenenode.h @@ -79,6 +79,7 @@ class shape_node { friend class basic_region; // region might want to modify node content when it's being inserted + friend class terrain_file; // binary terrain files hold complete content of the nodes public: // types diff --git a/scene/sceneterrain.cpp b/scene/sceneterrain.cpp new file mode 100644 index 00000000..afee3eda --- /dev/null +++ b/scene/sceneterrain.cpp @@ -0,0 +1,858 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#include "stdafx.h" +#include "scene/sceneterrain.h" + +#include "scene/scene.h" +#include "rendering/renderer.h" +#include "rendering/nullrenderer.h" +#include "utilities/Globals.h" +#include "utilities/Logs.h" +#include "utilities/parser.h" +#include "utilities/utilities.h" + +#include +#include +#include +#include +#include +#include + +/* +binary terrain file layout, all values little endian: + + uint32 file id ('EU07') + uint32 format id ('BTF1') + uint64 length of the source text file + uint32 checksum (crc-32) of the source text file + uint32 section size, uint32 cell size, uint32 number of sections along the side of the region + uint32 material count + uint32 section count + uint64 vertex count + material names, null-terminated + section directory, per section: + uint32 section index, uint64 offset of the section data in the file, uint32 size of the section data, float32 bounding radius of the section + section data, per section: + uint32 block id ('BTFS') + uint32 section index + uint32 shape count + per shape: + uint32 material index + uint32 index of the section cell the shape is located in + float64 squared minimum and maximum visibility range + float32 x12 diffuse, ambient and specular colours + float64 x3 centre, float32 bounding radius + uint32 vertex count + per vertex: float32 x3 position relative to the centre of the section, float32 x3 normal, float32 x2 texture coordinates + +the shapes are the ones the scenery loader would put in the section cells for the same text. where the loader would merge them +further into section-wide pieces depends on the materials, which are known only at run time, so that part is left for loading. +*/ + +namespace scene +{ + +// defined along with the scenery loader code which filters out legacy switch trackbeds +extern std::vector switchtrackbedtextures; + +// data shared by the pieces of a binary terrain file put to use +struct terrain_source +{ + struct material + { + material_handle handle{null_handle}; + bool translucent{false}; + bool clamps{false}; // texture coordinates are kept off the edges of the texture + bool clampt{false}; + bool skip{false}; // geometry using the material isn't loaded + }; + std::string file; + std::vector materials; +}; + +namespace +{ + +std::uint32_t const EU07_FILEHEADER{MAKE_ID4('E', 'U', '0', '7')}; +std::uint32_t const EU07_FILEVERSION_TERRAIN{MAKE_ID4('B', 'T', 'F', '1')}; +std::uint32_t const EU07_TERRAINBLOCK{MAKE_ID4('B', 'T', 'F', 'S')}; +std::size_t const EU07_TERRAINVERTEXSIZE{8 * sizeof(float)}; + +struct file_header +{ + std::uint64_t sourcesize{0}; + std::uint32_t sourcechecksum{0}; + std::uint32_t materialcount{0}; + std::uint32_t sectioncount{0}; + std::uint64_t vertexcount{0}; +}; + +// adds value to the data in the byte order of the file +template void put(std::string &Output, Type_ Value) +{ + char bytes[sizeof(Type_)]; + std::memcpy(bytes, &Value, sizeof(Type_)); + if (std::endian::native != std::endian::little) + { + std::reverse(std::begin(bytes), std::end(bytes)); + } + Output.append(bytes, sizeof(Type_)); +} + +// extracts values in the byte order of the file from a block of data, keeping track of its bounds +class data_reader +{ + public: + data_reader(char const *Data, std::size_t const Size) : m_data(Data), m_size(Size) {} + template Type_ get() + { + Type_ value{}; + if (false == has(sizeof(Type_))) + { + m_good = false; + m_position = m_size; + return value; + } + char bytes[sizeof(Type_)]; + std::memcpy(bytes, m_data + m_position, sizeof(Type_)); + if (std::endian::native != std::endian::little) + { + std::reverse(std::begin(bytes), std::end(bytes)); + } + std::memcpy(&value, bytes, sizeof(Type_)); + m_position += sizeof(Type_); + return value; + } + std::string get_string() + { + std::string text; + while (has(1) && m_data[m_position] != 0) + { + text += m_data[m_position++]; + } + if (false == has(1)) + { + m_good = false; + return text; + } + ++m_position; // terminator + return text; + } + // extracts a series of values at once. NOTE: the caller is expected to check the data is there + void get(float *Values, std::size_t const Count) + { + if (false == has(Count * sizeof(float))) + { + m_good = false; + m_position = m_size; + return; + } + if (std::endian::native == std::endian::little) + { + std::memcpy(Values, m_data + m_position, Count * sizeof(float)); + m_position += Count * sizeof(float); + } + else + { + for (std::size_t index{0}; index < Count; ++index) + { + Values[index] = get(); + } + } + } + void skip(std::size_t const Size) + { + if (false == has(Size)) + { + m_good = false; + m_position = m_size; + return; + } + m_position += Size; + } + bool has(std::size_t const Size) const + { + return Size <= m_size - m_position; + } + bool good() const + { + return m_good; + } + + private: + char const *m_data; + std::size_t m_size; + std::size_t m_position{0}; + bool m_good{true}; +}; + +// loads content of specified file into provided string. returns: true on success, false otherwise +bool read_file(std::string const &Filename, std::string &Data) +{ + std::ifstream input{Filename, std::ios::binary}; + if (input.fail()) + { + return false; + } + auto const size{input.rdbuf()->pubseekoff(0, std::ios_base::end)}; + input.rdbuf()->pubseekoff(0, std::ios_base::beg); + if (size < 0 || static_cast(size) >= Data.max_size()) + { + return false; + } + Data.resize(static_cast(size)); + input.read(Data.data(), static_cast(Data.size())); + return static_cast(std::max(0, input.gcount())) == Data.size(); +} + +// loads the fixed part of binary terrain file header. returns: true if the file can be used, false otherwise +bool read_header(std::istream &Input, file_header &Header) +{ + char bytes[48]; + Input.read(bytes, sizeof(bytes)); + if (Input.gcount() != static_cast(sizeof(bytes))) + { + return false; + } + data_reader data{bytes, sizeof(bytes)}; + if (data.get() != EU07_FILEHEADER || data.get() != EU07_FILEVERSION_TERRAIN) + { + // wrong file type + return false; + } + Header.sourcesize = data.get(); + Header.sourcechecksum = data.get(); + // the content is arranged for specific layout of the scene, and can't be used with any other + if (data.get() != static_cast(EU07_SECTIONSIZE) || data.get() != static_cast(EU07_CELLSIZE) || + data.get() != static_cast(EU07_REGIONSIDESECTIONCOUNT)) + { + return false; + } + Header.materialcount = data.get(); + Header.sectioncount = data.get(); + Header.vertexcount = data.get(); + return data.good(); +} + +bool read_header(std::string const &Filename, file_header &Header) +{ + std::ifstream input{Filename, std::ios::binary}; + return (false == input.fail()) && read_header(input, Header); +} + +// stand-in for the renderer used during conversion of text terrain. the conversion doesn't need +// anything of the materials but a way to tell them apart, so it goes by their names alone +class conversion_renderer : public null_renderer +{ + public: + material_handle Fetch_Material(std::string const &Filename, bool const Loadnow = true) override + { + // the names are brought to the form the material manager would look them up by + auto name{Filename}; + if (name.find("make:") != 0 && name.find("internal_src:") != 0) + { + if (contains(name, '|')) + { + name.erase(name.find('|')); + } + erase_extension(name); + replace_slashes(name); + erase_leading_slashes(name); + } + auto const lookup{m_lookup.find(name)}; + if (lookup != m_lookup.end()) + { + return lookup->second; + } + m_materials.emplace_back(std::make_unique()); + m_materials.back()->name = name; + m_names.emplace_back(name); + m_lookup.emplace(name, static_cast(m_materials.size())); + return static_cast(m_materials.size()); + } + IMaterial const *Material(material_handle const Material) const override + { + return (Material > 0 && static_cast(Material) <= m_materials.size() ? m_materials[Material - 1].get() : IMaterial::null_material()); + } + // names of the materials, in the order of their handles, starting with handle 1 + std::vector const &names() const + { + return m_names; + } + + private: + std::vector> m_materials; + std::vector m_names; + std::unordered_map m_lookup; +}; + +// puts the stand-in renderer in place of the real one for as long as the object lives. +// the conversion is done by the code which loads scenery geometry from text, and that code asks the global renderer for the materials +class conversion_scope +{ + public: + conversion_scope() : m_trackbeds(Global.CreateSwitchTrackbeds) + { + auto renderer{std::make_unique()}; + m_materials = renderer.get(); + m_renderer = std::move(renderer); + std::swap(GfxRenderer, m_renderer); + // the binary file holds everything the text does. geometry of legacy switch trackbeds is filtered out on load, as that depends on the settings + Global.CreateSwitchTrackbeds = false; + } + ~conversion_scope() + { + std::swap(GfxRenderer, m_renderer); + Global.CreateSwitchTrackbeds = m_trackbeds; + } + conversion_scope(conversion_scope const &) = delete; + conversion_scope &operator=(conversion_scope const &) = delete; + std::vector const &materials() const + { + return m_materials->names(); + } + + private: + std::unique_ptr m_renderer; + conversion_renderer *m_materials{nullptr}; + bool m_trackbeds; +}; + +// collects what the loader needs to know about the material with specified name +terrain_source::material fetch_material(std::string const &Name) +{ + terrain_source::material material; + material.handle = GfxRenderer->Fetch_Material(Name); + // the appearance is determined from the assigned diffuse texture, the same way it's done for the geometry loaded from text + auto const texturehandle = material.handle != null_handle ? GfxRenderer->Material(material.handle)->GetTexture(0) : null_handle; + auto const &texture = texturehandle ? GfxRenderer->Texture(texturehandle) : *ITexture::null_texture(); + material.clamps = texturehandle ? contains(texture.get_traits(), 's') : false; + material.clampt = texturehandle ? contains(texture.get_traits(), 't') : false; + material.translucent = (texturehandle != null_handle) && contains(texture.get_name(), '@') && (true == texture.get_has_alpha()); + if (Global.CreateSwitchTrackbeds) + { + // geometry with blacklisted texture, part of old switch trackbed + auto const materialname{GfxRenderer->Material(material.handle)->GetName()}; + material.skip = + std::any_of(std::begin(switchtrackbedtextures), std::end(switchtrackbedtextures), [&](std::string const &Texture) { return contains(materialname, Texture); }); + } + return material; +} + +} // namespace + +// calculates checksum (crc-32) of provided data +std::uint32_t terrain_file::checksum(char const *Data, std::size_t Size) +{ + // lookup for 8 bytes of input at a time + static auto const tables{[]() { + std::array, 8> lookup{}; + for (std::uint32_t index{0}; index < 256; ++index) + { + auto value{index}; + for (auto bit{0}; bit < 8; ++bit) + { + value = (value & 1 ? 0xEDB88320u ^ (value >> 1) : value >> 1); + } + lookup[0][index] = value; + } + for (std::uint32_t index{0}; index < 256; ++index) + { + for (std::size_t table{1}; table < lookup.size(); ++table) + { + lookup[table][index] = (lookup[table - 1][index] >> 8) ^ lookup[0][lookup[table - 1][index] & 0xFF]; + } + } + return lookup; + }()}; + + auto const *data{reinterpret_cast(Data)}; + std::uint32_t crc{0xFFFFFFFFu}; + while (Size >= 8) + { + auto const low{crc ^ (static_cast(data[0]) | (static_cast(data[1]) << 8) | (static_cast(data[2]) << 16) | (static_cast(data[3]) << 24))}; + crc = tables[7][low & 0xFF] ^ tables[6][(low >> 8) & 0xFF] ^ tables[5][(low >> 16) & 0xFF] ^ tables[4][low >> 24] ^ tables[3][data[4]] ^ tables[2][data[5]] ^ tables[1][data[6]] ^ tables[0][data[7]]; + data += 8; + Size -= 8; + } + while (Size-- > 0) + { + crc = tables[0][(crc ^ *data++) & 0xFF] ^ (crc >> 8); + } + return crc ^ 0xFFFFFFFFu; +} + +// converts provided content of a text terrain file to binary format and stores it in specified file. returns: true on success, false otherwise +bool terrain_file::write(std::string const &Text, std::string const &Binaryfile, std::string &Message) +{ + conversion_scope scope; + auto region{std::make_unique()}; + scratch_data scratchpad{}; + + { + cParser input{Text, cParser::buffer_TEXT}; + // content of the files the text would include isn't covered by its checksum, so there can't be any + input.expandIncludes = false; + auto token{input.getToken()}; + while (false == token.empty()) + { + if (token == "node") + { + node_data nodedata; + input.getTokens(4); + input >> nodedata.range_max >> nodedata.range_min >> nodedata.name >> nodedata.type; + if (nodedata.type != "triangles" && nodedata.type != "triangle_strip" && nodedata.type != "triangle_fan") + { + Message = "node of type \"" + nodedata.type + "\" (line " + std::to_string(input.Line()) + ") can't be a part of binary terrain"; + return false; + } + auto shape{shape_node().import(input, nodedata)}; + // the shapes are all sent to the section cells, which groups them the way it'd be done for translucent geometry. + // whether they can be merged into section-wide pieces instead is decided when the file is loaded, see the layout notes + shape.m_data.translucent = true; + region->insert(shape, scratchpad, true); + } + else if (token == "origin") + { + glm::dvec3 offset; + input.getTokens(3); + input >> offset.x >> offset.y >> offset.z; + scratchpad.location.offset.emplace(offset + (scratchpad.location.offset.empty() ? glm::dvec3() : scratchpad.location.offset.top())); + } + else if (token == "endorigin") + { + if (false == scratchpad.location.offset.empty()) + { + scratchpad.location.offset.pop(); + } + } + else if (token == "rotate") + { + input.getTokens(3); + input >> scratchpad.location.rotation.x >> scratchpad.location.rotation.y >> scratchpad.location.rotation.z; + } + else + { + Message = "\"" + token + "\" (line " + std::to_string(input.Line()) + ") can't be a part of binary terrain"; + return false; + } + token = input.getToken(); + } + } + + // sections holding any geometry + std::vector sections; + for (std::size_t index{0}; index < region->m_sections.size(); ++index) + { + auto const *section{region->m_sections[index]}; + if (section == nullptr) + { + continue; + } + if (std::any_of(std::begin(section->m_cells), std::end(section->m_cells), [](basic_cell const &Cell) { return false == Cell.m_shapestranslucent.empty(); })) + { + sections.emplace_back(static_cast(index)); + } + } + + auto const temporaryfile{Binaryfile + ".tmp"}; + std::uint64_t shapecount{0}; + std::uint64_t vertexcount{0}; + { + std::ofstream output{temporaryfile, std::ios::binary | std::ios::trunc}; + if (output.fail()) + { + Message = "can't write to file \"" + temporaryfile + "\""; + return false; + } + + std::string data; + put(data, EU07_FILEHEADER); + put(data, EU07_FILEVERSION_TERRAIN); + put(data, static_cast(Text.size())); + put(data, terrain_file::checksum(Text.data(), Text.size())); + put(data, static_cast(EU07_SECTIONSIZE)); + put(data, static_cast(EU07_CELLSIZE)); + put(data, static_cast(EU07_REGIONSIDESECTIONCOUNT)); + put(data, static_cast(scope.materials().size())); + put(data, static_cast(sections.size())); + auto const vertexcountposition{data.size()}; + put(data, vertexcount); // filled in when the count is known + for (auto const &material : scope.materials()) + { + data.append(material.c_str(), material.size() + 1); + } + auto const directoryposition{data.size()}; + // the directory is written once the section data is in place and its location known + std::size_t const directoryentrysize{sizeof(std::uint32_t) + sizeof(std::uint64_t) + sizeof(std::uint32_t) + sizeof(float)}; + data.append(sections.size() * directoryentrysize, '\0'); + output.write(data.data(), static_cast(data.size())); + std::uint64_t offset{data.size()}; + + std::string directory; + for (auto const sectionindex : sections) + { + auto *section{region->m_sections[sectionindex]}; + data.clear(); + put(data, EU07_TERRAINBLOCK); + put(data, sectionindex); + std::uint32_t sectionshapecount{0}; + for (auto const &cell : section->m_cells) + { + sectionshapecount += static_cast(cell.m_shapestranslucent.size()); + } + put(data, sectionshapecount); + for (std::size_t cellindex{0}; cellindex < section->m_cells.size(); ++cellindex) + { + for (auto &shape : section->m_cells[cellindex].m_shapestranslucent) + { + auto const radius{shape.radius()}; + auto const &shapedata{shape.data()}; + put(data, static_cast(shapedata.material - 1)); + put(data, static_cast(cellindex)); + put(data, shapedata.rangesquared_min); + put(data, shapedata.rangesquared_max); + for (auto const &colour : {shapedata.lighting.diffuse, shapedata.lighting.ambient, shapedata.lighting.specular}) + { + put(data, colour.r); + put(data, colour.g); + put(data, colour.b); + put(data, colour.a); + } + put(data, shapedata.area.center.x); + put(data, shapedata.area.center.y); + put(data, shapedata.area.center.z); + put(data, radius); + put(data, static_cast(shapedata.vertices.size())); + for (auto const &vertex : shapedata.vertices) + { + glm::vec3 const position{vertex.position - section->m_area.center}; + put(data, position.x); + put(data, position.y); + put(data, position.z); + put(data, vertex.normal.x); + put(data, vertex.normal.y); + put(data, vertex.normal.z); + put(data, vertex.texture.x); + put(data, vertex.texture.y); + } + ++shapecount; + vertexcount += shapedata.vertices.size(); + } + } + if (data.size() > std::numeric_limits::max()) + { + Message = "too much geometry in a single section of the scene"; + output.close(); + std::error_code error; + std::filesystem::remove(temporaryfile, error); + return false; + } + output.write(data.data(), static_cast(data.size())); + put(directory, sectionindex); + put(directory, offset); + put(directory, static_cast(data.size())); + put(directory, section->m_area.radius); + offset += data.size(); + } + // fill in the parts left for later + output.seekp(static_cast(directoryposition)); + output.write(directory.data(), static_cast(directory.size())); + data.clear(); + put(data, vertexcount); + output.seekp(static_cast(vertexcountposition)); + output.write(data.data(), static_cast(data.size())); + output.flush(); + if (output.fail()) + { + Message = "can't write to file \"" + temporaryfile + "\""; + output.close(); + std::error_code error; + std::filesystem::remove(temporaryfile, error); + return false; + } + } + // the finished file replaces the previous version in one go, so an interrupted conversion doesn't leave a broken one behind + std::error_code error; + std::filesystem::rename(temporaryfile, Binaryfile, error); + if (error) + { + Message = "can't write to file \"" + Binaryfile + "\""; + std::filesystem::remove(temporaryfile, error); + return false; + } + + Message = std::to_string(sections.size()) + " section(s), " + std::to_string(shapecount) + " shape(s), " + std::to_string(vertexcount / 3) + " triangle(s)"; + return true; +} + +// converts specified text terrain file to binary format. returns: true on success, false otherwise +bool terrain_file::convert(std::string const &Textfile, std::string const &Binaryfile, std::string *Message) +{ + std::string message; + std::string text; + auto result{false}; + if (read_file(Textfile, text)) + { + result = write(text, Binaryfile, message); + } + else + { + message = "can't read the file"; + } + if (Message != nullptr) + { + *Message = message; + } + return result; +} + +// makes sure binary version of specified text terrain file is in place and matches the text +terrain_file::state terrain_file::prepare(std::string const &Textfile, std::string const &Binaryfile) +{ + file_header header; + auto const binarypresent{read_header(Binaryfile, header)}; + + std::error_code error; + auto const textsize{std::filesystem::file_size(Textfile, error)}; + if (error) + { + // with no text to check the binary file against it's taken as it is + return (binarypresent ? state::binary : state::missing); + } + + std::string text; + auto textloaded{false}; + if (binarypresent && (header.sourcesize == textsize)) + { + // the length is the same; the checksum tells whether the content is, too + textloaded = read_file(Textfile, text); + if (textloaded && (text.size() == header.sourcesize) && (checksum(text.data(), text.size()) == header.sourcechecksum)) + { + return state::binary; + } + } + if (false == textloaded && false == read_file(Textfile, text)) + { + ErrorLog("Bad file: can't read terrain file \"" + Textfile + "\""); + // NOTE: binary file which doesn't match the text isn't used, the scenery is expected to show what its files say + return state::missing; + } + + WriteLog("Terrain file \"" + Textfile + "\": " + (binarypresent ? "binary version doesn't match the text" : "no usable binary version found") + ", converting..."); + std::string message; + if (write(text, Binaryfile, message)) + { + WriteLog("Terrain file \"" + Textfile + "\": conversion done, " + message); + return state::binary; + } + ErrorLog("Bad file: terrain file \"" + Textfile + "\" wasn't converted to binary format, " + message + ". The file will be loaded as text"); + if (binarypresent) + { + // the binary file left behind is of no use, and would be taken for the real thing if the text went missing + std::filesystem::remove(Binaryfile, error); + } + return state::text; +} + +// makes content of specified binary terrain file a part of provided region +bool terrain_file::attach(std::string const &Binaryfile, basic_region &Region, bool Deferred) +{ + std::ifstream input{Binaryfile, std::ios::binary}; + file_header header; + if (input.fail() || false == read_header(input, header)) + { + ErrorLog("Bad file: \"" + Binaryfile + "\" is of either unrecognized type or version"); + return false; + } + auto source{std::make_shared()}; + source->file = Binaryfile; + source->materials.reserve(header.materialcount); + for (std::uint32_t index{0}; index < header.materialcount; ++index) + { + std::string name; + if (false == static_cast(std::getline(input, name, '\0'))) + { + break; + } + source->materials.emplace_back(fetch_material(name)); + } + + std::string data(header.sectioncount * (sizeof(std::uint32_t) + sizeof(std::uint64_t) + sizeof(std::uint32_t) + sizeof(float)), '\0'); + input.read(data.data(), static_cast(data.size())); + data_reader reader{data.data(), static_cast(std::max(0, input.gcount()))}; + + std::uint32_t sectioncount{0}; + for (std::uint32_t index{0}; index < header.sectioncount && reader.good(); ++index) + { + auto const sectionindex{reader.get()}; + terrain_block block; + block.source = source; + block.offset = reader.get(); + block.size = reader.get(); + auto const radius{reader.get()}; + if (false == reader.good() || sectionindex >= Region.m_sections.size()) + { + break; + } + // location of the section follows from its index, see basic_region::section() + auto const column{static_cast(sectionindex % EU07_REGIONSIDESECTIONCOUNT)}; + auto const row{static_cast(sectionindex / EU07_REGIONSIDESECTIONCOUNT)}; + auto const centeroffset{-(EU07_REGIONSIDESECTIONCOUNT / 2 * EU07_SECTIONSIZE) + EU07_SECTIONSIZE / 2}; + auto §ion{Region.section(glm::dvec3{static_cast(centeroffset + column * EU07_SECTIONSIZE), 0.0, static_cast(centeroffset + row * EU07_SECTIONSIZE)})}; + // the section has to be seen as reaching as far as its geometry does, before that geometry is loaded + section.m_area.radius = std::max(section.m_area.radius, radius); + section.m_terrain.emplace_back(block); + if (false == Deferred) + { + section.load_terrain(); + } + ++sectioncount; + } + if (false == reader.good() || sectioncount != header.sectioncount || source->materials.size() != header.materialcount) + { + ErrorLog("Bad file: \"" + Binaryfile + "\" is damaged, part of its content may be missing"); + } + WriteLog("Terrain file \"" + Binaryfile + "\": " + std::to_string(sectioncount) + " section(s), " + std::to_string(header.vertexcount / 3) + " triangle(s)" + + (Deferred ? ", loaded when needed" : "")); + return true; +} + +// loads specified piece of a binary terrain file into provided section +void terrain_file::load(basic_section &Section, terrain_block const &Block) +{ + if (Block.source == nullptr) + { + return; + } + auto const &source{*Block.source}; + + std::string data(Block.size, '\0'); + { + std::ifstream input{source.file, std::ios::binary}; + input.seekg(static_cast(Block.offset)); + input.read(data.data(), static_cast(data.size())); + if (input.fail() || static_cast(std::max(0, input.gcount())) != data.size()) + { + ErrorLog("Bad file: failed to load terrain section from \"" + source.file + "\""); + return; + } + } + data_reader reader{data.data(), data.size()}; + if (reader.get() != EU07_TERRAINBLOCK) + { + // the file isn't what it was when its directory was read + ErrorLog("Bad file: failed to load terrain section from \"" + source.file + "\", the file was changed"); + return; + } + reader.get(); // section index + auto shapecount{reader.get()}; + + // large pieces of opaque geometry are held by the section itself, merged by their appearance. see basic_section::insert() + std::vector sectionshapes; + auto const place = [&](shape_node &Shape, std::vector &Shapes, glm::dvec3 const &Origin) { + Shape.origin(Origin); + if (true == Section.m_geometrycreated) + { + // the section was put to use already, it won't prepare the geometry on its own + Shape.create_geometry(Section.m_geometrybank); + } + Shapes.emplace_back(std::move(Shape)); + }; + + while (shapecount-- > 0 && reader.good()) + { + auto const materialindex{reader.get()}; + auto const cellindex{reader.get()}; + shape_node shape; + auto &shapedata{shape.m_data}; + shapedata.rangesquared_min = reader.get(); + shapedata.rangesquared_max = reader.get(); + for (auto *colour : {&shapedata.lighting.diffuse, &shapedata.lighting.ambient, &shapedata.lighting.specular}) + { + colour->r = reader.get(); + colour->g = reader.get(); + colour->b = reader.get(); + colour->a = reader.get(); + } + shapedata.area.center.x = reader.get(); + shapedata.area.center.y = reader.get(); + shapedata.area.center.z = reader.get(); + shapedata.area.radius = reader.get(); + auto const vertexcount{reader.get()}; + if (false == reader.good() || false == reader.has(static_cast(vertexcount) * EU07_TERRAINVERTEXSIZE) || materialindex >= source.materials.size() || + cellindex >= Section.m_cells.size()) + { + ErrorLog("Bad file: terrain section data in \"" + source.file + "\" is damaged"); + break; + } + auto const &material{source.materials[materialindex]}; + if (true == material.skip) + { + reader.skip(static_cast(vertexcount) * EU07_TERRAINVERTEXSIZE); + continue; + } + shapedata.material = material.handle; + shapedata.translucent = material.translucent; + shapedata.vertices.resize(vertexcount); + for (auto &vertex : shapedata.vertices) + { + float values[8]; + reader.get(values, 8); + vertex.position = Section.m_area.center + glm::dvec3{values[0], values[1], values[2]}; + vertex.normal = glm::vec3{values[3], values[4], values[5]}; + vertex.texture = glm::vec2{values[6], values[7]}; + // clamp texture coordinates if texture wrapping is off + // NOTE: for the text this is done before a large triangle gets divided along the section borders, here it comes after. + // the coordinates of the vertices added by the division can thus differ, by the clamp margin for a texture mapped the way clamping expects + if (true == material.clamps) + { + vertex.texture.s = std::clamp(vertex.texture.s, 0.001f, 0.999f); + } + if (true == material.clampt) + { + vertex.texture.t = std::clamp(vertex.texture.t, 0.001f, 0.999f); + } + } + + if (true == shapedata.translucent || shapedata.rangesquared_max <= 90000.0 || shapedata.rangesquared_min > 0.0) + { + // small, translucent or not always visible shapes are placed in the sub-cells + auto &cell{Section.m_cells[cellindex]}; + cell.m_active = true; + cell.m_area.radius = std::max(cell.m_area.radius, static_cast(glm::length(cell.m_area.center - shapedata.area.center) + shapedata.area.radius)); + place(shape, (shapedata.translucent ? cell.m_shapestranslucent : cell.m_shapesopaque), cell.m_area.center); + } + else + { + auto merged{false}; + for (auto &target : sectionshapes) + { + if (true == target.merge(shape)) + { + merged = true; + break; + } + } + if (false == merged) + { + sectionshapes.emplace_back(std::move(shape)); + } + } + } + for (auto &shape : sectionshapes) + { + place(shape, Section.m_shapes, Section.m_area.center); + } +} + +} // namespace scene + +//--------------------------------------------------------------------------- diff --git a/scene/sceneterrain.h b/scene/sceneterrain.h new file mode 100644 index 00000000..773cb821 --- /dev/null +++ b/scene/sceneterrain.h @@ -0,0 +1,57 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +#include +#include + +namespace scene +{ + +class basic_region; +class basic_section; +struct terrain_block; + +// binary terrain file (.btf): static geometry of a text terrain file (.txtf), arranged by the sections +// of the scene so each can be loaded on its own, when it comes into use. +// the binary file records length and checksum of the text it was made from, and is made anew when these don't match +class terrain_file +{ + public: + // types + // what a scenery is left with for a terrain file it refers to + enum class state + { + binary, // binary file is in place and up to date + text, // binary file can't be had, the text file has to be loaded the way any other scenery file is + missing // there's nothing to load + }; + // methods + // calculates checksum (crc-32) of provided data + static std::uint32_t checksum(char const *Data, std::size_t Size); + // converts specified text terrain file to binary format. returns: true on success, false otherwise. + // description of the result or of the failure is stored in Message, if provided + static bool convert(std::string const &Textfile, std::string const &Binaryfile, std::string *Message = nullptr); + // makes sure binary version of specified text terrain file is in place and matches the text + static state prepare(std::string const &Textfile, std::string const &Binaryfile); + // makes content of specified binary terrain file a part of provided region. the geometry is loaded right away, + // or left for the sections of the region to load when they're about to be used. returns: true on success, false otherwise + static bool attach(std::string const &Binaryfile, basic_region &Region, bool Deferred); + // loads specified piece of a binary terrain file into provided section + static void load(basic_section &Section, terrain_block const &Block); + + private: + // converts provided content of a text terrain file to binary format and stores it in specified file. returns: true on success, false otherwise + static bool write(std::string const &Text, std::string const &Binaryfile, std::string &Message); +}; + +} // namespace scene + +//--------------------------------------------------------------------------- diff --git a/simulation/simulationstateserializer.cpp b/simulation/simulationstateserializer.cpp index ff2ef26a..f6eb9666 100644 --- a/simulation/simulationstateserializer.cpp +++ b/simulation/simulationstateserializer.cpp @@ -17,6 +17,7 @@ http://mozilla.org/MPL/2.0/. #include "simulation/simulationenvironment.h" #include "scene/scenenodegroups.h" #include "scene/scenelayers.h" +#include "scene/sceneterrain.h" #include "rendering/particles.h" #include "world/Event.h" #include "world/MemCell.h" @@ -173,16 +174,10 @@ state_serializer::deserialize_continue(std::shared_ptr state scene::Groups.update_map(); Region->create_map_geometry(); - if( true == Global.file_binary_terrain - && false == state->scratchpad.binary.terrain - && state->scenariofile != "$.scn" ) { - // if we didn't find usable binary version of the scenario files, create them now for future use - // as long as the scenario file wasn't rainsted-created base file override - Region->serialize( state->scenariofile ); - } + // NOTE: legacy binary terrain files (.sbt) are no longer generated, only loaded if the scenery comes with them. + // binary terrain is made out of terrain files named by terrain directives instead, see scene::terrain_file - // geometry of the roads is generated on each load instead of being kept in the binary terrain file, - // so it's inserted in the region only after that file had its chance to be written + // geometry of the roads is generated on each load simulation::Roads.create_geometry( Scratchpad ); simulation::Junctions.create_geometry( Scratchpad ); simulation::Roadpoints.create_geometry(); @@ -410,6 +405,12 @@ state_serializer::deserialize_firstinit( cParser &Input, scene::scratch_data &Sc } } + // binary versions of terrain files only announce what they hold at this point, the sections of the scene load it when they need it. + // scenery opened for editing gets it all at once, the editor works with complete geometry + for( auto const &terrainfile : Scratchpad.binary.terrain_binaryfiles ) { + scene::terrain_file::attach( terrainfile, *Region, false == Global.editor_session ); + } + Scratchpad.binary.terrain_binaryfiles.clear(); simulation::Paths.InitTracks(); // the roads tie up their own lanes where they gain or lose some, the junctions tie the lanes of the roads together, @@ -991,6 +992,11 @@ include_binary_terrain( std::string File, scene::scratch_data &Scratchpad ) { auto &binary { Scratchpad.binary }; replace_slashes( File ); + if( false == binary.terrain_textfiles.empty() ) { + // the scenery keeps its terrain in files of the current format, which don't mix with a file holding all of its geometry + WriteLog( "Included SBT file: " + File + " ignored, the scenery uses terrain files" ); + return; + } if( std::find( binary.terrain_files.begin(), binary.terrain_files.end(), File ) != binary.terrain_files.end() ) { WriteLog( "Included SBT file: " + File + " is already in use, ignored" ); return; @@ -1049,6 +1055,85 @@ include_binary_terrain( std::string File, scene::scratch_data &Scratchpad ) { } } +// processes terrain file named by a terrain directive: static geometry kept as text (.txtf), with binary version (.btf) +// made out of it and loaded in its place. unlike the legacy binary terrain above, the binary file stands in +// only for the text file of the same name, and has no bearing on how the rest of the scenery is loaded +void +include_terrain_file( std::string File, cParser &Input, scene::scratch_data &Scratchpad ) { + + auto &binary { Scratchpad.binary }; + + replace_slashes( File ); + erase_extension( File ); + if( std::find( binary.terrain_textfiles.begin(), binary.terrain_textfiles.end(), File ) != binary.terrain_textfiles.end() ) { + WriteLog( "Terrain file: " + File + " is already in use, ignored" ); + return; + } + binary.terrain_textfiles.emplace_back( File ); + + auto const textfile { Global.asCurrentSceneryPath + File + ".txtf" }; + auto const binaryfile { Global.asCurrentSceneryPath + File + ".btf" }; + auto const textpresent { FileExists( textfile ) }; + // binary file holds the geometry where the text alone puts it, it can't stand in for a file placed with an offset or rotation + auto const relocated { + ( false == Scratchpad.location.offset.empty() && Scratchpad.location.offset.top() != glm::dvec3( 0.0 ) ) + || ( Scratchpad.location.rotation != glm::vec3( 0.f ) ) }; + + auto state { scene::terrain_file::state::text }; + if( true == relocated ) { + if( false == textpresent ) { + ErrorLog( "Bad scenario: terrain file \"" + File + "\" placed with an offset or rotation can be loaded only from the text, which is missing" ); + return; + } + } + else if( ( false == textpresent ) + || ( ( false == Global.editor_session ) && ( true == Global.file_binary_terrain ) ) ) { + // NOTE: scenery opened for editing works on its text files, the text is left as is also if binary terrain is turned off. + // a binary file which is all there is gets loaded regardless + state = scene::terrain_file::prepare( textfile, binaryfile ); + } + + switch( state ) { + case scene::terrain_file::state::binary: { + if( true == binary.terrain ) { + // legacy binary terrain file is in use, expected to hold all static geometry of the scenery, this terrain likely included + if( ( false == Scratchpad.initialized ) + && ( binary.geometry_skipped == 0 ) + && ( Global.file_binary_terrain_skipped == 0 ) ) { + // nothing was left out on account of that file yet, so the scenery can still do without it + binary.terrain = false; + binary.terrain_included = false; + binary.terrain_default = false; + binary.terrain_files.clear(); + Global.file_binary_terrain_state = false; + WriteLog( "Terrain file: " + File + " in use, SBT of the scenery is ignored" ); + } + else { + ErrorLog( "Bad scenario: terrain file \"" + File + "\" ignored, the scenery geometry already comes from an SBT file. Remove the SBT file to have the terrain file used" ); + return; + } + } + if( true == Scratchpad.initialized ) { + scene::terrain_file::attach( binaryfile, *simulation::Region, false == Global.editor_session ); + } + else { + binary.terrain_binaryfiles.emplace_back( binaryfile ); + } + break; + } + case scene::terrain_file::state::text: { + // read the way any other scenery file is + WriteLog( "Terrain file: " + File + " loaded as text" ); + Input.injectString( "include \"" + textfile + "\" end" ); + break; + } + default: { + ErrorLog( "Bad scenario: terrain file \"" + File + "\" not found" ); + break; + } + } +} + } // namespace void @@ -1057,13 +1142,17 @@ state_serializer::deserialize_terrain(cParser &Input, scene::scratch_data &Scrat std::string line; Input.getTokens(1); Input >> line; - if (Global.file_binary_terrain && line.ends_with(".sbt")) + // NOTE: the directive is read to its end first, as processing of a terrain file can leave content for the parser to go through next + skip_until(Input, "endterrain"); + + if (line.ends_with(".txtf") || line.ends_with(".btf")) + { + include_terrain_file(line, Input, Scratchpad); + } + else if (Global.file_binary_terrain && line.ends_with(".sbt")) { include_binary_terrain(line, Scratchpad); } - - skip_until(Input, "endterrain"); - } void