diff --git a/CMakeLists.txt b/CMakeLists.txt index c0792952..9de66300 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -194,6 +194,7 @@ set(SOURCES "rendering/terraintilepool.cpp" "rendering/terrainground.cpp" "simulation/loadprofile.cpp" +"simulation/cookdeps.cpp" "simulation/logiccook.cpp" "simulation/terrainbake.cpp" "simulation/simulationtime.cpp" diff --git a/application/application.cpp b/application/application.cpp index 47f0f1ea..fa41cb41 100644 --- a/application/application.cpp +++ b/application/application.cpp @@ -1258,6 +1258,13 @@ int eu07_application::init_settings(int Argc, char *Argv[]) // check it against the events in memory Global.cook_logic = true; } + else if (token == "-cook") + { + // prepare the scenery's cooked data - heightfields and logic - wherever it is missing + // or out of date, then quit without starting the simulation + Global.cook_only = true; + Global.bake_terrain = true; + } else if (token == "-nobaketerrain") { // load the scenery's terrain triangles as written, without baking or using a @@ -1281,7 +1288,7 @@ int eu07_application::init_settings(int Argc, char *Argv[]) << " [-loadprofile]" << " [-cooklogic]" << " [-verifylogic]" - << " [-nobaketerrain]" + << " [-nobaketerrain] [-cook]" << " (every switch also takes a double dash: --editor)" << std::endl; return -1; } diff --git a/application/drivermode.cpp b/application/drivermode.cpp index abf5c6ae..974dda6a 100644 --- a/application/drivermode.cpp +++ b/application/drivermode.cpp @@ -195,6 +195,18 @@ bool driver_mode::update() simulation::State.update_scripting_interface(); simulation::Environment.update(); + // a cooking run is over once the scenery has loaded: whatever was stale was cooked on the + // way in + if (true == Global.cook_only) + { + // closed directly: there is no simulation to wind down, and a quit command would wait + // in the queue for one + WriteLog("Cooking finished, quitting"); + Global.cook_only = false; + Application.queue_quit(true); + return true; + } + // unattended screenshot sequence, when one was asked for on the command line if (true == Shots.active()) { diff --git a/simulation/cookdeps.cpp b/simulation/cookdeps.cpp new file mode 100644 index 00000000..4a723372 --- /dev/null +++ b/simulation/cookdeps.cpp @@ -0,0 +1,149 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +module; +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +module eu07.simulation.cookdeps; +import eu07.utilities.logs; +import eu07.utilities.utilities; + +namespace simulation::cookdeps { + +namespace { + +constexpr char const *magic { "eu07-cookdeps 1" }; + +struct entry { + std::uintmax_t size { 0 }; + std::int64_t time { 0 }; + std::uint64_t hash { 0 }; + std::string path; +}; + +// fnv-1a over the file's bytes. stable across runs and builds, which is what a manifest needs; +// returns false when the file cannot be read +bool +hash_of( std::string const &Path, std::uintmax_t &Size, std::uint64_t &Hash ) { + + std::ifstream input( Path, std::ios::binary ); + if( false == input.good() ) { return false; } + Hash = 0xcbf29ce484222325ull; + Size = 0; + std::array buffer; + while( input ) { + input.read( buffer.data(), static_cast( buffer.size() ) ); + auto const count { static_cast( input.gcount() ) }; + for( std::size_t index = 0; index < count; ++index ) { + Hash ^= static_cast( buffer[ index ] ); + Hash *= 0x100000001b3ull; + } + Size += count; + } + return true; +} + +std::vector +unique( std::vector Files ) { + + std::sort( Files.begin(), Files.end() ); + Files.erase( std::unique( Files.begin(), Files.end() ), Files.end() ); + return Files; +} + +bool +read( std::string const &Path, std::vector &Entries ) { + + std::ifstream input( Path ); + std::string line; + if( ( false == static_cast( std::getline( input, line ) ) ) || ( line != magic ) ) { return false; } + while( std::getline( input, line ) ) { + std::istringstream fields( line ); + entry item; + if( false == static_cast( fields >> item.size >> item.time >> std::hex >> item.hash >> std::dec ) ) { return false; } + fields.get(); // the space before the path, which may itself hold spaces + std::getline( fields, item.path ); + Entries.push_back( item ); + } + return true; +} + +} // anonymous namespace + +bool +write( std::string const &Path, std::vector const &Files ) { + + std::ostringstream text; + text << magic << '\n'; + for( auto const &file : unique( Files ) ) { + entry item; + item.path = file; + if( false == hash_of( file, item.size, item.hash ) ) { + // a file that was read a moment ago and cannot be now; the artifact is recorded as + // depending on its absence, which any later read will contradict + item.size = 0; + item.hash = 0; + } + item.time = static_cast( last_modified( file ) ); + text << item.size << ' ' << item.time << ' ' << std::hex << item.hash << std::dec << ' ' << item.path << '\n'; + } + + std::ofstream output( Path, std::ios::trunc ); + output << text.str(); + if( false == output.good() ) { + ErrorLog( "Cooked data: cannot write \"" + Path + "\"" ); + return false; + } + return true; +} + +bool +current( std::string const &Path ) { + + std::vector entries; + if( false == read( Path, entries ) ) { return false; } + for( auto const &item : entries ) { + auto const time { static_cast( last_modified( item.path ) ) }; + std::error_code error; + auto const onsize { std::filesystem::file_size( item.path, error ) }; + if( ( false == static_cast( error ) ) && ( time != 0 ) && ( time == item.time ) && ( onsize == item.size ) ) { + // same size and modification time: taken as unchanged without reading it + continue; + } + std::uintmax_t size { 0 }; + std::uint64_t hash { 0 }; + if( ( false == hash_of( item.path, size, hash ) ) || ( size != item.size ) || ( hash != item.hash ) ) { + WriteLog( "Cooked data: \"" + item.path + "\" has changed since \"" + Path + "\" was written" ); + return false; + } + } + return true; +} + +bool +lists( std::string const &Path, std::vector const &Files ) { + + std::vector entries; + if( false == read( Path, entries ) ) { return false; } + std::vector listed; + listed.reserve( entries.size() ); + for( auto const &item : entries ) { listed.push_back( item.path ); } + return unique( listed ) == unique( Files ); +} + +} // namespace simulation::cookdeps diff --git a/simulation/cookdeps.cppm b/simulation/cookdeps.cppm new file mode 100644 index 00000000..b0d4068d --- /dev/null +++ b/simulation/cookdeps.cppm @@ -0,0 +1,41 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +module; +#include +#include + +export module eu07.simulation.cookdeps; + +export { + +// What a cooked artifact of a scenery was made from, and whether it still is. +// +// A manifest lists every file that went into the artifact with its size, modification time +// and a hash of its content. The artifact is current when every one of those files still has +// that content. Size and time are the quick check: when both match, the file is taken as +// unchanged; when either differs, the content is hashed, so a file that was only touched or +// copied does not force a rebuild, and one that was edited always does. +// +// A manifest cannot know about a file that was added to the scenery since it was written - an +// include line that was not there before. That shows when the scenery is read, so a caller +// compares the files it just read with the manifest afterwards, and treats the artifact as +// stale when the two differ. +namespace simulation::cookdeps { + +// writes the manifest of Files, hashing each. returns false and logs on failure +bool write( std::string const &Path, std::vector const &Files ); +// whether a manifest exists at Path and every file it lists still has the content it had +bool current( std::string const &Path ); +// whether the manifest at Path lists exactly Files, in any order and ignoring repeats +bool lists( std::string const &Path, std::vector const &Files ); + +} // namespace simulation::cookdeps + +} diff --git a/simulation/logiccook.cpp b/simulation/logiccook.cpp index 2af54ba4..542346c6 100644 --- a/simulation/logiccook.cpp +++ b/simulation/logiccook.cpp @@ -26,6 +26,7 @@ import eu07.simulation.simulationsounds; import eu07.utilities.logs; import eu07.utilities.globals; import eu07.utilities.utilities; +import eu07.simulation.cookdeps; namespace simulation { @@ -130,7 +131,7 @@ targets_of( basic_event const &Event, scene::logic::event_kind const Kind, scene } // anonymous namespace bool -cook_logic( std::string const &Sceneryfile ) { +cook_logic( std::string const &Sceneryfile, std::vector const &Sources ) { scene::logic::container_writer writer; auto const count { simulation::Events.event_count() }; @@ -158,19 +159,22 @@ cook_logic( std::string const &Sceneryfile ) { ErrorLog( "Logic container: " + std::to_string( unknown ) + " events of a kind this build cannot cook" ); } - return writer.write( scene::logic::container_path( Sceneryfile ) ); + auto const path { scene::logic::container_path( Sceneryfile ) }; + return ( true == writer.write( path ) ) + && ( true == cookdeps::write( path + ".deps", Sources ) ); } bool -logic_stale( std::string const &Sceneryfile ) { +logic_stale( std::string const &Sceneryfile, std::vector const &Sources ) { auto const path { scene::logic::container_path( Sceneryfile ) }; if( false == FileExists( path ) ) { return true; } - if( last_modified( scenery_file( Sceneryfile ) ) > last_modified( path ) ) { - WriteLog( "Logic container: \"" + path + "\" is older than the scenery, recooking" ); + if( ( false == cookdeps::current( path + ".deps" ) ) + || ( false == cookdeps::lists( path + ".deps", Sources ) ) ) { + WriteLog( "Logic container: \"" + path + "\" was made from files that have changed since, recooking" ); return true; } // the header says whether this build can read it at all, which covers a container diff --git a/simulation/logiccook.cppm b/simulation/logiccook.cppm index 4f3fd77a..b201a920 100644 --- a/simulation/logiccook.cppm +++ b/simulation/logiccook.cppm @@ -9,6 +9,7 @@ http://mozilla.org/MPL/2.0/. module; #include +#include export module eu07.simulation.logiccook; @@ -24,18 +25,16 @@ export { // determinism the plan asks for comes from the writer, not from where it is called. namespace simulation { -// writes the container next to the scenery. returns false and logs on failure -bool cook_logic( std::string const &Sceneryfile ); +// writes the container next to the scenery, with the manifest of the Sources it was made +// from. returns false and logs on failure +bool cook_logic( std::string const &Sceneryfile, std::vector const &Sources ); // reads a cooked container back and checks it against the events currently loaded. logs // every disagreement and returns true only when there are none bool verify_logic( std::string const &Sceneryfile ); -// whether the container next to this scenery needs cooking: there is none, this build -// cannot read the one that is there, or the scenery has been edited since. -// -// Edits to an .inc the scenery pulls in do NOT make it stale - only the .scn's own -// timestamp is compared, because the list of files that went into a load is not kept. Cook -// by hand with -cooklogic after editing an include. -bool logic_stale( std::string const &Sceneryfile ); +// whether the container next to this scenery needs cooking: there is none, this build cannot +// read the one that is there, a file it was made from has changed, or the scenery now reads +// a different set of files than Sources says it was made from +bool logic_stale( std::string const &Sceneryfile, std::vector const &Sources ); } // namespace simulation diff --git a/simulation/simulationstateserializer.cpp b/simulation/simulationstateserializer.cpp index de6aa2a8..b620ff72 100644 --- a/simulation/simulationstateserializer.cpp +++ b/simulation/simulationstateserializer.cpp @@ -101,6 +101,7 @@ state_serializer::deserialize_begin( std::string const &Scenariofile ) { WriteLog("Default SBT absent"); } scene::Groups.create(); + state->input.track_includes( state->included ); // before a token is read: pick up a heightfield baked on an earlier run, or arm the // bake when there is none. it has to happen here because the decision to skip terrain @@ -214,18 +215,22 @@ state_serializer::deserialize_continue(std::shared_ptr state Region->serialize( state->scenariofile ); } - terrainbake::finish( state->scenariofile ); + terrainbake::finish( state->scenariofile, *state->included ); loadprofile::report( state->scenariofile ); + // the logic depends on the scenario file itself as well as on what it includes: events + // can be written in either + auto sources { *state->included }; + sources.push_back( scenery_file( state->scenariofile ) ); if( true == Global.verify_logic ) { verify_logic( state->scenariofile ); } else if( ( true == Global.cook_logic ) - || ( true == logic_stale( state->scenariofile ) ) ) { + || ( true == logic_stale( state->scenariofile, sources ) ) ) { // cooked from the loaded state rather than from the text, and immediately read back, // so a container that disagrees with what it was cooked from says so on the spot - if( true == cook_logic( state->scenariofile ) ) { + if( true == cook_logic( state->scenariofile, sources ) ) { verify_logic( state->scenariofile ); } } diff --git a/simulation/simulationstateserializer.cppm b/simulation/simulationstateserializer.cppm index 69209d2d..6e8d3f58 100644 --- a/simulation/simulationstateserializer.cppm +++ b/simulation/simulationstateserializer.cppm @@ -43,6 +43,9 @@ struct deserializer_state { // the named scenery has no file on disk: the world is empty, so there's nothing to compile to SBT bool blank{ false }; cParser input; + // every file the scenery includes, as it is read: what anything cooked from this load + // depends on + std::shared_ptr> included { std::make_shared>() }; scene::scratch_data scratchpad; using deserializefunctionbind = std::function; std::unordered_map< diff --git a/simulation/terrainbake.cpp b/simulation/terrainbake.cpp index 530c9e6d..b3f21f16 100644 --- a/simulation/terrainbake.cpp +++ b/simulation/terrainbake.cpp @@ -32,6 +32,7 @@ import eu07.utilities.logs; import eu07.utilities.globals; import eu07.utilities.utilities; import eu07.simulation.loadprofile; +import eu07.simulation.cookdeps; namespace simulation::terrainbake { @@ -77,11 +78,6 @@ struct table_record { std::uint32_t mask; }; -struct file_stamp { - std::uint64_t size { 0 }; - std::int64_t time { 0 }; -}; - // one triangle as it waits in the spill file for the passes: its corners in world space with // their texture coordinates, the node it came from, which holds its material, and its number struct spilled { @@ -115,8 +111,6 @@ struct { std::unordered_map table; std::vector masks; std::size_t skipped { 0 }, filtered { 0 }, drawn { 0 }; - // size and modification time of every file a key was taken in, looked up once per load - std::unordered_map files; } state; // where a field is written: the finest beside the scenery as .ehf, coarser ones as @@ -133,6 +127,24 @@ table_path( std::string const &Heightfield ) { return Heightfield + ".nodes"; } +std::string +manifest_path( std::string const &Heightfield ) { + + return Heightfield + ".deps"; +} + +// throws away every file of a bake, so the next start makes a new one +void +discard( std::string const &Sceneryfile ) { + + std::error_code error; + for( std::uint32_t field = 0; field < fieldcount; ++field ) { + std::filesystem::remove( heightfield_path( Sceneryfile, field ), error ); + } + std::filesystem::remove( table_path( heightfield_path( Sceneryfile ) ), error ); + std::filesystem::remove( manifest_path( heightfield_path( Sceneryfile ) ), error ); +} + bool load_table( std::string const &Path ) { @@ -301,21 +313,8 @@ node_key( mix( Text.data(), Text.size() ); mix( "\n", 1 ); } }; - // what the file is now: an edit anywhere in it changes every key it holds - auto lookup { state.files.find( File ) }; - if( lookup == state.files.end() ) { - std::error_code error; - file_stamp stamp {}; - auto const size { std::filesystem::file_size( File, error ) }; - stamp.size = ( error ? 0 : static_cast( size ) ); - auto const time { std::filesystem::last_write_time( File, error ) }; - stamp.time = ( error ? 0 : static_cast( time.time_since_epoch().count() ) ); - lookup = state.files.emplace( File, stamp ).first; - } - + // the file's content is vouched for by the bake's manifest, so its name is enough here text( File ); - mix( &lookup->second.size, sizeof( lookup->second.size ) ); - mix( &lookup->second.time, sizeof( lookup->second.time ) ); auto const line { static_cast( Line ) }; mix( &line, sizeof( line ) ); for( auto const ¶meter : Parameters ) { text( parameter ); } @@ -348,7 +347,9 @@ begin( std::string const &Sceneryfile ) { // what counts as terrain, or one missing a part, is thrown away and baked again // rather than used std::vector baked; - auto usable { true == load_table( table_path( path ) ) }; + auto usable { + ( true == cookdeps::current( manifest_path( path ) ) ) + && ( true == load_table( table_path( path ) ) ) }; for( std::uint32_t field = 0; ( field < fieldcount ) && ( true == usable ); ++field ) { auto const fieldpath { heightfield_path( Sceneryfile, field ) }; if( false == FileExists( fieldpath ) ) { continue; } @@ -365,13 +366,10 @@ begin( std::string const &Sceneryfile ) { + std::to_string( state.table.size() ) + " triangle nodes replaced by them" ); return; } - WriteLog( "Terrain bake: the heightfields beside this scenery were baked under different rules, baking them again" ); + WriteLog( "Terrain bake: the heightfields beside this scenery are out of date, baking them again" ); state.table.clear(); state.masks.clear(); - for( std::uint32_t field = 0; field < fieldcount; ++field ) { - std::filesystem::remove( heightfield_path( Sceneryfile, field ), error ); - } - std::filesystem::remove( table_path( path ), error ); + discard( Sceneryfile ); } // the triangles wait on disk beside the heightfield rather than in memory: a large scenery @@ -483,12 +481,18 @@ add( std::uint64_t const Key, std::vector const &Vertices, std::st } void -finish( std::string const &Sceneryfile ) { +finish( std::string const &Sceneryfile, std::vector const &Included ) { if( true == planning() ) { WriteLog( "Terrain bake: " + std::to_string( state.skipped ) + " triangle nodes left to the heightfield, " + std::to_string( state.filtered ) + " drawn in part, " + std::to_string( state.drawn ) + " drawn as written" ); + // an include added or taken away since the bake: nodes of a new file were drawn as + // geometry this time, as none of their keys is known, but the next start bakes again + if( false == cookdeps::lists( manifest_path( heightfield_path( Sceneryfile ) ), Included ) ) { + WriteLog( "Terrain bake: the scenery includes different files than when it was baked; baking again on the next start" ); + discard( Sceneryfile ); + } } if( false == state.collecting ) { return; } @@ -581,6 +585,7 @@ finish( std::string const &Sceneryfile ) { + std::to_string( step * std::pow( fieldstride, field ) ) + " m"; } write_table( table_path( heightfield_path( Sceneryfile ) ) ); + cookdeps::write( manifest_path( heightfield_path( Sceneryfile ) ), Included ); auto const seconds { std::chrono::duration( std::chrono::steady_clock::now() - state.started ).count() }; diff --git a/simulation/terrainbake.cppm b/simulation/terrainbake.cppm index 7ede2d5b..fe40ca75 100644 --- a/simulation/terrainbake.cppm +++ b/simulation/terrainbake.cppm @@ -35,11 +35,11 @@ export { // triangles the heightfield really shows stop being drawn on their own, so nothing can vanish // from the scenery - the worst a bad bake can do is draw something twice. // -// A triangle node is recognised by the file and line it is written at - with that file's size -// and modification time - the parameters its include was given, and the origin and rotation in -// force, not by its position in the load. An edited file changes the keys of everything in it, -// and those nodes are then simply drawn again instead of the wrong ones being dropped. Hashing -// the node's text instead would cost a second parse of every triangle node in the scenery. +// A bake is used only while every file the scenery included when it was made still has the +// content it had (cookdeps): the scenario file itself is not among them, since the launcher +// rewrites it on every start and the terrain lives in what it includes. Within a bake that +// holds, a triangle node is recognised by the file and line it is written at, the parameters +// its include was given, and the origin and rotation in force. // // The rasterising is scene/terraincooker.h, the same code the tool uses. namespace simulation::terrainbake { @@ -77,8 +77,10 @@ node_decision examine( std::string_view const Type, glm::dvec3 const &Offset, glm::vec3 const &Rotation, bool const Worldspace ); // one triangle node, in world space, as it is about to be drawn void add( std::uint64_t const Key, std::vector const &Vertices, std::string_view const Material ); -// called once the scenery has been parsed. writes the heightfield and its node table -void finish( std::string const &Sceneryfile ); +// called once the scenery has been parsed, with every file it included. writes the +// heightfields, their node table and the manifest of what they were made from; or, when a +// bake was used, checks the scenery still includes what that bake was made from +void finish( std::string const &Sceneryfile, std::vector const &Included ); } // namespace simulation::terrainbake diff --git a/utilities/Globals.cppm b/utilities/Globals.cppm index 43c979b4..fb9bfb6e 100644 --- a/utilities/Globals.cppm +++ b/utilities/Globals.cppm @@ -131,6 +131,9 @@ struct global_settings { // bake a heightfield for a scenery that has none, the first time it is opened, and use // it from then on. -nobaketerrain turns it off bool bake_terrain{ true }; + // -cook: load the scenery, bring what is cooked from it up to date, and quit. the step that + // prepares a scenery's data ahead of play, so that the game never has to bake on its own + bool cook_only{ false }; // cooked heightfields drawn in place of the legacy terrain geometry, coarsest first // once sorted. a scenery needs several: a regional elevation model sampled every // hundred metres, and a corridor of its own for each line, sampled every two diff --git a/utilities/parser.cpp b/utilities/parser.cpp index 22d9fe36..ae949652 100644 --- a/utilities/parser.cpp +++ b/utilities/parser.cpp @@ -376,6 +376,10 @@ void cParser::startIncludeFromParser(cParser& srcParser, bool ToLower, std::stri ); mIncludeParser->allowRandomIncludes = allowRandomIncludes; mIncludeParser->autoclear(m_autoclear); + if (mIncluded) { + mIncluded->push_back(mPath + includefile); + mIncludeParser->mIncluded = mIncluded; + } if (mIncludeParser->mSize <= 0) { ErrorLog("Bad include: can't open file \"" + includefile + "\""); @@ -549,6 +553,8 @@ void cParser::injectString(const std::string &str) { mIncludeParser = std::make_shared(str, buffer_TEXT, "", LoadTraction, std::vector(), allowRandomIncludes); mIncludeParser->autoclear(m_autoclear); + // text put into the input can include files of its own + mIncludeParser->mIncluded = mIncluded; } } diff --git a/utilities/parser.cppm b/utilities/parser.cppm index 3f87c3bd..71196310 100644 --- a/utilities/parser.cppm +++ b/utilities/parser.cppm @@ -111,6 +111,9 @@ class cParser //: public std::stringstream std::size_t Line() const; // parameters of the innermost file being read, the (pN) values its include line gave std::vector const & Parameters() const; + // collects into Into the path of every file this parser includes, and every file those + // include in turn. what a cooked artifact of a scenery depends on is exactly this list + void track_includes( std::shared_ptr> Into ) { mIncluded = std::move( Into ); } // returns number of currently processed line in main file, -1 if inside include int LineMain() const; bool expandIncludes = true; @@ -130,6 +133,7 @@ class cParser //: public std::stringstream std::size_t count(); // members: bool m_autoclear { true }; // unretrieved tokens are discarded when another read command is issued (legacy behaviour) + std::shared_ptr> mIncluded; // where included files are recorded, if anywhere bool LoadTraction { true }; // load traction? std::shared_ptr mStream; // relevant kind of buffer is attached on creation. std::string mFile; // name of the open file, if any