/* This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #pragma once // One file holding a scenery's cooked terrain tiles. // // A terrain is tens of thousands of small files, which is the one thing a filesystem is worst at: // opening them costs more than reading them. They go into an archive instead, each entry // compressed on its own so that a tile can be read without touching the rest, and the index is // written last so that cooking never has to hold the tiles in memory. #include #include #include #include #include #include #include namespace pak { inline constexpr char magic[ 8 ] { 'E', 'U', '0', '7', 'P', 'A', 'K', '\0' }; inline constexpr std::uint32_t version { 2 }; inline constexpr std::size_t namelength { 64 }; #pragma pack( push, 1 ) struct file_header { char magic[ 8 ]; std::uint32_t version; std::uint32_t entrycount; std::uint64_t indexoffset; }; struct index_entry { char name[ namelength ]; std::uint64_t offset; std::uint64_t stored; // bytes in the file std::uint64_t plain; // bytes once decompressed; equal to stored when kept as it is }; #pragma pack( pop ) // Writes entries straight through to disk as they are added. class writer { public: bool open( std::string const &Path ) { m_file.open( Path, std::ios::binary | std::ios::trunc ); if( false == m_file.good() ) { return false; } file_header header {}; m_file.write( reinterpret_cast( &header ), sizeof( header ) ); m_at = sizeof( header ); return m_file.good(); } bool add( std::string const &Name, std::vector const &Data ) { if( ( false == m_file.good() ) || ( Name.size() >= namelength ) ) { return false; } index_entry entry {}; std::memcpy( entry.name, Name.c_str(), Name.size() ); entry.offset = m_at; entry.plain = Data.size(); m_packed.resize( ZSTD_compressBound( Data.size() ) ); auto const packed { ZSTD_compress( m_packed.data(), m_packed.size(), Data.data(), Data.size(), 3 ) }; auto const *bytes { reinterpret_cast( Data.data() ) }; auto size { Data.size() }; if( ( 0 == ZSTD_isError( packed ) ) && ( packed < Data.size() ) ) { bytes = reinterpret_cast( m_packed.data() ); size = packed; } entry.stored = size; m_file.write( bytes, static_cast( size ) ); m_at += size; m_index.push_back( entry ); return m_file.good(); } bool close() { if( false == m_file.good() ) { return false; } file_header header {}; std::memcpy( header.magic, magic, sizeof( magic ) ); header.version = version; header.entrycount = static_cast( m_index.size() ); header.indexoffset = m_at; for( auto const &entry : m_index ) { m_file.write( reinterpret_cast( &entry ), sizeof( entry ) ); } m_file.seekp( 0 ); m_file.write( reinterpret_cast( &header ), sizeof( header ) ); auto const good { m_file.good() }; m_file.close(); return good; } std::size_t entrycount() const { return m_index.size(); } std::uint64_t bytes() const { return m_at + m_index.size() * sizeof( index_entry ); } private: std::ofstream m_file; std::uint64_t m_at { 0 }; std::vector m_index; std::vector m_packed; }; // Reads entries back. One reader is used from one thread at a time: it keeps a handle and seeks. class reader { public: bool open( std::string const &Path ) { close(); m_file.open( Path, std::ios::binary ); if( false == m_file.good() ) { return false; } file_header header {}; m_file.read( reinterpret_cast( &header ), sizeof( header ) ); if( false == m_file.good() ) { return false; } if( 0 != std::memcmp( header.magic, magic, sizeof( magic ) ) ) { return false; } if( header.version != version ) { return false; } m_file.seekg( static_cast( header.indexoffset ) ); for( std::uint32_t entry = 0; entry < header.entrycount; ++entry ) { index_entry record {}; m_file.read( reinterpret_cast( &record ), sizeof( record ) ); if( false == m_file.good() ) { return false; } record.name[ namelength - 1 ] = '\0'; m_index.emplace( record.name, record ); } return true; } void close() { m_index.clear(); if( true == m_file.is_open() ) { m_file.close(); } m_file.clear(); } bool exists( std::string const &Name ) const { return m_index.contains( Name ); } std::size_t entrycount() const { return m_index.size(); } std::vector names() const { std::vector names; names.reserve( m_index.size() ); for( auto const &[ name, entry ] : m_index ) { names.push_back( name ); } return names; } bool read( std::string const &Name, std::vector &Out ) { auto const found { m_index.find( Name ) }; if( found == m_index.end() ) { return false; } auto const &entry { found->second }; m_stored.resize( entry.stored ); m_file.seekg( static_cast( entry.offset ) ); m_file.read( reinterpret_cast( m_stored.data() ), static_cast( entry.stored ) ); if( false == m_file.good() ) { m_file.clear(); return false; } if( entry.stored == entry.plain ) { Out.swap( m_stored ); return true; } Out.resize( entry.plain ); auto const plain { ZSTD_decompress( Out.data(), Out.size(), m_stored.data(), m_stored.size() ) }; return ( 0 == ZSTD_isError( plain ) ) && ( plain == entry.plain ); } private: std::ifstream m_file; std::unordered_map m_index; std::vector m_stored; }; } // namespace pak