From b877bc077287b9b8d7937dfe649a9e53bbc3a995 Mon Sep 17 00:00:00 2001 From: maj00r Date: Sun, 20 Sep 2026 08:04:46 +0200 Subject: [PATCH] Cook the ground into a quadtree of mesh tiles, and keep the mesh the scenery drew The ground in a scenery is already an irregular mesh, so the cook keeps it: it cuts the triangles along the tile grid, welds what coincides, and builds each coarser level by collapsing edges. Nothing is resampled, nothing is interpolated, and a hole the scenery left stays a hole. What was here before threw the triangles away, kept their vertices and built a fresh Delaunay triangulation over them, which filled every hole, merged the two heights of every retaining wall into one, and - because it never erased the super-triangle - emitted indices three too large together with triangles reaching out to the artificial corners. Tiles form a quadtree, as Cesium Quantized Mesh intends. A distant tile is a large one, so the horizon costs tens of tiles rather than the thousands a fixed grid needed, and the level to draw follows from the tile's own geometric error against what the screen can resolve - which is what the format carries that field for. Every vertex on a tile's border survives into all of its coarser levels, and so does every edge between two of them, so two neighbours drawn at different levels share their border exactly: there is nothing to stitch and nothing to morph. The file layout is the format's, with two deliberate departures, both written down in quantizedmeshformat.h: the geodetic header fields carry scenery metres, there being no geographic reference to convert from, and the material a vertex belongs to rides in an extension of ours. Texture coordinates are not stored at all - ground tiles in world space, so the coordinate follows from the position and from how many metres one repeat of the material covers, measured off the scenery once and kept in the terrain's table. Tiles live in one archive beside the scenery, an entry each, compressed on its own. Baltyk cooks to 546 kB against 2.2 MB of heightfields, quark to 361 kB against 9.6 MB, tomaszewo to 375 kB against 9.9 MB. Drawing is one call a frame: every resident tile lives in the same vertex and index buffer, with its vertices measured from the middle of the terrain so that the whole map fits in floats, and a frame is one glMultiDrawElementsBaseVertex over the tiles the walk chose and the frustum kept. On baltyk that settles at 291 tiles resident, 16 MB of mesh, 85 tiles and 6k triangles drawn. What counts as ground is what the scenery says, not what its triangles look like. A node under origin, rotate or scale is an object standing on the ground; a material with an alpha channel is a tree, a bush, a fence or a catenary mast. Everything else is laid whichever way it faces, so the side of an embankment or a cutting goes in with the rest instead of leaving a hole along every bank. Whatever is left out is recorded per node, and the scenery goes on drawing it. A cook is a cache: it is used again only when the archive, the note of which nodes went into it and the manifest of what the scenery was made from all agree. A manifest listing a file the scenery names but does not have used to fail that test for ever, since an absent file never matches - quark has such an include, and nothing beside it could ever be reused. tools/terraincook cooks the same way outside the engine, and meshtest beside it checks what the cook promises: a tile reads back as written, neighbouring tiles agree on their border to the millimetre, a hole is not filled in, a wall is laid exactly once, and a material's tiling comes out of the source. seamcheck.py goes: it tested the morphing constants of a heightfield nothing reads any more. --- CMakeLists.txt | 2 + TIN_IMPLEMENTATION.md | 232 ------ TIN_INTEGRATION_GUIDE.md | 267 ------ TIN_RENDERER_PLAN.md | 257 ------ TIN_SESSION_SUMMARY.md | 200 ----- TIN_STATUS.md | 162 ---- application/driveruipanels.cpp | 18 +- rendering/opengl33renderer.cpp | 8 +- rendering/terrainclipmap.cpp | 647 +++++++-------- rendering/terrainclipmap.cppm | 175 ++-- rendering/terrainmesharena.cpp | 255 ++++++ rendering/terrainmesharena.cppm | 87 ++ rendering/terrainstatus.cppm | 84 +- rendering/terraintileloader.cpp | 55 +- rendering/terraintileloader.cppm | 69 +- rendering/terraintilepool_tin.cpp | 130 --- rendering/terraintilepool_tin.cppm | 85 -- scene/pakformat.h | 179 +++++ scene/quantizedmeshcooker.cpp | 981 +++++++++++++++++++++++ scene/quantizedmeshcooker.h | 176 ++++ scene/quantizedmeshformat.h | 268 ++++--- scene/quantizedmeshreader.h | 512 +++++++----- scene/terraincooker_tin.cpp | 579 ------------- scene/terraincooker_tin.h | 194 ----- shaders/terrainclipmap_tin.frag | 23 - shaders/terrainclipmap_tin.vert | 30 - shaders/terrainmesh.frag | 56 ++ shaders/terrainmesh.vert | 32 + simulation/cookdeps.cpp | 18 +- simulation/simulationstateserializer.cpp | 11 +- simulation/terrainbake.cpp | 671 ++++++---------- simulation/terrainbake.cppm | 70 +- tools/terraincook/CMakeLists.txt | 25 +- tools/terraincook/main.cpp | 92 ++- tools/terraincook/meshtest.cpp | 221 +++++ tools/terraincook/seamcheck.py | 99 --- 36 files changed, 3356 insertions(+), 3614 deletions(-) delete mode 100644 TIN_IMPLEMENTATION.md delete mode 100644 TIN_INTEGRATION_GUIDE.md delete mode 100644 TIN_RENDERER_PLAN.md delete mode 100644 TIN_SESSION_SUMMARY.md delete mode 100644 TIN_STATUS.md create mode 100644 rendering/terrainmesharena.cpp create mode 100644 rendering/terrainmesharena.cppm delete mode 100644 rendering/terraintilepool_tin.cpp delete mode 100644 rendering/terraintilepool_tin.cppm create mode 100644 scene/pakformat.h create mode 100644 scene/quantizedmeshcooker.cpp create mode 100644 scene/quantizedmeshcooker.h delete mode 100644 scene/terraincooker_tin.cpp delete mode 100644 scene/terraincooker_tin.h delete mode 100644 shaders/terrainclipmap_tin.frag delete mode 100644 shaders/terrainclipmap_tin.vert create mode 100644 shaders/terrainmesh.frag create mode 100644 shaders/terrainmesh.vert create mode 100644 tools/terraincook/meshtest.cpp delete mode 100755 tools/terraincook/seamcheck.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 9de66300..230398cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -192,11 +192,13 @@ set(SOURCES "rendering/terrainclipmap.cpp" "rendering/terraintileloader.cpp" "rendering/terraintilepool.cpp" +"rendering/terrainmesharena.cpp" "rendering/terrainground.cpp" "simulation/loadprofile.cpp" "simulation/cookdeps.cpp" "simulation/logiccook.cpp" "simulation/terrainbake.cpp" +"scene/quantizedmeshcooker.cpp" "simulation/simulationtime.cpp" "scene/sceneeditor.cpp" "rendering/screenshot.cpp" diff --git a/TIN_IMPLEMENTATION.md b/TIN_IMPLEMENTATION.md deleted file mode 100644 index 052f1bd8..00000000 --- a/TIN_IMPLEMENTATION.md +++ /dev/null @@ -1,232 +0,0 @@ -# TIN (Triangulated Irregular Network) Terrain Format - -Implementation of Cesium Quantized Mesh format for adaptive terrain triangulation as an alternative to regular heightfield. - -## What's Implemented - -### 1. Format Definition (`scene/quantizedmeshformat.h`) - -Cesium Quantized Mesh format adapted for MaSzyna: -- **Header**: 128 bytes with tile bounds, vertex/triangle counts, LOD error metrics -- **Vertex data**: Quantized uint16[3] positions (XYZ) - 6 bytes per vertex -- **UV data**: Quantized uint16[2] texture coords - 4 bytes per vertex -- **Indices**: uint16 or uint32 depending on vertex count -- **Edge indices**: For seamless LOD stitching (north/south/east/west) -- **Extensions**: Material splat, metadata, meshlets - -**Compression**: -- Positions quantized to tile bounding box (2mm precision for 256m tiles) -- UVs quantized 0-1 range -- Optional per-tile zstd compression (same as heightfield) - -### 2. Triangulator (`scene/terraincooker_tin.{h,cpp}`) - -Constrained Delaunay triangulation using [CDT library](https://github.com/artem-ogre/CDT): - -**Features**: -- Input: Same triangle soup as heightfield cooker (compatible interface) -- Constraint edges: Track corridors, roads, water boundaries -- Multi-LOD generation: 4 levels by default, configurable -- Edge-collapse simplification per LOD -- Automatic edge extraction for seamless tile stitching - -**LOD Strategy**: -- Level 0 (finest): Full triangulation with all constraints -- Level 1-3: Progressive simplification, removing low-error vertices -- Geometric error tracked per LOD (for distance-based selection) -- Target: 30% vertex reduction per LOD level - -**Statistics** (from `binary-scene-format-plan.md` measurements): -``` -l204 terrain: 8.8M triangles -Expected TIN output: ~4.4M triangles (50% reduction) -Heightfield cost: 186 MB (2m grid) -TIN cost estimate: ~90-120 MB (better compression, variable density) -``` - -### 3. Integration (`tools/terraincook/main.cpp`) - -New mode: `-cooktin` alongside existing `-cook` - -```bash -terraincook -cooktin -tilesize 256 -lodlevels 4 -out cooked/ -``` - -**Output**: -- `terrain_X_Z_lodN.qm` per tile per LOD -- Binary Quantized Mesh format -- Compatible with existing tile coordinate system - -**Parameters**: -- `-tilesize `: Tile size (default: 256m) -- `-lodlevels `: Number of LOD levels (default: 4) -- `-raw`: Disable compression (for debugging) - -### 4. Key Design Decisions - -**Why TIN over Heightfield?** - -From measurements (`binary-scene-format-plan.md`): -- Track corridor (< 25m): 91.8% triangles have edges < 4m -- Open field (> 1600m): Only 25.7% need fine detail -- Uniform 2m grid wastes resolution where not needed -- TIN adapts: dense along infrastructure, coarse elsewhere - -**Why Cesium Quantized Mesh?** - -- Industry standard, proven at scale (Cesium terrain streaming) -- Efficient: quantization + index compression -- Edge constraints solve T-junction problem (no cracks between LODs) -- Extensible: material splat, normals, metadata -- Open format, not tied to any vendor - -**CDT vs Triangle vs Delaunator**: - -- **CDT** (chosen): MPL-2.0, header-only, C++, constraint edges -- **Triangle**: Unclear license, C, harder to integrate -- **Delaunator**: Fast but no constraints (can't preserve track edges) - -## What's NOT Yet Implemented - -### 1. Loader (`rendering/terraintileloader.cppm`) - -Current loader reads heightfield `.ehf` tiles. Needs: -- Parallel reader for `.qm` format -- Dequantization of vertices -- Upload to GPU (vertex buffer instead of texture array) - -### 2. Renderer (`rendering/terrainclipmap.cppm`) - -Current clipmap renders from texture arrays. Needs: -- Vertex buffer pool (replace `terrain_tile_pool` texture arrays) -- Index buffer per LOD -- Vertex input state (position, UV, material index) -- Edge stitching shader (use edge indices to eliminate cracks) - -### 3. Material Splat - -TIN vertices carry material indices. Needs: -- Per-vertex material blending in fragment shader -- Material texture array (same as heightfield) -- Possibly: bake splat into vertex colors for simpler shader - -### 4. Constraint Generation - -Current implementation takes constraints as input. Needs: -- Automatic extraction from track geometry -- Road boundaries from scenery -- Water edges from material boundaries - -## Build Setup - -### Dependencies - -Add to CMake or vcpkg: -``` -find_package(CDT CONFIG REQUIRED) -target_link_libraries(target PRIVATE CDT::CDT) -``` - -CDT is available in vcpkg: -```json -{ - "name": "cdt", - "version": "1.4.4", - "description": "Constrained Delaunay Triangulation" -} -``` - -### Files Added - -``` -scene/quantizedmeshformat.h - Format specification -scene/terraincooker_tin.h - Triangulator interface -scene/terraincooker_tin.cpp - Triangulator implementation -tools/terraincook/main.cpp - Integration (modified) -tools/terraincook/CMakeLists.txt - Build config (modified) -tools/terraincook/test_tin.cpp - Simple test program -``` - -## Testing - -### 1. Cook Test Data - -```bash -# Build terraincook with CDT dependency -cd tools/terraincook && mkdir build && cd build -cmake .. -DCMAKE_BUILD_TYPE=Release -make - -# Run on small scenery subset -./terraincook ../../test_data/teren -cooktin -out cooked_tin/ -``` - -Expected output: -``` -Triangulating tiles: 100/100 -Triangulation complete: - Input triangles: 8979 - Output triangles: 4512 (49.7% reduction) - Output vertices: 2389 -Wrote 400 quantized mesh files to cooked_tin/ -``` - -### 2. Verify Tile Format - -```bash -hexdump -C cooked_tin/terrain_0_0_lod0.qm | head -20 -``` - -Should see: -``` -00000000 45 55 30 37 51 4d 53 48 01 00 00 00 ... |EU07QMSH| -``` - -### 3. Compare to Heightfield - -```bash -du -sh cooked/terrain.ehf cooked_tin/*.qm -``` - -Expected: TIN ~30-40% smaller for l204 terrain. - -## Next Steps (Priority Order) - -1. **Verify triangulation quality** - - Export preview images (height, overlay, triangle density) - - Check edge constraints preserved - - Measure actual reduction vs. theory - -2. **Implement loader** - - Read `.qm` files on background thread - - Dequantize and upload to GPU - - Integrate with existing `terrain_tile_loader` thread pool - -3. **Adapt renderer** - - Replace texture fetch with vertex attributes - - Add edge stitching (sample coarser LOD along shared edges) - - Preserve existing CDLOD distance logic - -4. **Constraint extraction** - - Parse track geometry from `.scm` - - Generate corridor outline (track width + margin) - - Feed to triangulator as constraints - -5. **Performance validation** - - Measure vs. heightfield: memory, load time, frame time - - Confirm 50% vertex reduction translates to visible win - - Check no regressions in visual quality - -## References - -- [Cesium Quantized Mesh Spec](https://github.com/CesiumGS/quantized-mesh) -- [CDT Library](https://github.com/artem-ogre/CDT) -- `binary-scene-format-plan.md` - Original measurements and rationale -- `scene/heightfieldformat.h` - Current heightfield format for comparison - -## Notes - -- **Hybrid approach viable**: Keep heightfield for flat areas, TIN for corridors -- **Format extensible**: Meshlets for mesh shading, normals, etc. -- **Deterministic**: CDT is deterministic for same input (reproducible builds) -- **Backward compat**: Old `.ehf` files still work, TIN is opt-in via `-cooktin` diff --git a/TIN_INTEGRATION_GUIDE.md b/TIN_INTEGRATION_GUIDE.md deleted file mode 100644 index 7ed50d3b..00000000 --- a/TIN_INTEGRATION_GUIDE.md +++ /dev/null @@ -1,267 +0,0 @@ -# TIN Migration - Complete Implementation Guide - -## Status: 95% Complete - Ready for Integration Testing - -### ✅ Fully Implemented - -#### 1. Format & Data Structures -- `scene/quantizedmeshformat.h` - Cesium Quantized Mesh spec ✅ -- `scene/quantizedmeshreader.h` - Reader for `.qm` files ✅ -- `scene/terraincooker_tin.{h,cpp}` - CDT triangulator + LOD ✅ - -#### 2. Tooling -- `tools/terraincook/main.cpp` - TIN is default format ✅ -- `tools/terraincook/CMakeLists.txt` - CDT dependency ✅ - -#### 3. Loading Pipeline -- `rendering/terraintileloader.{cppm,cpp}` - Reads TIN tiles ✅ -- `rendering/terraintilepool_tin.{cppm,cpp}` - VBO/IBO pool ✅ - -#### 4. Renderer Interface -- `rendering/terrainclipmap.cppm` - Updated for TIN ✅ -- Key methods implemented: - - `open()` ✅ - - `close()` ✅ - - `upload()` ✅ - - `level_for()` ✅ - - `resolve_uniforms()` ✅ - -#### 5. Shaders -- `shaders/terrainclipmap_tin.vert` - Vertex attributes ✅ -- `shaders/terrainclipmap_tin.frag` - Material lookup ✅ - -### 🔧 Remaining Integration Work - -#### Critical (for basic rendering): - -**1. Complete `terrainclipmap.cpp`** - -Currently has partial TIN implementation. Need to: - -a) Replace `render()` method (lines 360-468) with version from `TIN_RENDER_SNIPPET.cpp` - -b) Fix `retire()` method: -```cpp -void terrain_clipmap::retire( resident_tile const &Tile ) { - m_pool.free( Tile.gpu_data ); // OLD: pool->release(Tile.slot) -} -``` - -c) Update `scan()` method - mostly unchanged but remove heightfield references: -```cpp -// Change: m_reader.contains(x, z) -// To: m_reader.contains(x, z, wantedlevel) -``` - -d) Update remaining method signatures - search for `m_reader.` calls and adapt to new reader interface - -**2. Update Shader Loading** - -In `open()` (already done but verify): -```cpp -gl::shader vertex( "terrainclipmap_tin.vert" ); // Was: terrainclipmap.vert -gl::shader fragment( "terrainclipmap_tin.frag" ); // Was: terrainclipmap.frag -``` - -**3. Material Table Loading** - -Currently placeholder in `open()`: -```cpp -// TODO: Load material names from first tile's extension -std::vector materials = { "grass", "gravel", "dirt" }; -``` - -Should read from `.qm` file extensions or separate material manifest. - -#### Nice-to-Have (for production quality): - -**4. Edge Stitching** - -Use edge indices from loader payload to eliminate cracks between LOD levels. - -In `render()`, before drawing tile at LOD N where neighbor is at LOD N+1: -```cpp -// Modify vertices on shared edge to match coarser neighbor -for( auto edge_idx : tile.north_edge ) { - // Fetch from neighbor's south_edge -} -``` - -**5. Proper Material Blending** - -Current frag shader does single material lookup. For smooth transitions: -```glsl -// Read 4 material weights from vertex -in vec4 vMaterialWeights; -in uvec4 vMaterialIndices; - -vec4 color = - vMaterialWeights.x * texture(ground, vec3(vUV, vMaterialIndices.x)) + - vMaterialWeights.y * texture(ground, vec3(vUV, vMaterialIndices.y)) + - // ... etc -``` - -**6. Tile Size Auto-Detection** - -Currently hardcoded: -```cpp -m_tilesize = 256.0f; -m_lodlevels = 4; -``` - -Should scan directory and parse from first tile header. - -**7. Memory Budget Enforcement** - -Track allocated vertices/indices and enforce limit: -```cpp -while( m_pool.allocated_bytes() > budget && !oldest_tiles.empty() ) { - retire( oldest_tiles.back() ); -} -``` - -### 📝 Integration Steps - -#### Phase 1: Manual File Edits (30 min) - -1. Open `rendering/terrainclipmap.cpp` -2. Find `render()` method (line ~360) -3. Replace entire method with content from `TIN_RENDER_SNIPPET.cpp` -4. Search for `m_reader.` calls, update for quantizedmesh::reader API: - - `m_reader.tilesize()` → `m_tilesize` - - `m_reader.levels()` → `m_lodlevels` - - `m_reader.tiles()` → scan directory -5. Remove references to: - - `m_pools` (old texture pools) - - `m_indices` (old index buffers) - - `m_instances` (old instance buffer) - - `m_vao` (VAO now per-tile in pool) - -#### Phase 2: Build Test (1 hour) - -```bash -# 1. Install CDT if needed -vcpkg install cdt - -# 2. Configure -cmake -B build -S . -DCMAKE_BUILD_TYPE=RelWithDebInfo - -# 3. Build (expect some linker errors - fix iteratively) -cmake --build build --target maszyna -j8 - -# Common fixes needed: -# - Add terraintilepool_tin.cpp to CMakeLists.txt -# - Add terrainclipmap_tin shaders to shader list -# - Remove old heightfield includes if any remain -``` - -#### Phase 3: Cook Test Data (30 min) - -```bash -# Build cooker -cmake --build build --target terraincook - -# Cook small test area -./build/tools/terraincook/terraincook \ - test_data/teren/*.scm \ - -cook \ - -tilesize 256 \ - -lodlevels 3 \ - -out cooked_tin/ - -# Verify output -ls -lh cooked_tin/ -# Should see: terrain_0_0_lod0.qm, terrain_0_0_lod1.qm, etc. -``` - -#### Phase 4: Runtime Test (1-2 hours) - -```bash -# Run with test scenery -./build/maszyna -scenery cooked_tin/ - -# Debug checklist: -# 1. Does terrain load? (check logs for "Terrain TIN: ...") -# 2. Do tiles upload? (check GPU memory) -# 3. Does anything render? (might be black first time) -# 4. Are positions correct? (tiles should align with track) -# 5. Do LODs switch? (fly high, check triangle count drops) -``` - -### 🐛 Known Issues & Workarounds - -**Issue 1: Black terrain** -- **Cause**: Material textures not loading -- **Fix**: Check `m_ground.create()` returns true, verify texture paths - -**Issue 2: Terrain at wrong position** -- **Cause**: Camera-relative offset calculation wrong -- **Fix**: In render(), verify `tile_offset = origin - camera_pos` - -**Issue 3: Cracks between tiles** -- **Cause**: Edge stitching not implemented -- **Workaround**: Use single LOD (set `-lodlevels 1` in cook) -- **Proper fix**: Implement Phase "Nice-to-Have" #4 - -**Issue 4: Performance worse than heightfield** -- **Cause**: Too many draw calls (one per tile) -- **Fix**: Batch tiles, or reduce tile count (larger tiles) - -### 📊 Expected Results - -**Memory (l204 scenery)**: -- Old: ~250 MB (heightfield texture arrays) -- New: ~150 MB (vertex/index buffers) -- Savings: 40% - -**File Size**: -- Old: 186 MB (`.ehf`) -- New: ~100 MB (`.qm` files) -- Savings: 46% - -**Triangle Count**: -- Old: 8.8M uniform -- New: 4.4M adaptive -- Savings: 50% - -**Frame Time** (target: no worse than old): -- Old: instanced draw, ~0.5ms terrain -- New: per-tile draws, budget ~1.0ms -- If >2ms: need batching - -### 🎯 Success Criteria - -- [ ] Builds without errors -- [ ] Loads `.qm` tiles from directory -- [ ] Terrain renders at correct world position -- [ ] Multiple LODs visible (verify triangle count changes with distance) -- [ ] Materials display (grass/gravel textures visible) -- [ ] No crashes when flying across scenery -- [ ] Frame time < 2ms for terrain pass -- [ ] Memory usage < 200 MB for full l204 - -### 🔄 Rollback Plan - -If integration fails: - -1. Revert `rendering/terrainclipmap.{cppm,cpp}` -2. Revert `rendering/terraintileloader.{cppm,cpp}` -3. Keep all other files (they don't break anything) -4. Old heightfield pipeline continues working -5. Fix issues in separate branch before trying again - -### 📚 Reference Files - -- **TIN_RENDER_SNIPPET.cpp** - Complete render() implementation -- **TIN_IMPLEMENTATION.md** - Full design rationale -- **TIN_STATUS.md** - What's done, what remains -- **TIN_SESSION_SUMMARY.md** - Development log -- **TIN_RENDERER_PLAN.md** - Detailed renderer migration plan - ---- - -**Current State**: Code is 95% complete. Remaining work is integration (file edits, build fixes, testing). - -**Time to Working**: 2-4 hours if following steps above. - -**Recommendation**: Start with Phase 1 (manual file edits), then build incrementally fixing errors as they appear. diff --git a/TIN_RENDERER_PLAN.md b/TIN_RENDERER_PLAN.md deleted file mode 100644 index 198bd18b..00000000 --- a/TIN_RENDERER_PLAN.md +++ /dev/null @@ -1,257 +0,0 @@ -# TIN Renderer Migration - Full Replacement - -## Status: Loader Complete, Renderer Next - -### ✅ Complete - -1. **Format** (`quantizedmeshformat.h`) - Cesium Quantized Mesh spec -2. **Triangulator** (`terraincooker_tin.{h,cpp}`) - CDT + LOD generation -3. **Cooker** (`tools/terraincook/main.cpp`) - `-cook` now generates TIN by default -4. **Loader** (`terraintileloader.{cppm,cpp}`) - Reads `.qm` files, dequantizes vertices - -### 🚧 Next: Renderer (`rendering/terrainclipmap.{cppm,cpp}`) - -#### Current State (Heightfield) -```cpp -// Texture arrays per LOD -terrain_tile_pool (GL_TEXTURE_2D_ARRAY) - - heights: R16 - - materials: R8 - -// Shader samples texture -layout(binding = 0) uniform sampler2DArray heights; -vec4 h = texelFetch(heights, ivec3(x, y, layer)); -``` - -#### Target State (TIN) -```cpp -// Vertex buffer per tile -struct tile_vertex { - glm::vec3 position; // 12 bytes - glm::vec2 uv; // 8 bytes - std::uint8_t material;// 1 byte - std::uint8_t pad[3]; // alignment -}; // 24 bytes per vertex - -// Index buffer per tile -std::vector indices; - -// Shader gets attributes -layout(location = 0) in vec3 position; -layout(location = 1) in vec2 uv; -layout(location = 2) in uint material_idx; -``` - -### Implementation Plan - -#### Phase 1: Replace `terrain_tile_pool` - -**Current** (`rendering/terraintilepool.cppm`): -- Manages texture array layers -- Upload = `glTexSubImage3D` to layer - -**New** (`rendering/terraintilepool_tin.cppm`): -```cpp -class terrain_tile_pool { -public: - struct tile_handle { - gl::buffer vbo; // vertex buffer - gl::buffer ibo; // index buffer - std::uint32_t vertex_count; - std::uint32_t index_count; - }; - - tile_handle upload( - std::vector const &positions_x, - std::vector const &positions_y, - std::vector const &positions_z, - std::vector const &uvs_u, - std::vector const &uvs_v, - std::vector const &materials, - std::vector const &indices, - glm::dvec3 const &origin // for relative positioning - ); - - void free( tile_handle const &Handle ); -}; -``` - -#### Phase 2: Adapt `terrain_clipmap::render()` - -**Current flow**: -1. Frustum cull tiles → instance list -2. One instanced draw per LOD -3. Shader computes vertex from `gl_VertexID` -4. Shader samples heightmap texture - -**New flow**: -1. Frustum cull tiles → draw list -2. For each tile: - - Bind its VBO/IBO - - `glDrawElementsBaseInstance` with tile's transform -3. Shader receives vertex attributes directly - -**Key change**: From massive instanced draw to per-tile indexed draws. - -#### Phase 3: Update Shaders - -**terrainclipmap.vert** (before): -```glsl -// No vertex input - compute from ID -layout(location = 0) out vec2 vUV; - -void main() { - int x = gl_VertexID % sideLength; - int y = gl_VertexID / sideLength; - - // Fetch height from texture - float h = texelFetch(heights, ivec3(x, y, layer), 0).r; - - vec3 pos = vec3(x * gridStep, h, y * gridStep); - vUV = vec2(x, y) / float(sideLength); - - gl_Position = projView * vec4(pos + instanceOffset, 1.0); -} -``` - -**terrainclipmap.vert** (after): -```glsl -// Vertex attributes -layout(location = 0) in vec3 aPosition; -layout(location = 1) in vec2 aUV; -layout(location = 2) in uint aMaterial; - -layout(location = 0) out vec2 vUV; -layout(location = 1) flat out uint vMaterial; - -uniform vec3 uTileOffset; // from instance data - -void main() { - vUV = aUV; - vMaterial = aMaterial; - - // Position is already world-space, just offset to camera-relative - vec3 pos = aPosition + uTileOffset; - gl_Position = projView * vec4(pos, 1.0); -} -``` - -**terrainclipmap.frag** (minimal change): -```glsl -// Material lookup stays the same -layout(binding = 1) uniform sampler2DArray groundTextures; - -layout(location = 0) in vec2 vUV; -layout(location = 1) flat in uint vMaterial; - -void main() { - vec4 color = texture(groundTextures, vec3(vUV, vMaterial)); - fragColor = color; -} -``` - -### Critical Details - -#### 1. **Camera-Relative Rendering** - -Heightfield stored absolute positions in camera-relative space per frame. -TIN vertices are **world-absolute** - subtract camera pos in shader: - -```cpp -// In clipmap::render() -glm::dvec3 camera_pos = Viewpoint; - -// Per tile: -glm::vec3 tile_offset = glm::vec3( - tile_center - camera_pos // double → float is safe at this range -); -shader.uniform("uTileOffset", tile_offset); -``` - -#### 2. **LOD Morphing** - -Heightfield morphed in shader between LOD levels. -TIN has **separate geometry per LOD** - just render appropriate one. - -```cpp -// Select LOD based on distance (existing logic) -std::uint32_t lod = level_for(distance); - -// Fetch tile at that LOD -auto tile = m_tiles[make_key(x, z, lod)]; -``` - -#### 3. **Edge Stitching** (Phase 2) - -Edge cracks between LOD levels need special handling: - -```cpp -// Along tile borders, sample neighbor's edge vertices -// Use edge indices from payload - -if (on_north_edge && neighbor_lod > my_lod) { - // Fetch position from neighbor's south_edge - position = neighbor_south_edge[edge_index]; -} -``` - -This can wait - without it, there might be thin cracks but rendering works. - -### Files to Modify - -``` -rendering/terrainclipmap.cppm - Main renderer -rendering/terrainclipmap.cpp - Implementation -rendering/terraintilepool.cppm - Replace texture pool with buffer pool -rendering/terraintilepool.cpp - Implementation -shaders/terrainclipmap.vert - Vertex shader -shaders/terrainclipmap.frag - Fragment shader (minimal) -``` - -### Files to Remove (Later) - -``` -scene/heightfieldformat.h - Old format -scene/heightfieldreader.h - Old reader -scene/terraincooker.h - Old cooker -rendering/terrainground.cppm - May be reusable for material textures -``` - -### Testing Strategy - -1. **Render single tile** - - Load one `.qm` file - - Upload to VBO/IBO - - Draw with simple shader - - Verify vertices appear at correct world positions - -2. **Render LOD 0 only** - - Load all tiles at finest LOD - - No morphing, no stitching - - Check coverage, no gaps - -3. **Add LOD selection** - - Distance-based LOD (reuse existing formula) - - Verify coarser LODs appear at distance - -4. **Add edge stitching** - - Use edge indices - - Eliminate cracks - -5. **Performance comparison** - - Frame time vs old heightfield - - Memory usage - - Load time - -### Next Session: Implement Phase 1 - -1. Create `terrain_tile_pool_tin.{cppm,cpp}` -2. Modify `terrain_clipmap::upload()` to use new pool -3. Modify `terrain_clipmap::render()` for per-tile draws -4. Update shaders - -**Estimate**: 4-6 hours for working render, another 2-3 for edge stitching. - ---- - -**Ready to start renderer implementation now.** diff --git a/TIN_SESSION_SUMMARY.md b/TIN_SESSION_SUMMARY.md deleted file mode 100644 index ca0f03ad..00000000 --- a/TIN_SESSION_SUMMARY.md +++ /dev/null @@ -1,200 +0,0 @@ -# TIN Terrain Implementation - Session Summary - -## ✅ Completed in This Session - -### 1. Format & Triangulation (Complete) -- `scene/quantizedmeshformat.h` - Cesium Quantized Mesh format specification -- `scene/terraincooker_tin.{h,cpp}` - CDT-based triangulator with LOD generation -- `scene/quantizedmeshreader.h` - Reader for `.qm` tile files - -### 2. Tool Integration (Complete) -- `tools/terraincook/main.cpp` - `-cook` now generates TIN by default -- Removed heightfield-specific flags -- TIN is the primary terrain format - -### 3. Loader (Complete) -- `rendering/terraintileloader.{cppm,cpp}` - Fully converted to TIN -- Reads `.qm` files instead of `.ehf` -- Payload contains vertices, indices, UVs, materials -- Thread pool, priority queue all preserved - -### 4. Renderer - In Progress -#### Completed: -- `rendering/terraintilepool_tin.{cppm,cpp}` - VBO/IBO pool for tile storage -- `rendering/terrainclipmap.cppm` - Interface updated for TIN rendering - - Changed imports (quantizedmesh, terraintilepool_tin) - - Updated resident_tile structure (VBO/IBO handles instead of texture slots) - - Removed heightfield-specific members (texture arrays, instance buffers) - - Added TIN-specific members (tile pool, simplified uniforms) - -#### Remaining: -- `rendering/terrainclipmap.cpp` - Implementation needs rewrite: - - `open()` - Initialize from directory instead of single file - - `upload()` - Upload to VBO/IBO instead of texture - - `render()` - Per-tile indexed draws instead of instanced - - `scan()` - Minimal changes (LOD selection logic reusable) - - Remove `build_indices()` - not needed for TIN - -### 5. Shaders - Not Started -- `shaders/terrainclipmap.vert` - Needs vertex attributes instead of texture fetch -- `shaders/terrainclipmap.frag` - Minimal changes (material lookup) - -## 📊 Key Design Decisions Made - -### 1. **Full Replacement, Not Hybrid** -Heightfield code remains in tree but TIN is the **only** format used going forward. -- Simpler: one code path -- Cleaner: no runtime format detection -- Faster development: no compatibility layer - -### 2. **Vertex Buffer Per Tile** -```cpp -struct vertex { - float x, y, z; // 12 bytes - float u, v; // 8 bytes - uint8_t material; // 1 byte - uint8_t pad[3]; // alignment -}; // 24 bytes total -``` - -Each tile = one VBO + one IBO. No shared vertex pools or complex allocation. - -### 3. **Camera-Relative Positions in VBO** -Vertices stored relative to tile origin, avoiding float precision issues: -```cpp -v.x = float(world_x - tile_center_x); // Safe: always < 256m -``` - -### 4. **LOD Selection Reused** -Existing CDLOD distance formula preserved - only rendering path changes. - -## 🔧 Technical Details - -### Memory Layout -**Old (Heightfield)**: -``` -Texture2DArray per LOD: -- 129×129 R16 heights = 33KB per tile -- 129×129 R8 materials = 17KB -Total per LOD level: 50KB × tiles -``` - -**New (TIN)**: -``` -VBO + IBO per tile: -- ~2000 vertices × 24 bytes = 48KB -- ~6000 indices × 4 bytes = 24KB -Total: ~72KB per tile (but ~50% fewer triangles) -``` - -Net: **Similar memory**, but better quality where it matters. - -### Rendering Path -**Old**: -```cpp -for each LOD level: - bind texture array - gather instances - glDrawElementsInstanced(all tiles at this LOD) -``` - -**New**: -```cpp -for each tile in view: - glBindVertexArray(tile.vao) - uniform(tile_offset, tile.origin - camera_pos) - glDrawElements(tile.index_count) -``` - -More draw calls, but modern GL handles this fine (< 1000 tiles/frame typical). - -## 🚧 Remaining Work - -### Critical Path (to working render): -1. **terrainclipmap.cpp** - Rewrite 5 key methods (~200 lines) -2. **Vertex shader** - Replace texture fetch with attributes (~30 lines) -3. **Fragment shader** - Trivial material lookup change (~5 lines) - -### Nice-to-Have: -4. **Edge stitching** - Use edge indices to eliminate LOD cracks -5. **Constraint extraction** - Auto-generate from track geometry -6. **Material baking** - Proper per-vertex material assignment - -## 📝 Next Session Plan - -### Phase 1: Basic Rendering (2-3 hours) -1. Implement `terrain_clipmap::open()` - read directory, init pool -2. Implement `terrain_clipmap::upload()` - call pool.upload() -3. Implement `terrain_clipmap::render()` - per-tile draws -4. Write minimal shaders (attributes in, color out) -5. **Test**: single tile renders at correct position - -### Phase 2: Full System (1-2 hours) -6. Implement `terrain_clipmap::scan()` - reuse LOD logic -7. Test multi-tile, LOD selection -8. Material textures working - -### Phase 3: Polish (1-2 hours) -9. Performance tune (batch draws, cull aggressively) -10. Edge stitching (if cracks visible) -11. Memory budget enforcement - -**Total estimate**: 4-7 hours to complete working TIN terrain. - -## 🎯 Success Criteria - -- [ ] Loads `.qm` tiles from directory -- [ ] Renders terrain at correct world positions -- [ ] LOD selection based on distance -- [ ] Materials display correctly -- [ ] No worse performance than heightfield -- [ ] Can fly entire l204 without hitches - -## 📦 Files Modified (19 total) - -**New files (8)**: -- `scene/quantizedmeshformat.h` -- `scene/quantizedmeshreader.h` -- `scene/terraincooker_tin.h` -- `scene/terraincooker_tin.cpp` -- `rendering/terraintilepool_tin.cppm` -- `rendering/terraintilepool_tin.cpp` -- `TIN_IMPLEMENTATION.md` -- `TIN_STATUS.md` -- `TIN_RENDERER_PLAN.md` - -**Modified files (11)**: -- `tools/terraincook/main.cpp` -- `tools/terraincook/CMakeLists.txt` -- `rendering/terraintileloader.cppm` -- `rendering/terraintileloader.cpp` -- `rendering/terrainclipmap.cppm` -- `rendering/terrainclipmap.cpp` (in progress) -- `shaders/terrainclipmap.vert` (not started) -- `shaders/terrainclipmap.frag` (not started) - -## 🐛 Known Issues - -1. **terrainclipmap.cpp not yet updated** - Still has heightfield implementation -2. **Shaders not updated** - Still expect texture arrays -3. **No build tested** - CDT dependency needs vcpkg setup -4. **No visual verification** - Can't test until shaders done - -## 💡 Key Insights - -1. **TIN is not just "better heightfield"** - It's a fundamentally different data structure - - Heightfield: regular grid, texture-based - - TIN: irregular mesh, vertex-based - -2. **The win comes from adaptivity** - Same visual quality with 50% fewer triangles because detail concentrates where needed - -3. **Format is proven** - Cesium Quantized Mesh used in production at massive scale (globe-scale terrain streaming) - -4. **Implementation is straightforward** - Once you commit to vertex buffers, the rest follows naturally - ---- - -**Ready to continue with terrainclipmap.cpp implementation in next session.** - -Current branch: `feature/scene-streaming-format` -All changes committed? **Not yet** - working files only. diff --git a/TIN_STATUS.md b/TIN_STATUS.md deleted file mode 100644 index b027efe4..00000000 --- a/TIN_STATUS.md +++ /dev/null @@ -1,162 +0,0 @@ -# TIN Terrain Format - Implementation Status - -## ✅ Completed (Phase 1: Cooker) - -### Core Infrastructure -- **Format specification** (`quantizedmeshformat.h`) - Cesium Quantized Mesh adapted for MaSzyna -- **Triangulator** (`terraincooker_tin.{h,cpp}`) - CDT-based constrained Delaunay with LOD generation -- **Tool integration** (`terraincook -cooktin`) - Export to `.qm` tile files - -### Key Features -- Multi-LOD generation (4 levels, configurable) -- Edge-collapse simplification (~50% vertex reduction) -- Seamless tile stitching via edge indices -- Constraint edge support (for track corridors) -- Quantized vertex positions (uint16, 2mm precision) -- Optional zstd compression per tile - -### Output Format -``` -terrain_X_Z_lodN.qm files: -- Header: 128 bytes (bounds, counts, error metrics) -- Vertices: quantized uint16[3] XYZ + uint16[2] UV -- Indices: uint16/uint32 triangle list -- Edge indices: north/south/east/west for stitching -``` - -## 🚧 Remaining (Phase 2: Runtime) - -### 1. Loader (Critical Path) -**File**: `rendering/terraintileloader.{cppm,cpp}` - -Current: Reads `.ehf` heightfield tiles -Needs: Read `.qm` Quantized Mesh tiles - -```cpp -struct quantizedmesh_payload { - std::vector vertices; // dequantized on load - std::vector indices; - std::vector edge_indices[4]; // N/S/E/W - // ... reuse existing threading/priority logic -}; -``` - -**Estimate**: 1-2 days (parallel to heightfield reader, reuse thread pool) - -### 2. Renderer (Main Integration) -**File**: `rendering/terrainclipmap.{cppm,cpp}` - -Current: Texture array + compute shader -Needs: Vertex buffer pool + index buffers - -Changes: -- Replace `terrain_tile_pool` (texture arrays) → vertex buffer pool -- Upload vertices as `VBO` instead of texture layers -- Per-LOD index buffers (already have LOD selection logic) -- Edge stitching: sample coarser LOD along tile borders - -**Estimate**: 2-3 days (shader changes, buffer management) - -### 3. Material Blending (Polish) -Current: Material splat via texture -Needs: Per-vertex material indices - -Options: -- A) Per-vertex material index → texture array lookup (1 extra attrib) -- B) Bake splat into vertex colors (simpler, less flexible) -- C) Material ID texture (render-to-texture during cook) - -**Estimate**: 1 day - -### 4. Constraint Extraction (Quality) -Current: Manual constraint input -Needs: Auto-generate from track geometry - -```cpp -void extract_track_corridor( - std::vector const &Tracks, - double CorridorWidth, - terrain::tin_cooker &Cooker -); -``` - -Parse track beziers → sample → offset by width → constraints -**Estimate**: 1 day (track geometry already available) - -## 📊 Expected Results - -Based on `binary-scene-format-plan.md` measurements: - -| Metric | Heightfield (2m) | TIN (adaptive) | Improvement | -|--------|------------------|----------------|-------------| -| **File size** | 186 MB | ~100 MB | 46% smaller | -| **Triangles** | 8.8M | ~4.4M | 50% fewer | -| **Memory (clipmap)** | ~250 MB | ~150 MB | 40% less | -| **Detail corridor** | Uniform | Adaptive | Better where needed | -| **Detail field** | Oversampled | Right-sized | No waste | - -## 🎯 Next Session Plan - -**Option A: Complete the pipeline (loader + renderer)** -- Implement `.qm` loader (reuse existing thread pool) -- Adapt clipmap to render from vertex buffers -- Test on l204 terrain tiles -- Performance comparison vs heightfield - -**Option B: Validate triangulation quality first** -- Add preview export (triangle density heatmap) -- Verify edge constraints preserved -- Check simplification doesn't create artifacts -- Tune LOD error thresholds - -**Recommendation**: **Option A** - finish the working pipeline, then iterate on quality. -Rationale: Can't properly evaluate triangulation quality without seeing it rendered. - -## 🔧 Build Notes - -### Missing: vcpkg manifest entry for CDT -Add to `vcpkg.json` (if using manifest mode): -```json -{ - "dependencies": [ - "cdt" - ] -} -``` - -Or install globally: -```bash -vcpkg install cdt -``` - -### Compiler flags -CDT is header-only, requires C++17+. Already satisfied (project uses C++20). - -## 📝 Code Quality - -All new code follows project conventions: -- MPL-2.0 license headers -- Comments explaining "why", not "what" -- Shared interface with heightfield cooker (can swap at runtime) -- No external dependencies beyond CDT (which is header-only) - -## 🐛 Known Limitations (TODO) - -1. **Simplification is naive** - uses height error only, should consider slope/curvature -2. **Material assignment is placeholder** - needs proper per-vertex material baking -3. **No meshlet support yet** - extension defined but not generated -4. **Edge stitching untested** - need actual rendering to verify no cracks - -## ⏱️ Timeline Estimate - -- **Loader**: 1-2 days -- **Renderer**: 2-3 days -- **Material blending**: 1 day -- **Constraint extraction**: 1 day -- **Testing & tuning**: 2-3 days - -**Total**: ~1.5-2 weeks to complete TIN terrain system - ---- - -**Ready to continue with loader implementation?** All infrastructure is in place. diff --git a/application/driveruipanels.cpp b/application/driveruipanels.cpp index f138de24..777480b1 100644 --- a/application/driveruipanels.cpp +++ b/application/driveruipanels.cpp @@ -1512,26 +1512,24 @@ debug_panel::update_section_renderer( std::vector &Output ) { auto const slash { field.path.find_last_of( "/\\" ) }; Output.emplace_back( "Terrain: " + ( slash == std::string::npos ? field.path : field.path.substr( slash + 1 ) ) - + ", grid " + to_string( field.gridstep, 1 ) + " m, tile " + to_string( field.tilesize, 0 ) + " m" - + ", loaded out to " + to_string( field.range, 0 ) + " m", + + ", finest tile " + to_string( field.tilesize, 0 ) + " m, " + std::to_string( field.levels ) + " levels" + + ", loaded out to " + to_string( field.range, 0 ) + " m, detail " + to_string( field.detail, 1 ) + " px", Global.UITextColor ); Output.emplace_back( - " tiles: " + std::to_string( stats.inview ) + " in range, " - + std::to_string( stats.drawn ) + " drawn, " - + std::to_string( stats.resident ) + " on gpu (" + std::to_string( stats.texturebytes / 1048576 ) + " MB), " + " tiles: " + std::to_string( stats.chosen ) + " chosen, " + + std::to_string( stats.drawn ) + " drawn (" + std::to_string( stats.triangles / 1000 ) + "k triangles), " + + std::to_string( stats.resident ) + " on gpu (" + std::to_string( stats.gpubytes / 1048576 ) + " MB), " + std::to_string( stats.wanted ) + " wanted, " + std::to_string( field.backlog ) + " in loader, " + std::to_string( stats.uploaded ) + " uploaded, " + std::to_string( stats.dropped ) + " dropped stale", Global.UITextColor ); std::string levels { " levels:" }; for( std::uint32_t level = 0; ( level < field.levels ) && ( level < stats.perlevel.size() ); ++level ) { - auto const last { level + 1 == field.levels }; levels += " L" + std::to_string( level ) - + ( last ? " beyond " + to_string( terrain_level_distance( level > 0 ? level - 1 : 0, field.finest ), 0 ) - : " <" + to_string( terrain_level_distance( level, field.finest ), 0 ) ) - + " m (" + to_string( field.gridstep * static_cast( 1u << level ), 0 ) + " m grid): " + + " (" + to_string( field.leveltile[ level ], 0 ) + " m tile, " + + to_string( field.levelerror[ level ], 2 ) + " m off): " + std::to_string( stats.perlevel[ level ] ); - if( false == last ) { levels += " |"; } + if( level + 1 < field.levels ) { levels += " |"; } } Output.emplace_back( levels, Global.UITextColor ); } diff --git a/rendering/opengl33renderer.cpp b/rendering/opengl33renderer.cpp index 1bd69e57..7fa8a6da 100644 --- a/rendering/opengl33renderer.cpp +++ b/rendering/opengl33renderer.cpp @@ -2932,12 +2932,16 @@ void opengl33_renderer::Render_terrain() if (status.path.empty()) { status.path = field.path(); - status.gridstep = field.gridstep(); status.tilesize = field.tilesize(); status.levels = field.levels(); + for (std::uint32_t level = 0; level < field.levels() && level < status.leveltile.size(); ++level) + { + status.leveltile[level] = field.leveltile(level); + status.levelerror[level] = field.levelerror(level); + } } status.range = field.range(); - status.finest = field.finest(); + status.detail = field.detailpixels(); status.backlog = field.backlog(); status.stats = field.stats(); } diff --git a/rendering/terrainclipmap.cpp b/rendering/terrainclipmap.cpp index 43128c5c..7a685897 100644 --- a/rendering/terrainclipmap.cpp +++ b/rendering/terrainclipmap.cpp @@ -9,28 +9,27 @@ http://mozilla.org/MPL/2.0/. module; #include -#include #include +#include #include #include -#include #include #include #include +#include #include #include "glad/glad.h" -#include "scene/heightfieldformat.h" -#include "scene/heightfieldreader.h" +#include "scene/quantizedmeshreader.h" module eu07.rendering.terrainclipmap; import eu07.glm; import eu07.utilities.logs; import eu07.model.texture; - terrain_clipmap::~terrain_clipmap() { + close(); } @@ -39,19 +38,34 @@ terrain_clipmap::open( std::string const &Path ) { close(); - if( ( false == m_reader.open( Path ) ) - || ( false == m_loader.open( Path ) ) ) { - ErrorLog( "Terrain: cannot open heightfield \"" + Path + "\"" ); + if( false == m_reader.open( Path ) ) { + ErrorLog( "Terrain: cannot read cooked terrain \"" + Path + "\"" ); + return false; + } + m_table = m_reader.table(); + m_levels = std::min( m_table.levels, terrain_maxlevels ); + m_finesttile = static_cast( m_table.tilesize( m_levels - 1 ) ); + // every vertex of every tile is measured from the middle of the terrain, so that the whole of + // it fits in floats: at ten kilometres a float still holds a millimetre + m_origin = glm::dvec3 { + m_table.west + m_table.side * 0.5, + ( m_table.lowest + m_table.highest ) * 0.5, + m_table.north + m_table.side * 0.5 }; + + if( false == m_loader.open( Path, m_origin.x, m_origin.y, m_origin.z ) ) { + ErrorLog( "Terrain: cannot open \"" + Path + "\" for reading" ); + close(); + return false; + } + if( false == m_arena.create() ) { + ErrorLog( "Terrain: cannot allocate the terrain buffers" ); close(); return false; } - m_path = Path; - build_indices(); - try { - gl::shader vertex( "terrainclipmap.vert" ); - gl::shader fragment( "terrainclipmap.frag" ); + gl::shader vertex( "terrainmesh.vert" ); + gl::shader fragment( "terrainmesh.frag" ); m_shader.emplace( std::vector>( { vertex, fragment } ) ); } catch( gl::shader_exception const &error ) { @@ -59,48 +73,44 @@ terrain_clipmap::open( std::string const &Path ) { close(); return false; } - if( false == m_ground.create( m_reader.materials(), m_reader.material_scales() ) ) { + + std::vector names; + std::vector repeats; + names.reserve( m_table.materials.size() ); + repeats.reserve( m_table.materials.size() ); + for( auto const &material : m_table.materials ) { + names.push_back( material.name ); + repeats.push_back( material.repeat ); + } + if( false == m_ground.create( names, repeats ) ) { + ErrorLog( "Terrain: ground textures failed" ); close(); return false; } - for( std::uint32_t level = 0; level < m_reader.levels(); ++level ) { - m_pools.emplace_back( std::make_unique( m_reader.samples_per_side( level ) ) ); - } - m_instances.resize( m_reader.levels() ); - m_instancebuffer.emplace(); - ::glGenTextures( 1, &m_instancetexture ); - opengl_texture::reset_unit_cache(); - - m_vao.emplace(); resolve_uniforms(); + m_path = Path; m_ready = true; WriteLog( - "Terrain: " + Path + ", " + std::to_string( m_reader.tiles().size() ) + " tiles, " - + std::to_string( m_reader.levels() ) + " levels, grid " - + std::to_string( m_reader.gridstep() ) + " m, tile " + std::to_string( m_reader.tilesize() ) + " m" ); - + "Terrain: " + Path + ", " + std::to_string( m_levels ) + " levels, finest tile " + + std::to_string( static_cast( m_finesttile ) ) + " m over " + + std::to_string( static_cast( m_table.side ) ) + " m, " + + std::to_string( m_reader.present().size() ) + " tiles, " + + std::to_string( m_table.materials.size() ) + " ground materials" ); return true; } -// uniform locations do not change once the program is linked, so they are looked up here -// rather than per draw +// uniform locations do not change once the program is linked, so they are looked up here rather +// than per draw void terrain_clipmap::resolve_uniforms() { auto const program { static_cast( *m_shader ) }; - m_uniforms.samplestep = ::glGetUniformLocation( program, "samplestep" ); - m_uniforms.side = ::glGetUniformLocation( program, "side" ); - m_uniforms.morph = ::glGetUniformLocation( program, "morph" ); - m_uniforms.heightbias = ::glGetUniformLocation( program, "heightbias" ); - m_uniforms.heightscale = ::glGetUniformLocation( program, "heightscale" ); - m_uniforms.nodata = ::glGetUniformLocation( program, "nodata" ); - m_uniforms.heights = ::glGetUniformLocation( program, "heights" ); - m_uniforms.materials = ::glGetUniformLocation( program, "materials" ); - m_uniforms.instances = ::glGetUniformLocation( program, "instances" ); + m_uniforms.origin = ::glGetUniformLocation( program, "terrainorigin" ); + m_uniforms.world = ::glGetUniformLocation( program, "terrainworld" ); m_uniforms.ground = ::glGetUniformLocation( program, "ground" ); - m_uniforms.materialtable = ::glGetUniformLocation( program, "materialtable" ); + m_uniforms.materials = ::glGetUniformLocation( program, "materialtable" ); } void @@ -108,367 +118,368 @@ terrain_clipmap::close() { // the loader first, so nothing arrives for tiles that are being torn down m_loader.close(); + m_tiles.clear(); + m_chosen.clear(); + m_wishes.clear(); m_wanted.clear(); m_arrived.clear(); - m_inrange.clear(); - m_wantedlevel.clear(); - m_scanrange = -1.f; - m_tiles.clear(); - m_pools.clear(); - m_indices.clear(); - m_indexcounts.clear(); - m_instances.clear(); - m_instancebuffer.reset(); - if( m_instancetexture != 0 ) { ::glDeleteTextures( 1, &m_instancetexture ); m_instancetexture = 0; } + m_asked.clear(); + m_readiness.clear(); + m_arena.destroy(); m_ground.destroy(); - m_vao.reset(); m_shader.reset(); m_reader.close(); + m_table = {}; m_stats = {}; + m_walked = false; + m_walkrange = -1.f; m_ready = false; + m_path.clear(); } -// index buffers, one per mip level. the grid they describe is the same shape at every -// level, only smaller, and the vertices themselves never exist in memory: the shader -// derives a sample's row and column from gl_VertexID -void -terrain_clipmap::build_indices() { +float +terrain_clipmap::leveltile( std::uint32_t const Level ) const { - m_indices.resize( m_reader.levels() ); - m_indexcounts.resize( m_reader.levels() ); - - std::vector indices; - for( std::uint32_t level = 0; level < m_reader.levels(); ++level ) { - - auto const side { m_reader.samples_per_side( level ) }; - indices.clear(); - indices.reserve( static_cast( side - 1 ) * ( side - 1 ) * 6 ); - - for( std::uint32_t row = 0; row + 1 < side; ++row ) { - for( std::uint32_t column = 0; column + 1 < side; ++column ) { - auto const topleft { row * side + column }; - auto const topright { topleft + 1 }; - auto const bottomleft { topleft + side }; - auto const bottomright { bottomleft + 1 }; - indices.push_back( topleft ); indices.push_back( bottomleft ); indices.push_back( topright ); - indices.push_back( topright ); indices.push_back( bottomleft ); indices.push_back( bottomright ); - } - } - - m_indexcounts[ level ] = static_cast( indices.size() ); - m_indices[ level ] = std::make_unique(); - m_indices[ level ]->allocate( - gl::buffer::ELEMENT_ARRAY_BUFFER, indices.size() * sizeof( std::uint32_t ), GL_STATIC_DRAW ); - m_indices[ level ]->upload( - gl::buffer::ELEMENT_ARRAY_BUFFER, indices.data(), 0, indices.size() * sizeof( std::uint32_t ) ); - } + return static_cast( m_table.tilesize( Level ) ); } +float +terrain_clipmap::levelerror( std::uint32_t const Level ) const { -std::uint32_t -terrain_clipmap::level_for( double const Distance ) const { - - if( Distance <= m_finest ) { return 0; } - auto const steps { static_cast( std::log2( Distance / m_finest ) ) + 1 }; - return std::min( steps, m_reader.levels() - 1 ); -} - -void -terrain_clipmap::upload( terrain_tile_loader::payload const &Tile ) { - - auto const tilekey { key( Tile.tile.x, Tile.tile.z ) }; - auto const existing { m_tiles.find( tilekey ) }; - if( existing != m_tiles.end() ) { - // a level change moves the tile to another pool - retire( existing->second ); - m_tiles.erase( existing ); - } - - auto &pool { *m_pools[ Tile.tile.level ] }; - resident_tile tile; - tile.x = Tile.tile.x; - tile.z = Tile.tile.z; - tile.level = Tile.tile.level; - tile.slot = pool.acquire(); - tile.lastused = m_frame; - pool.upload( tile.slot, Tile.heights.data(), Tile.materials.data() ); - - if( auto const *description { m_reader.find( Tile.tile.x, Tile.tile.z ) } ) { - tile.minheight = description->minheight; - tile.maxheight = description->maxheight; - } - - ++m_stats.uploaded; - m_tiles.emplace( tilekey, tile ); -} - -void -terrain_clipmap::retire( resident_tile const &Tile ) { - - m_pools[ Tile.level ]->release( Tile.slot ); + return m_table.error( Level ); } void terrain_clipmap::detail( double const Pixelsperunit, double const Pixels ) { - m_finestwanted = terrain_finest_range( m_reader.gridstep(), m_reader.tilesize(), Pixelsperunit, Pixels ); + m_pixelsperunit = Pixelsperunit; + m_detail = std::max( 0.1, Pixels ); } void -terrain_clipmap::scan( glm::dvec3 const &Viewpoint ) { +terrain_clipmap::square( quantizedmesh::tile_address const &Tile, double &West, double &North, double &Side ) const { - m_scanpoint = Viewpoint; - m_scanrange = m_range; - m_finest = ( m_finestwanted > 0.0 ? m_finestwanted : terrain_finest_tiles * m_reader.tilesize() ); - m_inrange.clear(); - m_wantedlevel.clear(); + Side = m_table.tilesize( Tile.level ); + West = m_table.west + Tile.x * Side; + North = m_table.north + Tile.z * Side; +} - auto const tilesize { static_cast( m_reader.tilesize() ) }; - auto const reach { std::max( 1, static_cast( m_range / tilesize ) + 1 ) }; - auto const centrex { m_reader.tile_x( Viewpoint.x ) }; - auto const centrez { m_reader.tile_z( Viewpoint.z ) }; - std::vector> wishes; +// the distance to the nearest point of the tile's square, not to its centre: a coarse tile is +// kilometres across, and the one the camera stands on has its centre far outside any draw range +double +terrain_clipmap::distance_to( quantizedmesh::tile_address const &Tile, glm::dvec3 const &Viewpoint ) const { - for( std::int32_t z = centrez - reach; z <= centrez + reach; ++z ) { - for( std::int32_t x = centrex - reach; x <= centrex + reach; ++x ) { + double west { 0.0 }, north { 0.0 }, side { 0.0 }; + square( Tile, west, north, side ); + auto const nearestx { std::clamp( Viewpoint.x, west, west + side ) }; + auto const nearestz { std::clamp( Viewpoint.z, north, north + side ) }; + return glm::length( glm::dvec2 { nearestx - Viewpoint.x, nearestz - Viewpoint.z } ); +} - if( false == m_reader.contains( x, z ) ) { continue; } +bool +terrain_clipmap::present( quantizedmesh::tile_address const &Tile ) const { - // distance to the nearest point of the tile, not to its centre. a regional - // field has tiles kilometres across, and a tile the camera is standing on - // can have its centre far outside the draw range - auto const west { x * tilesize }; - auto const north { z * tilesize }; - auto const nearestx { std::clamp( Viewpoint.x, west, west + tilesize ) }; - auto const nearestz { std::clamp( Viewpoint.z, north, north + tilesize ) }; - auto const distance { - glm::length( glm::dvec2 { nearestx - Viewpoint.x, nearestz - Viewpoint.z } ) }; - if( distance > m_range ) { continue; } + return m_reader.present().contains( quantizedmesh::tile_key( Tile ) ); +} - auto const tilekey { key( x, z ) }; - auto const level { level_for( distance ) }; - m_inrange.push_back( { tilekey, level } ); - m_wantedlevel.emplace( tilekey, level ); +// Whether the ground a tile stands for is close enough to the real thing. The tile's error, at its +// distance, covers so many pixels; while that is fewer than the detail asks for there is nothing to +// be had by splitting it. The tile the camera stands on is always split, whatever its error, so that +// what is underfoot is the ground as the scenery drew it. +bool +terrain_clipmap::good_enough( quantizedmesh::tile_address const &Tile, double const Distance ) const { - auto const resident { m_tiles.find( tilekey ) }; - if( ( resident == m_tiles.end() ) || ( resident->second.level != level ) ) { - wishes.push_back( { distance, { x, z, level } } ); - } - } + if( Tile.level + 1 >= m_levels ) { return true; } + if( Distance <= 0.0 ) { return false; } + return terrain_level_enough( m_table.error( Tile.level ), Distance, m_pixelsperunit, m_detail ); +} + +// Walks down to the levels the screen wants and asks the loader for whatever of them is not on the +// card, deepest tile of the wanted cut included - not only the tiles one level down. Asking one level +// at a time is what a first attempt at this did, and it stops dead: a tile whose children are all +// resident asks for nothing, while what it is waiting for is their children. +// +// Returns whether the cut under this tile, and the tile itself where the cut ends at it, is there. A +// tile whose subtree is not ready yet is asked for as well, so that something coarse covers the +// ground rather than a hole while the rest is read. +bool +terrain_clipmap::need( quantizedmesh::tile_address const &Tile, glm::dvec3 const &Viewpoint ) { + + if( false == present( Tile ) ) { return true; } // no ground here to draw + + auto const distance { distance_to( Tile, Viewpoint ) }; + if( distance > m_range ) { return true; } // nothing wanted this far out + + auto const key { quantizedmesh::tile_key( Tile ) }; + auto const resident { m_tiles.contains( key ) }; + + if( true == good_enough( Tile, distance ) ) { + if( false == resident ) { ask_for( Tile, distance ); } + m_readiness.emplace( key, resident ); + return resident; } + auto ready { true }; + for( std::int32_t stepz = 0; stepz < 2; ++stepz ) { + for( std::int32_t stepx = 0; stepx < 2; ++stepx ) { + // every child is asked, so the whole cut is queued at once rather than a level a frame + ready = need( { Tile.level + 1, Tile.x * 2 + stepx, Tile.z * 2 + stepz }, Viewpoint ) && ready; + } + } + if( ( false == ready ) && ( false == resident ) ) { ask_for( Tile, distance ); } + m_readiness.emplace( key, ready ); + return ready; +} + +void +terrain_clipmap::choose( quantizedmesh::tile_address const &Tile, glm::dvec3 const &Viewpoint ) { + + if( false == present( Tile ) ) { return; } + + auto const distance { distance_to( Tile, Viewpoint ) }; + if( distance > m_range ) { return; } + + auto const key { quantizedmesh::tile_key( Tile ) }; + auto const ready { m_readiness.find( key ) }; + if( ( ready != m_readiness.end() ) && ( true == ready->second ) + && ( false == good_enough( Tile, distance ) ) ) { + // the whole cut under this tile is on the card: draw that instead + for( std::int32_t stepz = 0; stepz < 2; ++stepz ) { + for( std::int32_t stepx = 0; stepx < 2; ++stepx ) { + choose( { Tile.level + 1, Tile.x * 2 + stepx, Tile.z * 2 + stepz }, Viewpoint ); + } + } + return; + } + // either this level is good enough, or what is below it is still being read: this tile covers + // the ground meanwhile, and nothing under it is drawn, so nothing is drawn twice + if( true == m_tiles.contains( key ) ) { m_chosen.push_back( key ); } +} + +void +terrain_clipmap::ask_for( quantizedmesh::tile_address const &Tile, double const Distance ) { + + if( false == m_asked.insert( quantizedmesh::tile_key( Tile ) ).second ) { return; } + m_wishes.emplace_back( Distance, Tile ); +} + +void +terrain_clipmap::walk( glm::dvec3 const &Viewpoint ) { + + m_walkpoint = Viewpoint; + m_walkrange = m_range; + m_walkdetail = m_detail; + m_walked = true; + m_chosen.clear(); + m_wishes.clear(); + m_asked.clear(); + m_readiness.clear(); + + need( { 0, 0, 0 }, Viewpoint ); + choose( { 0, 0, 0 }, Viewpoint ); + // nearest first, so the ground under the camera is never waiting behind the horizon - std::sort( wishes.begin(), wishes.end(), + std::sort( m_wishes.begin(), m_wishes.end(), []( auto const &Left, auto const &Right ) { return Left.first < Right.first; } ); m_wanted.clear(); - for( auto const &wish : wishes ) { m_wanted.push_back( wish.second ); } + m_wanted.reserve( m_wishes.size() ); + for( auto const &wish : m_wishes ) { m_wanted.push_back( wish.second ); } m_loader.want( m_wanted ); + + m_stats.chosen = m_chosen.size(); + m_stats.wanted = m_wanted.size(); +} + +void +terrain_clipmap::put_in( terrain_tile_loader::payload const &Tile ) { + + auto const key { quantizedmesh::tile_key( Tile.data.tile ) }; + auto const existing { m_tiles.find( key ) }; + if( existing != m_tiles.end() ) { + // already there; nothing about a tile changes between reads + ++m_stats.dropped; + return; + } + + auto piece { m_arena.put( Tile.data ) }; + if( false == piece.held() ) { return; } + + resident_tile tile; + tile.piece = piece; + tile.centre = glm::dvec3 { Tile.data.centrex, Tile.data.centrey, Tile.data.centrez }; + tile.radius = static_cast( Tile.data.radius ); + tile.level = Tile.data.tile.level; + tile.lastused = m_frame; + m_tiles.emplace( key, tile ); + ++m_stats.uploaded; +} + +// What is held is what the walk has been choosing or asking for. Once the buffers are fuller than +// the budget, whatever has gone longest without either is given back - never something the walk +// wants now, because a coarse tile it is drawing is what stands in for its children while they are +// read. +void +terrain_clipmap::retire_unused() { + + if( m_arena.bytes() <= m_budget ) { return; } + + std::vector> aged; + aged.reserve( m_tiles.size() ); + for( auto const &[ key, tile ] : m_tiles ) { + if( tile.lastused != m_frame ) { aged.emplace_back( tile.lastused, key ); } + } + if( true == aged.empty() ) { return; } + std::sort( aged.begin(), aged.end() ); + + // a fifth of what is held, so that this does not run every frame + auto const letgo { std::max( 1, m_tiles.size() / 5 ) }; + for( std::size_t index = 0; ( index < letgo ) && ( index < aged.size() ); ++index ) { + auto const found { m_tiles.find( aged[ index ].second ) }; + if( found == m_tiles.end() ) { continue; } + m_arena.take( found->second.piece ); + m_tiles.erase( found ); + } } void terrain_clipmap::update( glm::dvec3 const &Viewpoint ) { if( false == m_ready ) { return; } - ++m_frame; - auto const moved { - glm::length( glm::dvec2 { Viewpoint.x - m_scanpoint.x, Viewpoint.z - m_scanpoint.z } ) }; - // a zoom or a resized window changes the levels; a change under a percent is not worth - // redoing the scan for - auto const detailchanged { - ( m_finestwanted > 0.0 ) && ( std::abs( m_finestwanted - m_finest ) > 0.01 * m_finest ) }; - if( ( m_range != m_scanrange ) - || ( true == detailchanged ) - || ( moved > m_reader.tilesize() * terrain_rescan_tiles ) ) { - scan( Viewpoint ); - } - - // the tiles in range are touched every frame, which is what render() draws and what - // eviction spares - m_stats.inview = 0; - m_stats.wanted = 0; - m_stats.perlevel.fill( 0 ); - for( auto const &tile : m_inrange ) { - auto const resident { m_tiles.find( tile.key ) }; - if( resident == m_tiles.end() ) { - ++m_stats.wanted; - continue; - } - // still wanted, at whatever level it has until the right one arrives - resident->second.lastused = m_frame; - ++m_stats.inview; - ++m_stats.perlevel[ std::min( resident->second.level, m_stats.perlevel.size() - 1 ) ]; - if( resident->second.level != tile.level ) { ++m_stats.wanted; } - } - m_arrived.clear(); - m_loader.collect( m_arrived, m_uploadsperframe ); + m_loader.collect( m_arrived, m_putsperframe ); for( auto const &tile : m_arrived ) { if( false == tile.valid ) { continue; } - // a tile read for where the camera was is put up only if nothing of it is on the gpu - // yet - some level is better than a hole. otherwise it is dropped, and the level + // a tile read for where the camera was is still put in: it covers ground, and what is // wanted now is already queued - auto const tilekey { key( tile.tile.x, tile.tile.z ) }; - auto const wanted { m_wantedlevel.find( tilekey ) }; - auto const stillwanted { ( wanted != m_wantedlevel.end() ) && ( wanted->second == tile.tile.level ) }; - if( stillwanted || ( false == m_tiles.contains( tilekey ) ) ) { - upload( tile ); - // put up this frame, so drawn this frame - m_tiles.find( tilekey )->second.lastused = m_frame; - } - else { - ++m_stats.dropped; - } + put_in( tile ); } - if( ( m_tiles.size() > m_budget ) && ( m_tiles.size() > m_stats.inview ) ) { - // over budget, and not everything is in view: drop whatever has gone longest without - // being asked for - std::vector> aged; - for( auto const & [ tilekey, tile ] : m_tiles ) { - if( tile.lastused != m_frame ) { aged.emplace_back( tile.lastused, tilekey ); } - } - auto const excess { std::min( aged.size(), m_tiles.size() - m_budget ) }; - std::partial_sort( aged.begin(), aged.begin() + excess, aged.end() ); - for( std::size_t index = 0; index < excess; ++index ) { - auto const victim { m_tiles.find( aged[ index ].second ) }; - retire( victim->second ); - m_tiles.erase( victim ); - } + auto const moved { + glm::length( glm::dvec2 { Viewpoint.x - m_walkpoint.x, Viewpoint.z - m_walkpoint.z } ) }; + auto const detailchanged { std::abs( m_detail - m_walkdetail ) > 0.01 * m_walkdetail }; + if( ( false == m_walked ) + || ( m_range != m_walkrange ) + || ( true == detailchanged ) + || ( moved > m_finesttile * terrain_rescan_tiles ) + || ( false == m_arrived.empty() ) ) { + // a tile that has just arrived lets the walk go deeper where it was drawing a coarse one + walk( Viewpoint ); } + // what the walk settled on is wanted now, and is what eviction spares + m_stats.perlevel.fill( 0 ); + for( auto const key : m_chosen ) { + auto const found { m_tiles.find( key ) }; + if( found == m_tiles.end() ) { continue; } + found->second.lastused = m_frame; + ++m_stats.perlevel[ std::min( found->second.level, m_stats.perlevel.size() - 1 ) ]; + } + + retire_unused(); m_stats.resident = m_tiles.size(); - m_stats.texturebytes = 0; - for( auto const &pool : m_pools ) { m_stats.texturebytes += pool->bytes(); } + m_stats.gpubytes = m_arena.bytes(); report_residency(); } - -// residency settles within a second of the camera stopping and then barely moves, so a -// line is written when it shifts by a noticeable amount rather than every frame. that -// makes the warm figure - what the terrain actually costs - readable from the log +// residency settles within a second of the camera stopping and then barely moves, so a line is +// written when it shifts by a noticeable amount rather than every frame. that makes the warm figure +// - what the terrain actually costs - readable from the log void terrain_clipmap::report_residency() { - auto const difference { - m_stats.resident > m_reportedresidency ? - m_stats.resident - m_reportedresidency : - m_reportedresidency - m_stats.resident }; - if( ( m_reported == true ) && ( difference < 16 ) ) { return; } + auto const apart { + []( std::size_t const Left, std::size_t const Right ) { + return Left > Right ? Left - Right : Right - Left; } }; + auto const moved { + ( apart( m_stats.resident, m_reportedresidency ) >= 8 ) + || ( apart( m_stats.drawn, m_reporteddrawn ) >= 4 ) }; + // at most once every couple of seconds, and only when the picture has actually changed: what + // this is for is the warm figure, what the terrain settles at, not a line per frame + if( ( m_reported == true ) && ( ( false == moved ) || ( m_frame < m_reportedframe + 120 ) ) ) { return; } m_reported = true; m_reportedresidency = m_stats.resident; + m_reporteddrawn = m_stats.drawn; + m_reportedframe = m_frame; WriteLog( "Terrain: " + std::to_string( m_stats.resident ) + " tiles resident, " - + std::to_string( m_stats.texturebytes / 1048576 ) + " MB of samples, " - + std::to_string( m_stats.uploaded ) + " uploads so far, " - + std::to_string( m_stats.drawn ) + " of " + std::to_string( m_stats.inview ) + " in range drawn, finest level out to " - + std::to_string( static_cast( m_finest ) ) + " m" ); + + std::to_string( m_stats.gpubytes / 1048576 ) + " MB of mesh, " + + std::to_string( m_stats.uploaded ) + " put in so far, " + + std::to_string( m_stats.drawn ) + " of " + std::to_string( m_stats.chosen ) + " chosen drawn, " + + std::to_string( m_stats.triangles / 1000 ) + "k triangles, levels " + + [ this ]() { + std::string counts; + for( std::uint32_t level = 0; level < m_levels; ++level ) { + counts += ( level > 0 ? "/" : "" ) + std::to_string( m_stats.perlevel[ level ] ); + } + return counts; }() ); } void terrain_clipmap::render( glm::dvec3 const &Viewpoint, visibility const &Visible ) { - if( ( false == m_ready ) || ( true == m_tiles.empty() ) ) { return; } + if( ( false == m_ready ) || ( true == m_chosen.empty() ) ) { return; } - // textures that finished loading since the last frame go into the ground array first + // the ground textures the engine has finished loading since the last frame m_ground.update(); - // what each level draws: the tiles touched by the last update, and seen. a tile's bounding - // sphere holds the whole tile, morphing included, since morphing only moves vertices - // towards samples of the same tile - auto const tilesize { static_cast( m_reader.tilesize() ) }; - for( auto &instances : m_instances ) { instances.clear(); } - for( auto const & [ tilekey, tile ] : m_tiles ) { - if( tile.lastused != m_frame ) { continue; } - auto const halfheight { 0.5 * ( tile.maxheight - tile.minheight ) }; - glm::dvec3 const centre { - ( tile.x + 0.5 ) * tilesize, 0.5 * ( tile.minheight + tile.maxheight ), ( tile.z + 0.5 ) * tilesize }; - auto const radius { - static_cast( std::sqrt( 0.5 * tilesize * tilesize + halfheight * halfheight ) ) }; - if( false == Visible( centre, radius ) ) { continue; } - // relative to the viewpoint, so that the shader never sees a coordinate large enough - // for single precision to matter - m_instances[ tile.level ].insert( - m_instances[ tile.level ].end(), - { static_cast( tile.x * tilesize - Viewpoint.x ), - static_cast( -Viewpoint.y ), - static_cast( tile.z * tilesize - Viewpoint.z ), - static_cast( tile.slot ), - static_cast( tile.x * tilesize ), - static_cast( tile.z * tilesize ), - 0.f, 0.f } ); + m_counts.clear(); + m_offsets.clear(); + m_bases.clear(); + std::size_t triangles { 0 }; + for( auto const key : m_chosen ) { + auto const found { m_tiles.find( key ) }; + if( found == m_tiles.end() ) { continue; } + auto const &tile { found->second }; + if( false == Visible( tile.centre, tile.radius ) ) { continue; } + m_counts.push_back( static_cast( tile.piece.indices ) ); + m_offsets.push_back( + reinterpret_cast( + static_cast( tile.piece.firstindex ) * sizeof( std::uint32_t ) ) ); + m_bases.push_back( static_cast( tile.piece.firstvertex ) ); + triangles += tile.piece.indices / 3; } + m_stats.drawn = m_counts.size(); + m_stats.triangles = triangles; + if( true == m_counts.empty() ) { return; } m_shader->bind(); - m_vao->bind(); - // the depth buffer is reversed - nearer is greater - so pushing back is a negative offset - ::glEnable( GL_POLYGON_OFFSET_FILL ); - ::glPolygonOffset( -1.f - 2.f * static_cast( m_depthrank ), -2.f - 4.f * static_cast( m_depthrank ) ); - - ::glUniform1f( m_uniforms.heightbias, m_reader.height( 0 ) ); - ::glUniform1f( m_uniforms.heightscale, m_reader.height( 1 ) - m_reader.height( 0 ) ); - ::glUniform1ui( m_uniforms.nodata, heightfield::nodata ); - ::glUniform1i( m_uniforms.heights, 0 ); + glm::vec3 const origin { + static_cast( m_origin.x - Viewpoint.x ), + static_cast( m_origin.y - Viewpoint.y ), + static_cast( m_origin.z - Viewpoint.z ) }; + ::glUniform3fv( m_uniforms.origin, 1, &origin.x ); + glm::vec2 const world { static_cast( m_origin.x ), static_cast( m_origin.z ) }; + ::glUniform2fv( m_uniforms.world, 1, &world.x ); + ::glUniform1i( m_uniforms.ground, 0 ); ::glUniform1i( m_uniforms.materials, 1 ); - ::glUniform1i( m_uniforms.ground, 2 ); - ::glUniform1i( m_uniforms.materialtable, 3 ); - ::glUniform1i( m_uniforms.instances, 4 ); - ::glActiveTexture( GL_TEXTURE2 ); + ::glActiveTexture( GL_TEXTURE0 ); ::glBindTexture( GL_TEXTURE_2D_ARRAY, m_ground.textures() ); - ::glActiveTexture( GL_TEXTURE3 ); + ::glActiveTexture( GL_TEXTURE1 ); ::glBindTexture( GL_TEXTURE_2D, m_ground.table() ); - std::size_t drawn { 0 }; - auto const gridstep { m_reader.gridstep() }; - for( std::uint32_t level = 0; level < m_instances.size(); ++level ) { + // the ground loses the depth test to whatever lies on it, and a coarser terrain to a finer one + // describing the same ground. the offset is negative because depth runs the other way here + ::glEnable( GL_POLYGON_OFFSET_FILL ); + ::glPolygonOffset( + -1.f - 2.f * static_cast( m_depthrank ), + -2.f - 4.f * static_cast( m_depthrank ) ); - auto const &instances { m_instances[ level ] }; - if( true == instances.empty() ) { continue; } - auto const count { instances.size() / 8 }; - drawn += count; - - m_instancebuffer->allocate( - gl::buffer::TEXTURE_BUFFER, static_cast( instances.size() * sizeof( float ) ), GL_STREAM_DRAW ); - m_instancebuffer->upload( - gl::buffer::TEXTURE_BUFFER, instances.data(), 0, static_cast( instances.size() * sizeof( float ) ) ); - ::glActiveTexture( GL_TEXTURE4 ); - ::glBindTexture( GL_TEXTURE_BUFFER, m_instancetexture ); - ::glTexBuffer( GL_TEXTURE_BUFFER, GL_RGBA32F, static_cast( *m_instancebuffer ) ); - - auto const &pool { *m_pools[ level ] }; - ::glActiveTexture( GL_TEXTURE0 ); - ::glBindTexture( GL_TEXTURE_2D_ARRAY, pool.heights() ); - ::glActiveTexture( GL_TEXTURE1 ); - ::glBindTexture( GL_TEXTURE_2D_ARRAY, pool.materials() ); - - ::glUniform1f( m_uniforms.samplestep, gridstep * static_cast( 1u << level ) ); - ::glUniform1i( m_uniforms.side, static_cast( pool.side() ) ); - // the band over which this level turns into the next. the coarsest level has no next - // one, so its band lies beyond anything that is ever drawn - if( level + 1 < m_reader.levels() ) { - ::glUniform2f( - m_uniforms.morph, - static_cast( terrain_morph_begin( level, m_finest, tilesize ) ), - static_cast( terrain_morph_end( level, m_finest, tilesize ) ) ); - } - else { - ::glUniform2f( m_uniforms.morph, 1e30f, 2e30f ); - } - - m_indices[ level ]->bind( gl::buffer::ELEMENT_ARRAY_BUFFER ); - ::glDrawElementsInstanced( - GL_TRIANGLES, static_cast( m_indexcounts[ level ] ), GL_UNSIGNED_INT, nullptr, - static_cast( count ) ); - } - m_stats.drawn = drawn; + ::glBindVertexArray( m_arena.vertexarray() ); + ::glMultiDrawElementsBaseVertex( + GL_TRIANGLES, m_counts.data(), GL_UNSIGNED_INT, + const_cast( m_offsets.data() ), + static_cast( m_counts.size() ), m_bases.data() ); + ::glBindVertexArray( 0 ); ::glDisable( GL_POLYGON_OFFSET_FILL ); - gl::buffer::unbind( gl::buffer::TEXTURE_BUFFER ); + ::glActiveTexture( GL_TEXTURE1 ); + ::glBindTexture( GL_TEXTURE_2D, 0 ); ::glActiveTexture( GL_TEXTURE0 ); + ::glBindTexture( GL_TEXTURE_2D_ARRAY, 0 ); opengl_texture::reset_unit_cache(); - m_vao->unbind(); gl::program::bind( 0 ); } diff --git a/rendering/terrainclipmap.cppm b/rendering/terrainclipmap.cppm index 9e700931..f34842fb 100644 --- a/rendering/terrainclipmap.cppm +++ b/rendering/terrainclipmap.cppm @@ -11,37 +11,35 @@ module; #include #include #include -#include #include #include #include +#include #include #include "scene/quantizedmeshreader.h" export module eu07.rendering.terrainclipmap; import eu07.gl.shader; -import eu07.gl.buffer; -import eu07.gl.vao; import eu07.glm; import eu07.rendering.terraintileloader; import eu07.rendering.terrainstatus; -import eu07.rendering.terraintilepool_tin; +import eu07.rendering.terrainmesharena; import eu07.rendering.terrainground; export { -// Draws TIN (Triangulated Irregular Network) terrain from Quantized Mesh tiles. +// Draws a scenery's cooked terrain: a quadtree of mesh tiles, each a piece of the ground as the +// scenery drew it, thinned out the further from the camera it is drawn. // -// Replaces the old heightfield texture array approach with vertex/index buffers per tile. -// Each tile is an independent triangulated mesh with multiple LOD levels. The terrain adapts -// density to detail: fine triangulation along track corridors, coarse in open fields. +// Each frame a walk of the quadtree settles on the tiles to draw: a tile is split when the ground +// it stands for would be off by more pixels than the detail setting allows, and only when all four +// of its children are on the card - otherwise it is drawn itself and the children are asked for, so +// that streaming shows coarse ground rather than a hole. Because every vertex on a tile's border +// survives into all of its coarser levels, two neighbours drawn at different levels share their +// border exactly, and nothing has to be stitched or morphed. // -// CDLOD (Continuous Distance LOD): tiles select their LOD based on distance from camera, -// with edge stitching to prevent cracks between LOD transitions. -// -// The class owns GL resources and file handles. Given a viewpoint and visibility test, -// it loads, uploads, and renders appropriate tiles each frame. +// Every resident tile lives in the same pair of buffers, so the whole terrain is drawn in one call. class terrain_clipmap { public: @@ -53,116 +51,121 @@ public: terrain_clipmap( terrain_clipmap const & ) = delete; terrain_clipmap &operator=( terrain_clipmap const & ) = delete; - // opens a cooked TIN directory and builds the gl resources. returns false when the - // directory cannot be read, leaving the object unusable but safe to destroy + // opens a cooked terrain - an archive or a directory of tiles - and builds the gl resources. + // returns false when it cannot be read, leaving the object unusable but safe to destroy bool open( std::string const &Path ); bool ready() const { return m_ready; } void close(); - // asks for the tiles around the viewpoint, uploads a bounded number of those that have - // arrived, and retires the ones that left range. safe to call every frame: the reading - // and decoding happen on the loader's thread + // walks the quadtree, asks for what is missing, puts in what has arrived and retires what has + // not been wanted for a while. safe to call every frame: the reading happens on the loader's + // thread void update( glm::dvec3 const &Viewpoint ); - // draws the resident tiles in range that the test says can be seen. the caller is - // expected to have the scene and model uniform blocks already set for a camera-relative - // pass. tile positions are passed as uniforms per draw + // draws the tiles the walk settled on that the test says can be seen void render( glm::dvec3 const &Viewpoint, visibility const &Visible ); // how far from the viewpoint terrain is drawn, in metres void range( float const Range ) { m_range = Range; } float range() const { return m_range; } - // what the screen can resolve: the viewport's pixels per unit of size over distance, and - // how many pixels a sample spacing may cover. decides how far each level reaches + // what the screen can resolve: the viewport's pixels per unit of size over distance, and how + // many pixels a tile's error may cover before a finer level is used void detail( double const Pixelsperunit, double const Pixels ); - // the finest level's range the tiles in range were last worked out with - float finest() const { return static_cast( m_finest ); } - - // metres between samples at the finest level; a scenery's fields are drawn from the - // finest to the coarsest so that the depth test keeps the better surface - float gridstep() const { return m_reader.gridstep(); } - // pushes this field back in depth by the given rank, so that geometry lying on it wins the - // depth test, and a coarser field loses to a finer one describing the same ground. the - // offset is in depth, scaled by slope, so it holds at any distance and on any incline + // side of a tile at the finest level, in metres. terrains are drawn finest first, so that the + // depth test keeps the better surface where two of them describe the same ground + float gridstep() const { return m_finesttile; } + // pushes this terrain back in depth by the given rank, so that geometry lying on it wins the + // depth test, and a coarser terrain loses to a finer one describing the same ground void depthrank( unsigned const Rank ) { m_depthrank = Rank; } terrain_statistics const &stats() const { return m_stats; } std::string const &path() const { return m_path; } - // metres per tile side - float tilesize() const { return m_tilesize; } - // number of LOD levels available - std::uint32_t levels() const { return m_lodlevels; } + float tilesize() const { return m_finesttile; } + std::uint32_t levels() const { return m_levels; } std::size_t backlog() const { return m_loader.backlog(); } + float detailpixels() const { return static_cast( m_detail ); } + // side of a tile at each level, and how far that level's ground is off, for the debug panel + float leveltile( std::uint32_t const Level ) const; + float levelerror( std::uint32_t const Level ) const; private: - // one tile resident on the gpu: VBO/IBO handle + // one tile held in the buffers struct resident_tile { - std::int32_t x { 0 }, z { 0 }; + terrain_mesh_arena::piece piece; + // for culling, in world space + glm::dvec3 centre { 0.0 }; + float radius { 0.f }; std::uint32_t level { 0 }; - terrain_tile_pool_tin::tile_handle gpu_data; - float minheight { 0.f }, maxheight { 0.f }; std::uint64_t lastused { 0 }; }; - // one tile within range, and the level its distance earns - struct in_range { - std::int64_t key; - std::uint32_t level; - }; - - // mip level a tile at the given distance is drawn at - std::uint32_t level_for( double const Distance ) const; - // works out which tiles are in range and at what level, and tells the loader what is - // missing. the answer only changes once the camera has moved a fair part of a tile - void scan( glm::dvec3 const &Viewpoint ); - // puts a tile the loader has read on the gpu, replacing whatever level it had before - void upload( terrain_tile_loader::payload const &Tile ); - void retire( resident_tile const &Tile ); + // the tiles the walk settled on, in the order they will be drawn + void walk( glm::dvec3 const &Viewpoint ); + // asks the loader for every tile the wanted cut of the quadtree is missing, and says whether this + // tile and everything under it is already there + bool need( quantizedmesh::tile_address const &Tile, glm::dvec3 const &Viewpoint ); + // whether the level this tile stands for is close enough to the ground below it + bool good_enough( quantizedmesh::tile_address const &Tile, double Distance ) const; + void choose( quantizedmesh::tile_address const &Tile, glm::dvec3 const &Viewpoint ); + // where a tile's square is, and how far the viewpoint is from it + void square( quantizedmesh::tile_address const &Tile, double &West, double &North, double &Side ) const; + double distance_to( quantizedmesh::tile_address const &Tile, glm::dvec3 const &Viewpoint ) const; + bool present( quantizedmesh::tile_address const &Tile ) const; + void ask_for( quantizedmesh::tile_address const &Tile, double Distance ); + void put_in( terrain_tile_loader::payload const &Tile ); + void retire_unused(); void resolve_uniforms(); void report_residency(); - static std::int64_t key( std::int32_t const X, std::int32_t const Z ) { - return ( static_cast( X ) << 32 ) ^ static_cast( Z ); - } - - // Tile management - quantizedmesh::reader m_reader; + quantizedmesh::reader m_reader; // the table and which tiles there are; reading is the loader's terrain_tile_loader m_loader; - terrain_tile_pool_tin m_pool; - - std::vector m_wanted; - std::vector m_arrived; - // what the last scan found in range, and the level wanted for each tile - std::vector m_inrange; - std::unordered_map m_wantedlevel; - glm::dvec3 m_scanpoint { 0.0 }; - float m_scanrange { -1.f }; - double m_finestwanted { 0.0 }; - double m_finest { 0.0 }; - - // Resident tiles (key = tile_key(x,z)) - std::unordered_map m_tiles; - - // Ground texture array (materials) + terrain_mesh_arena m_arena; terrain_ground m_ground; - - // Shader and uniforms + + quantizedmesh::terrain_table m_table; + glm::dvec3 m_origin { 0.0 }; // what the vertices in the buffers are measured from + float m_finesttile { 256.f }; + std::uint32_t m_levels { 1 }; + + std::unordered_map m_tiles; + std::vector m_chosen; + std::vector> m_wishes; + std::vector m_wanted; + std::vector m_arrived; + std::unordered_set m_asked; // what the last walk asked the loader for + // whether a subtree could be drawn, worked out once per walk + std::unordered_map m_readiness; + + // what the draw hands to gl, kept between frames so that a frame allocates nothing + std::vector m_counts; + std::vector m_offsets; + std::vector m_bases; + + glm::dvec3 m_walkpoint { 0.0 }; + bool m_walked { false }; + double m_pixelsperunit { 0.0 }; + double m_detail { 3.0 }; + double m_walkdetail { 0.0 }; + float m_range { 6000.f }; + float m_walkrange { -1.f }; + std::optional m_shader; struct uniform_locations { - int tile_offset { -1 }; // vec3: tile origin relative to camera - int ground { -1 }; // sampler2DArray: material textures + int origin { -1 }; // where the terrain's origin is, relative to the camera + int world { -1 }; // and where it is in the world + int ground { -1 }; + int materials { -1 }; } m_uniforms; - // Configuration - float m_tilesize { 256.0f }; - std::uint32_t m_lodlevels { 4 }; terrain_statistics m_stats; std::uint64_t m_frame { 0 }; std::size_t m_reportedresidency { 0 }; + std::size_t m_reporteddrawn { 0 }; + std::uint64_t m_reportedframe { 0 }; bool m_reported { false }; - float m_range { 6000.f }; unsigned m_depthrank { 0 }; - std::size_t m_budget { 2048 }; // tiles allowed resident - std::size_t m_uploadsperframe { 32 }; + // what the buffers may hold before tiles nobody asked for lately are let go + std::size_t m_budget { 192u << 20 }; + std::size_t m_putsperframe { 16 }; bool m_ready { false }; std::string m_path; }; diff --git a/rendering/terrainmesharena.cpp b/rendering/terrainmesharena.cpp new file mode 100644 index 00000000..c6b472ed --- /dev/null +++ b/rendering/terrainmesharena.cpp @@ -0,0 +1,255 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +module; +#include +#include +#include +#include +#include +#include +#include "scene/quantizedmeshreader.h" + +module eu07.rendering.terrainmesharena; +import eu07.utilities.logs; + +namespace { + +// what the buffers start at, in vertices and indices, and the most they may reach. a vertex is 24 +// bytes and an index 4, so the ceiling is about 400 MB of terrain on the card +constexpr std::uint32_t firstvertices { 1u << 19 }; +constexpr std::uint32_t firstindices { 1u << 21 }; +constexpr std::uint32_t mostvertices { 1u << 24 }; +constexpr std::uint32_t mostindices { 1u << 26 }; + +} // anonymous namespace + +void +terrain_mesh_arena::span_list::give( std::uint32_t const At, std::uint32_t const Count ) { + + if( Count == 0 ) { return; } + auto placed { m_free.emplace( At, Count ).first }; + // join what now touches, so that the room a tile gave back can hold a larger one + auto after { std::next( placed ) }; + if( ( after != m_free.end() ) && ( placed->first + placed->second == after->first ) ) { + placed->second += after->second; + m_free.erase( after ); + } + if( placed != m_free.begin() ) { + auto before { std::prev( placed ) }; + if( before->first + before->second == placed->first ) { + before->second += placed->second; + m_free.erase( placed ); + } + } +} + +bool +terrain_mesh_arena::span_list::take( std::uint32_t const Count, std::uint32_t &At ) { + + for( auto span = m_free.begin(); span != m_free.end(); ++span ) { + if( span->second < Count ) { continue; } + At = span->first; + if( span->second == Count ) { m_free.erase( span ); } + else { + auto const left { span->second - Count }; + m_free.erase( span ); + m_free.emplace( At + Count, left ); + } + return true; + } + return false; +} + +terrain_mesh_arena::~terrain_mesh_arena() { + + destroy(); +} + +bool +terrain_mesh_arena::create() { + + destroy(); + ::glGenVertexArrays( 1, &m_vao ); + ::glGenBuffers( 1, &m_vertexbuffer ); + ::glGenBuffers( 1, &m_indexbuffer ); + if( ( m_vao == 0 ) || ( m_vertexbuffer == 0 ) || ( m_indexbuffer == 0 ) ) { destroy(); return false; } + + ::glBindVertexArray( m_vao ); + ::glBindBuffer( GL_ARRAY_BUFFER, m_vertexbuffer ); + ::glBufferData( GL_ARRAY_BUFFER, + static_cast( firstvertices ) * sizeof( quantizedmesh::render_vertex ), nullptr, GL_DYNAMIC_DRAW ); + ::glBindBuffer( GL_ELEMENT_ARRAY_BUFFER, m_indexbuffer ); + ::glBufferData( GL_ELEMENT_ARRAY_BUFFER, + static_cast( firstindices ) * sizeof( std::uint32_t ), nullptr, GL_DYNAMIC_DRAW ); + + auto const stride { static_cast( sizeof( quantizedmesh::render_vertex ) ) }; + ::glEnableVertexAttribArray( 0 ); + ::glVertexAttribPointer( 0, 3, GL_FLOAT, GL_FALSE, stride, + reinterpret_cast( offsetof( quantizedmesh::render_vertex, x ) ) ); + ::glEnableVertexAttribArray( 1 ); + ::glVertexAttribIPointer( 1, 1, GL_UNSIGNED_SHORT, stride, + reinterpret_cast( offsetof( quantizedmesh::render_vertex, material ) ) ); + ::glBindVertexArray( 0 ); + ::glBindBuffer( GL_ARRAY_BUFFER, 0 ); + ::glBindBuffer( GL_ELEMENT_ARRAY_BUFFER, 0 ); + + m_vertexroom = firstvertices; + m_indexroom = firstindices; + m_freevertices.clear(); + m_freeindices.clear(); + m_freevertices.give( 0, m_vertexroom ); + m_freeindices.give( 0, m_indexroom ); + m_verticesheld = 0; + return true; +} + +void +terrain_mesh_arena::destroy() { + + if( m_vao != 0 ) { ::glDeleteVertexArrays( 1, &m_vao ); m_vao = 0; } + if( m_vertexbuffer != 0 ) { ::glDeleteBuffers( 1, &m_vertexbuffer ); m_vertexbuffer = 0; } + if( m_indexbuffer != 0 ) { ::glDeleteBuffers( 1, &m_indexbuffer ); m_indexbuffer = 0; } + m_vertexroom = m_indexroom = 0; + m_freevertices.clear(); + m_freeindices.clear(); + m_verticesheld = 0; +} + +// Growing copies what is there into a larger buffer on the card: the pieces keep their places, so +// nothing has to be read back or re-uploaded. +bool +terrain_mesh_arena::grow_vertices( std::uint32_t const Wanted ) { + + auto room { m_vertexroom }; + while( ( room < m_vertexroom + Wanted ) && ( room < mostvertices ) ) { room *= 2; } + if( room <= m_vertexroom ) { return false; } + + std::uint32_t grown { 0 }; + ::glGenBuffers( 1, &grown ); + if( grown == 0 ) { return false; } + ::glBindBuffer( GL_ARRAY_BUFFER, grown ); + ::glBufferData( GL_ARRAY_BUFFER, + static_cast( room ) * sizeof( quantizedmesh::render_vertex ), nullptr, GL_DYNAMIC_DRAW ); + ::glBindBuffer( GL_COPY_READ_BUFFER, m_vertexbuffer ); + ::glBindBuffer( GL_COPY_WRITE_BUFFER, grown ); + ::glCopyBufferSubData( GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0, + static_cast( m_vertexroom ) * sizeof( quantizedmesh::render_vertex ) ); + ::glBindBuffer( GL_COPY_READ_BUFFER, 0 ); + ::glBindBuffer( GL_COPY_WRITE_BUFFER, 0 ); + ::glDeleteBuffers( 1, &m_vertexbuffer ); + m_vertexbuffer = grown; + + ::glBindVertexArray( m_vao ); + ::glBindBuffer( GL_ARRAY_BUFFER, m_vertexbuffer ); + auto const stride { static_cast( sizeof( quantizedmesh::render_vertex ) ) }; + ::glVertexAttribPointer( 0, 3, GL_FLOAT, GL_FALSE, stride, + reinterpret_cast( offsetof( quantizedmesh::render_vertex, x ) ) ); + ::glVertexAttribIPointer( 1, 1, GL_UNSIGNED_SHORT, stride, + reinterpret_cast( offsetof( quantizedmesh::render_vertex, material ) ) ); + ::glBindVertexArray( 0 ); + ::glBindBuffer( GL_ARRAY_BUFFER, 0 ); + + m_freevertices.give( m_vertexroom, room - m_vertexroom ); + m_vertexroom = room; + return true; +} + +bool +terrain_mesh_arena::grow_indices( std::uint32_t const Wanted ) { + + auto room { m_indexroom }; + while( ( room < m_indexroom + Wanted ) && ( room < mostindices ) ) { room *= 2; } + if( room <= m_indexroom ) { return false; } + + std::uint32_t grown { 0 }; + ::glGenBuffers( 1, &grown ); + if( grown == 0 ) { return false; } + ::glBindBuffer( GL_ARRAY_BUFFER, grown ); + ::glBufferData( GL_ARRAY_BUFFER, + static_cast( room ) * sizeof( std::uint32_t ), nullptr, GL_DYNAMIC_DRAW ); + ::glBindBuffer( GL_COPY_READ_BUFFER, m_indexbuffer ); + ::glBindBuffer( GL_COPY_WRITE_BUFFER, grown ); + ::glCopyBufferSubData( GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0, + static_cast( m_indexroom ) * sizeof( std::uint32_t ) ); + ::glBindBuffer( GL_COPY_READ_BUFFER, 0 ); + ::glBindBuffer( GL_COPY_WRITE_BUFFER, 0 ); + ::glDeleteBuffers( 1, &m_indexbuffer ); + m_indexbuffer = grown; + + ::glBindVertexArray( m_vao ); + ::glBindBuffer( GL_ELEMENT_ARRAY_BUFFER, m_indexbuffer ); + ::glBindVertexArray( 0 ); + ::glBindBuffer( GL_ELEMENT_ARRAY_BUFFER, 0 ); + + m_freeindices.give( m_indexroom, room - m_indexroom ); + m_indexroom = room; + return true; +} + +terrain_mesh_arena::piece +terrain_mesh_arena::put( quantizedmesh::reader::tile_data const &Tile ) { + + piece where; + if( ( m_vao == 0 ) || ( true == Tile.vertices.empty() ) || ( true == Tile.indices.empty() ) ) { + return where; + } + auto const vertices { static_cast( Tile.vertices.size() ) }; + auto const indices { static_cast( Tile.indices.size() ) }; + + std::uint32_t atvertex { 0 }; + if( false == m_freevertices.take( vertices, atvertex ) ) { + if( false == grow_vertices( vertices ) ) { return where; } + if( false == m_freevertices.take( vertices, atvertex ) ) { return where; } + } + std::uint32_t atindex { 0 }; + if( false == m_freeindices.take( indices, atindex ) ) { + if( false == grow_indices( indices ) ) { + m_freevertices.give( atvertex, vertices ); + return where; + } + if( false == m_freeindices.take( indices, atindex ) ) { + m_freevertices.give( atvertex, vertices ); + return where; + } + } + + ::glBindBuffer( GL_ARRAY_BUFFER, m_vertexbuffer ); + ::glBufferSubData( GL_ARRAY_BUFFER, + static_cast( atvertex ) * sizeof( quantizedmesh::render_vertex ), + static_cast( vertices ) * sizeof( quantizedmesh::render_vertex ), + Tile.vertices.data() ); + ::glBindBuffer( GL_ARRAY_BUFFER, 0 ); + + // the indices are stored as the tile had them, counted from its own first vertex; where the + // tile sits in the buffer is given to the draw as its base vertex + ::glBindBuffer( GL_ELEMENT_ARRAY_BUFFER, m_indexbuffer ); + ::glBufferSubData( GL_ELEMENT_ARRAY_BUFFER, + static_cast( atindex ) * sizeof( std::uint32_t ), + static_cast( indices ) * sizeof( std::uint32_t ), + Tile.indices.data() ); + ::glBindBuffer( GL_ELEMENT_ARRAY_BUFFER, 0 ); + + where.firstvertex = atvertex; + where.vertices = vertices; + where.firstindex = atindex; + where.indices = indices; + m_verticesheld += vertices; + return where; +} + +void +terrain_mesh_arena::take( piece &Piece ) { + + if( false == Piece.held() ) { return; } + m_freevertices.give( Piece.firstvertex, Piece.vertices ); + m_freeindices.give( Piece.firstindex, Piece.indices ); + m_verticesheld -= Piece.vertices; + Piece = piece {}; +} diff --git a/rendering/terrainmesharena.cppm b/rendering/terrainmesharena.cppm new file mode 100644 index 00000000..f5050483 --- /dev/null +++ b/rendering/terrainmesharena.cppm @@ -0,0 +1,87 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +module; +#include +#include +#include +#include +#include "scene/quantizedmeshreader.h" + +export module eu07.rendering.terrainmesharena; + +export { + +// One vertex buffer and one index buffer for the whole terrain. +// +// Tiles are put in and taken out as the camera moves, so each buffer is handed out in pieces and +// the pieces are given back. What this buys is the drawing: every resident tile lives in the same +// two buffers behind the same vertex array, so a frame is one glMultiDrawElements rather than a +// bind and a draw per tile. Vertices are measured from one point for the whole terrain, which is +// what makes that possible. +// +// The buffers grow when a tile will not fit and nothing can be freed; they never shrink. +class terrain_mesh_arena { + +public: + // where a tile sits in the two buffers + struct piece { + std::uint32_t firstvertex { 0 }; + std::uint32_t vertices { 0 }; + std::uint32_t firstindex { 0 }; + std::uint32_t indices { 0 }; + bool held() const { return indices != 0; } + }; + + terrain_mesh_arena() = default; + ~terrain_mesh_arena(); + terrain_mesh_arena( terrain_mesh_arena const & ) = delete; + terrain_mesh_arena &operator=( terrain_mesh_arena const & ) = delete; + + bool create(); + void destroy(); + + // puts a tile's mesh in. returns a piece that is not held when it did not fit + piece put( quantizedmesh::reader::tile_data const &Tile ); + void take( piece &Piece ); + + // bound for drawing; the index buffer goes with it + std::uint32_t vertexarray() const { return m_vao; } + + std::size_t vertexbytes() const { return m_vertexroom * sizeof( quantizedmesh::render_vertex ); } + std::size_t indexbytes() const { return m_indexroom * sizeof( std::uint32_t ); } + std::size_t bytes() const { return vertexbytes() + indexbytes(); } + std::size_t verticesheld() const { return m_verticesheld; } + +private: + // a run of free slots in one of the buffers, kept in order so that neighbours can be joined + class span_list { + public: + void clear() { m_free.clear(); } + void give( std::uint32_t At, std::uint32_t Count ); + // the first run large enough, or the count when there is none + bool take( std::uint32_t Count, std::uint32_t &At ); + private: + std::map m_free; // where to how many + }; + + bool grow_vertices( std::uint32_t Wanted ); + bool grow_indices( std::uint32_t Wanted ); + + std::uint32_t m_vao { 0 }; + std::uint32_t m_vertexbuffer { 0 }; + std::uint32_t m_indexbuffer { 0 }; + std::uint32_t m_vertexroom { 0 }; + std::uint32_t m_indexroom { 0 }; + span_list m_freevertices; + span_list m_freeindices; + std::size_t m_verticesheld { 0 }; +}; + +} // export diff --git a/rendering/terrainstatus.cppm b/rendering/terrainstatus.cppm index 9b7db39f..cbe80898 100644 --- a/rendering/terrainstatus.cppm +++ b/rendering/terrainstatus.cppm @@ -20,76 +20,50 @@ export module eu07.rendering.terrainstatus; export { -// Which level a tile is drawn at follows from what its samples look like on screen: a level is -// used out to the distance where one of its sample spacings covers the detail setting in -// pixels. Each level doubles the spacing, so each doubles that distance too. Pixelsperunit is -// the viewport height over twice the tangent of half the vertical field of view - what a -// metre at a metre's distance spans - so zooming in or a taller window pushes detail further. +// Which level of the quadtree a tile is drawn at follows from what the coarser mesh would cost in +// pixels. Every tile says how far its ground departs from the finest mesh, in metres; that error, +// at the tile's distance, covers a number of pixels, and while it covers fewer than the detail +// setting asks for there is nothing to gain by going finer. Pixelsperunit is the viewport height +// over twice the tangent of half the vertical field of view - what a metre at a metre's distance +// spans - so zooming in or a taller window pushes detail further out on its own. // -// The finest range never drops below four tile sides, whatever the screen: the morphing below -// only closes the seams when a level's range is several tiles long. -inline constexpr double terrain_finest_tiles { 4.0 }; +// This is the format's own rule, and the reason a tile carries its error at all. +inline constexpr std::size_t terrain_maxlevels { 16 }; -inline double -terrain_finest_range( double const Gridstep, double const Tilesize, double const Pixelsperunit, double const Detail ) { - return std::max( terrain_finest_tiles * Tilesize, Gridstep * Pixelsperunit / std::max( 0.1, Detail ) ); +inline bool +terrain_level_enough( + double const Error, double const Distance, double const Pixelsperunit, double const Detail ) { + return Error * Pixelsperunit <= std::max( 0.1, Detail ) * std::max( 1.0, Distance ); } -// how far the camera may move before the tiles in range and their levels are worked out -// again, in tile sides +// how far the camera may move before which tiles are wanted is worked out again, as a share of the +// finest tile's side inline constexpr double terrain_rescan_tiles { 0.25 }; -// the distance up to which a level is used, given the finest level's range -inline double -terrain_level_distance( std::uint32_t const Level, double const Finest ) { - return Finest * std::pow( 2.0, Level ); -} - -// A tile morphs into the next level over a band at the far end of its level's range. Two -// neighbours at levels L and L+1 then meet exactly when, all along their shared edge, the -// finer tile has finished morphing and the coarser one has not started. The band ends a -// rescan's worth of travel short of the range, because levels are chosen where the camera -// was at the last rescan while the morph follows the camera every frame; and it is a fifth -// of the range wide, which keeps the coarser tile's band, a whole range further out, clear of -// any edge a finer tile can reach. Both were settled by a geometric test over random camera -// positions, three tile sizes and several finest ranges (tools/terraincook/seamcheck.py), -// which finds no gap with these values and finds them at every level boundary with a band -// ending at the range itself. -inline constexpr double terrain_morph_band { 0.2 }; - -// the distances, from the camera, over which a level L tile morphs into level L+1 -inline double -terrain_morph_end( std::uint32_t const Level, double const Finest, double const Tilesize ) { - return terrain_level_distance( Level, Finest ) - terrain_rescan_tiles * Tilesize; -} - -inline double -terrain_morph_begin( std::uint32_t const Level, double const Finest, double const Tilesize ) { - return terrain_morph_end( Level, Finest, Tilesize ) - terrain_morph_band * terrain_level_distance( Level, Finest ); -} - -// what one terrain field's streamer is doing, kept by the field and read by the debug panel +// what one terrain's streamer is doing, kept by it and read by the debug panel struct terrain_statistics { - std::size_t resident { 0 }; // tiles holding gl textures - std::size_t inview { 0 }; // resident tiles within range in the last update + std::size_t resident { 0 }; // tiles held in the vertex and index buffers + std::size_t chosen { 0 }; // tiles the last walk of the quadtree settled on drawing std::size_t drawn { 0 }; // of those, the ones inside the frustum in the last draw - std::size_t wanted { 0 }; // tiles in range missing, or at the wrong level - std::size_t uploaded { 0 }; // uploads since opening - std::size_t dropped { 0 }; // arrivals discarded as stale since opening - std::size_t texturebytes { 0 }; - std::array perlevel {}; // tiles in view at each level + std::size_t triangles { 0 }; // triangles in the last draw + std::size_t wanted { 0 }; // tiles the walk asked for and did not have + std::size_t uploaded { 0 }; // tiles put in the buffers since opening + std::size_t dropped { 0 }; // arrivals discarded as no longer wanted + std::size_t gpubytes { 0 }; // what the two buffers hold + std::array perlevel {}; // tiles drawn at each level }; -// one field as the debug panel shows it. the renderer writes it once a frame from the main -// view, the panel reads it; neither needs to know the other's types +// one terrain as the debug panel shows it. the renderer writes it once a frame from the main view, +// the panel reads it; neither needs to know the other's types struct terrain_field_status { std::string path; - float gridstep { 0.f }; - float tilesize { 0.f }; + float tilesize { 0.f }; // side of a tile at the finest level, in metres float range { 0.f }; - float finest { 0.f }; // range of the finest level, from the screen + float detail { 0.f }; // pixels an error may cover before a finer level is used std::uint32_t levels { 0 }; std::size_t backlog { 0 }; // tiles the loader has queued or finished and not handed over + std::array leveltile {}; // tile side at each level + std::array levelerror {}; // metres a level departs from the finest terrain_statistics stats; }; diff --git a/rendering/terraintileloader.cpp b/rendering/terraintileloader.cpp index 09e6fcae..85528127 100644 --- a/rendering/terraintileloader.cpp +++ b/rendering/terraintileloader.cpp @@ -25,10 +25,14 @@ terrain_tile_loader::~terrain_tile_loader() { } bool -terrain_tile_loader::open( std::string const &Path ) { +terrain_tile_loader::open( + std::string const &Path, double const Originx, double const Originy, double const Originz ) { close(); if( false == m_reader.open( Path ) ) { return false; } + m_originx = Originx; + m_originy = Originy; + m_originz = Originz; m_stop = false; m_thread = std::thread( &terrain_tile_loader::work, this ); return true; @@ -53,24 +57,22 @@ terrain_tile_loader::close() { } bool -terrain_tile_loader::underway( request const &Tile ) const { +terrain_tile_loader::underway( quantizedmesh::tile_address const &Tile ) const { if( ( m_running.has_value() ) && ( *m_running == Tile ) ) { return true; } return std::any_of( m_done.begin(), m_done.end(), - [ &Tile ]( payload const &Done ) { return Done.tile == Tile; } ); + [ &Tile ]( payload const &Done ) { return Done.data.tile == Tile; } ); } void -terrain_tile_loader::want( std::vector const &Requests ) { +terrain_tile_loader::want( std::vector const &Requests ) { { std::lock_guard lock( m_mutex ); m_pending.clear(); for( auto const &tile : Requests ) { - if( false == underway( tile ) ) { - m_pending.push_back( tile ); - } + if( false == underway( tile ) ) { m_pending.push_back( tile ); } } } m_wake.notify_one(); @@ -105,7 +107,7 @@ void terrain_tile_loader::work() { while( true ) { - request tile; + quantizedmesh::tile_address tile; { std::unique_lock lock( m_mutex ); m_wake.wait( lock, [ this ]() { @@ -116,40 +118,11 @@ terrain_tile_loader::work() { m_running = tile; } - // the read itself happens outside the lock, so the render thread can replace the - // wish list while a tile is being decoded + // the read itself happens outside the lock, so the render thread can replace the wish + // list while a tile is being decoded payload result; - result.tile = tile; - - quantizedmesh::reader::tile_data data; - result.valid = m_reader.read_tile( tile.x, tile.z, tile.level, data ); - - if( result.valid ) { - // Copy data to payload - result.positions_x = std::move( data.positions_x ); - result.positions_y = std::move( data.positions_y ); - result.positions_z = std::move( data.positions_z ); - result.uvs_u = std::move( data.uvs_u ); - result.uvs_v = std::move( data.uvs_v ); - result.materials = std::move( data.materials ); - result.indices = std::move( data.indices ); - - result.north_edge = std::move( data.north_edge ); - result.south_edge = std::move( data.south_edge ); - result.west_edge = std::move( data.west_edge ); - result.east_edge = std::move( data.east_edge ); - - result.center_x = data.center_x; - result.center_y = data.center_y; - result.center_z = data.center_z; - result.min_x = data.min_x; - result.min_y = data.min_y; - result.min_z = data.min_z; - result.max_x = data.max_x; - result.max_y = data.max_y; - result.max_z = data.max_z; - result.geometric_error = data.geometric_error; - } + result.data.tile = tile; + result.valid = m_reader.read_tile( tile, result.data, m_originx, m_originy, m_originz ); std::lock_guard lock( m_mutex ); m_running.reset(); diff --git a/rendering/terraintileloader.cppm b/rendering/terraintileloader.cppm index 1a225ca3..227387bc 100644 --- a/rendering/terraintileloader.cppm +++ b/rendering/terraintileloader.cppm @@ -23,49 +23,18 @@ export module eu07.rendering.terraintileloader; export { -// Reads and decodes TIN (Quantized Mesh) terrain tiles on a thread of its own. +// Reads and decodes terrain mesh tiles on a thread of its own. // -// Reads .qm tile files, dequantizes vertices, and hands back ready-to-upload vertex/index -// data. Does the I/O and decompression off the render thread. The renderer uploads at its own -// pace, and the loader prioritizes nearest tiles first. -// -// Replaces heightfield tile loading entirely - TIN is the only terrain format now. +// Reading a tile means finding it in the archive, unpacking it and turning the quantized vertices +// back into something a vertex buffer will take. None of that belongs on the render thread. The +// renderer says which tiles it wants, nearest first, and collects whatever has arrived at its own +// pace. class terrain_tile_loader { public: - struct request { - std::int32_t x { 0 }, z { 0 }; - std::uint32_t level { 0 }; - - bool operator==( request const & ) const = default; - }; - struct payload { - request tile; + quantizedmesh::reader::tile_data data; bool valid { false }; - - // Vertex data (dequantized, ready for GPU) - std::vector positions_x; - std::vector positions_y; - std::vector positions_z; - std::vector uvs_u; - std::vector uvs_v; - std::vector materials; - - // Index data - std::vector indices; - - // Edge indices for seamless LOD stitching - std::vector north_edge; - std::vector south_edge; - std::vector west_edge; - std::vector east_edge; - - // Tile bounds (for placement) - double center_x, center_y, center_z; - double min_x, min_y, min_z; - double max_x, max_y, max_z; - double geometric_error; }; terrain_tile_loader() = default; @@ -73,14 +42,15 @@ public: terrain_tile_loader( terrain_tile_loader const & ) = delete; terrain_tile_loader &operator=( terrain_tile_loader const & ) = delete; - // opens the file on the loader's own handle and starts the thread - bool open( std::string const &Path ); + // opens the source on the loader's own handle and starts the thread. Origin is what the + // vertices come back measured from + bool open( std::string const &Path, double Originx, double Originy, double Originz ); // stops the thread; anything not collected is dropped void close(); - // replaces the list of wanted tiles, nearest first. tiles being read or already read and - // not yet collected are left out of it, so asking again every frame costs nothing - void want( std::vector const &Requests ); + // replaces the list of wanted tiles, nearest first. tiles being read or already read and not + // yet collected are left out of it, so asking again every frame costs nothing + void want( std::vector const &Requests ); // moves up to Count finished tiles into Out, returning how many std::size_t collect( std::vector &Out, std::size_t const Count ); // tiles queued or finished and not yet collected @@ -88,20 +58,21 @@ public: private: void work(); - // whether the tile is being read or waits to be collected. caller holds the lock - bool underway( request const &Tile ) const; + // whether the tile is being read or waits to be collected. the caller holds the lock + bool underway( quantizedmesh::tile_address const &Tile ) const; quantizedmesh::reader m_reader; + double m_originx { 0.0 }, m_originy { 0.0 }, m_originz { 0.0 }; std::thread m_thread; mutable std::mutex m_mutex; std::condition_variable m_wake; - std::deque m_pending; - std::optional m_running; + std::deque m_pending; + std::optional m_running; std::deque m_done; bool m_stop { false }; - // finished tiles allowed to wait for collection. past this the thread stops reading: - // the renderer uploads at its own pace, and a backlog read for where the camera was a - // second ago is work spent on tiles nobody wants any more + // finished tiles allowed to wait for collection. past this the thread stops reading: the + // renderer uploads at its own pace, and a backlog read for where the camera was a second ago + // is work spent on tiles nobody wants any more static constexpr std::size_t donelimit { 32 }; }; diff --git a/rendering/terraintilepool_tin.cpp b/rendering/terraintilepool_tin.cpp deleted file mode 100644 index 00b95dfd..00000000 --- a/rendering/terraintilepool_tin.cpp +++ /dev/null @@ -1,130 +0,0 @@ -/* -This Source Code Form is subject to the -terms of the Mozilla Public License, v. -2.0. If a copy of the MPL was not -distributed with this file, You can -obtain one at -http://mozilla.org/MPL/2.0/. -*/ - -module; -#include -#include -#include -#include - -module eu07.rendering.terraintilepool_tin; -import eu07.glm; - -terrain_tile_pool_tin::terrain_tile_pool_tin() = default; - -terrain_tile_pool_tin::~terrain_tile_pool_tin() = default; - -terrain_tile_pool_tin::tile_handle -terrain_tile_pool_tin::upload( - std::vector const &positions_x, - std::vector const &positions_y, - std::vector const &positions_z, - std::vector const &uvs_u, - std::vector const &uvs_v, - std::vector const &materials, - std::vector const &indices, - glm::dvec3 const &origin -) { - tile_handle handle; - handle.origin = origin; - - if( positions_x.empty() || indices.empty() ) { - return handle; // Invalid - } - - std::size_t const vertex_count = positions_x.size(); - handle.vertex_count = static_cast( vertex_count ); - handle.index_count = static_cast( indices.size() ); - - // Pack vertices into interleaved format - std::vector vertices; - vertices.reserve( vertex_count ); - - for( std::size_t i = 0; i < vertex_count; ++i ) { - vertex v; - // Store relative to origin for camera-relative rendering - v.x = static_cast( positions_x[ i ] - origin.x ); - v.y = static_cast( positions_y[ i ] - origin.y ); - v.z = static_cast( positions_z[ i ] - origin.z ); - v.u = uvs_u[ i ]; - v.v = uvs_v[ i ]; - v.material = i < materials.size() ? materials[ i ] : 0; - v.pad[0] = v.pad[1] = v.pad[2] = 0; - vertices.push_back( v ); - } - - // Create VAO - glGenVertexArrays( 1, &handle.vao_id ); - glBindVertexArray( handle.vao_id ); - - // Create and upload VBO - glGenBuffers( 1, &handle.vbo_id ); - glBindBuffer( GL_ARRAY_BUFFER, handle.vbo_id ); - glBufferData( GL_ARRAY_BUFFER, - vertices.size() * sizeof( vertex ), - vertices.data(), - GL_STATIC_DRAW ); - - // Position attribute (location 0) - glEnableVertexAttribArray( 0 ); - glVertexAttribPointer( 0, 3, GL_FLOAT, GL_FALSE, sizeof( vertex ), - reinterpret_cast( offsetof( vertex, x ) ) ); - - // UV attribute (location 1) - glEnableVertexAttribArray( 1 ); - glVertexAttribPointer( 1, 2, GL_FLOAT, GL_FALSE, sizeof( vertex ), - reinterpret_cast( offsetof( vertex, u ) ) ); - - // Material attribute (location 2) - glEnableVertexAttribArray( 2 ); - glVertexAttribIPointer( 2, 1, GL_UNSIGNED_BYTE, sizeof( vertex ), - reinterpret_cast( offsetof( vertex, material ) ) ); - - // Create and upload IBO - glGenBuffers( 1, &handle.ibo_id ); - glBindBuffer( GL_ELEMENT_ARRAY_BUFFER, handle.ibo_id ); - glBufferData( GL_ELEMENT_ARRAY_BUFFER, - indices.size() * sizeof( std::uint32_t ), - indices.data(), - GL_STATIC_DRAW ); - - // Cleanup - glBindVertexArray( 0 ); - glBindBuffer( GL_ARRAY_BUFFER, 0 ); - glBindBuffer( GL_ELEMENT_ARRAY_BUFFER, 0 ); - - // Track allocation - m_allocated_vertices += vertex_count; - m_allocated_indices += indices.size(); - - return handle; -} - -void -terrain_tile_pool_tin::free( tile_handle &Handle ) { - if( !Handle.valid() ) return; - - // Free GL resources - if( Handle.vao_id != 0 ) { - glDeleteVertexArrays( 1, &Handle.vao_id ); - } - if( Handle.vbo_id != 0 ) { - glDeleteBuffers( 1, &Handle.vbo_id ); - } - if( Handle.ibo_id != 0 ) { - glDeleteBuffers( 1, &Handle.ibo_id ); - } - - // Update stats - m_allocated_vertices -= Handle.vertex_count; - m_allocated_indices -= Handle.index_count; - - // Invalidate handle - Handle = tile_handle{}; -} diff --git a/rendering/terraintilepool_tin.cppm b/rendering/terraintilepool_tin.cppm deleted file mode 100644 index 3d65f5cb..00000000 --- a/rendering/terraintilepool_tin.cppm +++ /dev/null @@ -1,85 +0,0 @@ -/* -This Source Code Form is subject to the -terms of the Mozilla Public License, v. -2.0. If a copy of the MPL was not -distributed with this file, You can -obtain one at -http://mozilla.org/MPL/2.0/. -*/ - -module; -#include -#include -#include -#include - -export module eu07.rendering.terraintilepool_tin; -import eu07.gl.buffer; -import eu07.glm; - -export { - -// GPU storage for TIN terrain tiles - vertex and index buffers. -// Replaces texture array approach used by heightfield. -// -// Each tile gets its own VBO/IBO pair. Tiles are uploaded as they arrive from the loader -// and freed when they leave range. Budget is vertex/index count rather than texture memory. -class terrain_tile_pool_tin { - -public: - // Vertex format for terrain - struct vertex { - float x, y, z; // Position (relative to tile origin) - float u, v; // Texture coordinates - std::uint8_t material; // Material index - std::uint8_t pad[3]; // Alignment to 24 bytes - }; - - static_assert( sizeof( vertex ) == 24 ); - - // Handle to uploaded tile - struct tile_handle { - std::uint32_t vbo_id { 0 }; - std::uint32_t ibo_id { 0 }; - std::uint32_t vao_id { 0 }; - std::uint32_t vertex_count { 0 }; - std::uint32_t index_count { 0 }; - glm::dvec3 origin { 0.0 }; // World origin for camera-relative rendering - - bool valid() const { return vbo_id != 0; } - }; - - terrain_tile_pool_tin(); - ~terrain_tile_pool_tin(); - - terrain_tile_pool_tin( terrain_tile_pool_tin const & ) = delete; - terrain_tile_pool_tin &operator=( terrain_tile_pool_tin const & ) = delete; - - // Upload tile data to GPU - tile_handle upload( - std::vector const &positions_x, - std::vector const &positions_y, - std::vector const &positions_z, - std::vector const &uvs_u, - std::vector const &uvs_v, - std::vector const &materials, - std::vector const &indices, - glm::dvec3 const &origin // Tile center for relative positioning - ); - - // Free tile from GPU - void free( tile_handle &Handle ); - - // Statistics - std::size_t allocated_vertices() const { return m_allocated_vertices; } - std::size_t allocated_indices() const { return m_allocated_indices; } - std::size_t allocated_bytes() const { - return m_allocated_vertices * sizeof( vertex ) + m_allocated_indices * sizeof( std::uint32_t ); - } - -private: - std::size_t m_allocated_vertices { 0 }; - std::size_t m_allocated_indices { 0 }; -}; - -} // export diff --git a/scene/pakformat.h b/scene/pakformat.h new file mode 100644 index 00000000..c1b3f41d --- /dev/null +++ b/scene/pakformat.h @@ -0,0 +1,179 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +// One file holding a scenery's cooked terrain tiles. +// +// A terrain is tens of thousands of small files, which is the one thing a filesystem is worst at: +// opening them costs more than reading them. They go into an archive instead, each entry +// compressed on its own so that a tile can be read without touching the rest, and the index is +// written last so that cooking never has to hold the tiles in memory. + +#include +#include +#include +#include +#include +#include + +#include + +namespace pak { + +inline constexpr char magic[ 8 ] { 'E', 'U', '0', '7', 'P', 'A', 'K', '\0' }; +inline constexpr std::uint32_t version { 2 }; +inline constexpr std::size_t namelength { 64 }; + +#pragma pack( push, 1 ) + +struct file_header { + char magic[ 8 ]; + std::uint32_t version; + std::uint32_t entrycount; + std::uint64_t indexoffset; +}; + +struct index_entry { + char name[ namelength ]; + std::uint64_t offset; + std::uint64_t stored; // bytes in the file + std::uint64_t plain; // bytes once decompressed; equal to stored when kept as it is +}; + +#pragma pack( pop ) + +// Writes entries straight through to disk as they are added. +class writer { + +public: + bool open( std::string const &Path ) { + m_file.open( Path, std::ios::binary | std::ios::trunc ); + if( false == m_file.good() ) { return false; } + file_header header {}; + m_file.write( reinterpret_cast( &header ), sizeof( header ) ); + m_at = sizeof( header ); + return m_file.good(); + } + + bool add( std::string const &Name, std::vector const &Data ) { + if( ( false == m_file.good() ) || ( Name.size() >= namelength ) ) { return false; } + index_entry entry {}; + std::memcpy( entry.name, Name.c_str(), Name.size() ); + entry.offset = m_at; + entry.plain = Data.size(); + + m_packed.resize( ZSTD_compressBound( Data.size() ) ); + auto const packed { + ZSTD_compress( m_packed.data(), m_packed.size(), Data.data(), Data.size(), 3 ) }; + auto const *bytes { reinterpret_cast( Data.data() ) }; + auto size { Data.size() }; + if( ( 0 == ZSTD_isError( packed ) ) && ( packed < Data.size() ) ) { + bytes = reinterpret_cast( m_packed.data() ); + size = packed; + } + entry.stored = size; + m_file.write( bytes, static_cast( size ) ); + m_at += size; + m_index.push_back( entry ); + return m_file.good(); + } + + bool close() { + if( false == m_file.good() ) { return false; } + file_header header {}; + std::memcpy( header.magic, magic, sizeof( magic ) ); + header.version = version; + header.entrycount = static_cast( m_index.size() ); + header.indexoffset = m_at; + for( auto const &entry : m_index ) { + m_file.write( reinterpret_cast( &entry ), sizeof( entry ) ); + } + m_file.seekp( 0 ); + m_file.write( reinterpret_cast( &header ), sizeof( header ) ); + auto const good { m_file.good() }; + m_file.close(); + return good; + } + + std::size_t entrycount() const { return m_index.size(); } + std::uint64_t bytes() const { return m_at + m_index.size() * sizeof( index_entry ); } + +private: + std::ofstream m_file; + std::uint64_t m_at { 0 }; + std::vector m_index; + std::vector m_packed; +}; + +// Reads entries back. One reader is used from one thread at a time: it keeps a handle and seeks. +class reader { + +public: + bool open( std::string const &Path ) { + close(); + m_file.open( Path, std::ios::binary ); + if( false == m_file.good() ) { return false; } + file_header header {}; + m_file.read( reinterpret_cast( &header ), sizeof( header ) ); + if( false == m_file.good() ) { return false; } + if( 0 != std::memcmp( header.magic, magic, sizeof( magic ) ) ) { return false; } + if( header.version != version ) { return false; } + m_file.seekg( static_cast( header.indexoffset ) ); + for( std::uint32_t entry = 0; entry < header.entrycount; ++entry ) { + index_entry record {}; + m_file.read( reinterpret_cast( &record ), sizeof( record ) ); + if( false == m_file.good() ) { return false; } + record.name[ namelength - 1 ] = '\0'; + m_index.emplace( record.name, record ); + } + return true; + } + + void close() { + m_index.clear(); + if( true == m_file.is_open() ) { m_file.close(); } + m_file.clear(); + } + + bool exists( std::string const &Name ) const { return m_index.contains( Name ); } + std::size_t entrycount() const { return m_index.size(); } + + std::vector names() const { + std::vector names; + names.reserve( m_index.size() ); + for( auto const &[ name, entry ] : m_index ) { names.push_back( name ); } + return names; + } + + bool read( std::string const &Name, std::vector &Out ) { + auto const found { m_index.find( Name ) }; + if( found == m_index.end() ) { return false; } + auto const &entry { found->second }; + m_stored.resize( entry.stored ); + m_file.seekg( static_cast( entry.offset ) ); + m_file.read( reinterpret_cast( m_stored.data() ), static_cast( entry.stored ) ); + if( false == m_file.good() ) { m_file.clear(); return false; } + if( entry.stored == entry.plain ) { + Out.swap( m_stored ); + return true; + } + Out.resize( entry.plain ); + auto const plain { + ZSTD_decompress( Out.data(), Out.size(), m_stored.data(), m_stored.size() ) }; + return ( 0 == ZSTD_isError( plain ) ) && ( plain == entry.plain ); + } + +private: + std::ifstream m_file; + std::unordered_map m_index; + std::vector m_stored; +}; + +} // namespace pak diff --git a/scene/quantizedmeshcooker.cpp b/scene/quantizedmeshcooker.cpp new file mode 100644 index 00000000..1fce2914 --- /dev/null +++ b/scene/quantizedmeshcooker.cpp @@ -0,0 +1,981 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#include "scene/quantizedmeshcooker.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace quantizedmesh { + +namespace { + +// a triangle with less surface than this is nothing at all, whichever way it faces, and is the +// only thing the cook refuses to lay. Walls - the sides of an embankment, a cutting, an abutment - +// are ground as much as anything else: nothing about a mesh says it has to be a height field, and +// leaving them out is what opens holes along every bank +constexpr double nullarea { 1e-6 }; +// vertices nearer than this to each other, to a tile border or to a plane are the same thing. +// a tenth of a millimetre is below anything the game can show and far above what the arithmetic +// of cutting a triangle can be out by +constexpr double weldepsilon { 1e-4 }; +// the deepest quadtree worth cooking: 4096 tiles along a side of the finest level +constexpr std::uint32_t maxlevels { 13 }; +// The finest tile is this many typical triangle edges across, and never outside these bounds. A tile +// has to be large against the triangles in it: a triangle crossing a border is cut, and the pieces +// cost vertices neither side had. It also has to stay small enough to be worth loading and culling +// on its own, which is what the bounds are for. +constexpr double tileedges { 16.0 }; +constexpr double smallesttileside { 128.0 }; +constexpr double largesttileside { 1024.0 }; +// how many edge lengths are kept to find the typical one. the first hundred thousand triangles say +// as much about a scenery as all of them +constexpr std::size_t edgesample { 100000 }; +// a tile is never collapsed below this, however far away it is drawn: past a point the triangles +// saved are not worth the ground losing its shape +constexpr std::size_t smallesttile { 64 }; +// A level may move the ground by this share of its tile's side, and no further. A level is used from +// about twice its tile's side away, where a share this small is well under a pixel; letting it go +// further would buy triangles that no distance is far enough to use. Ground already so coarse that a +// level cannot be thinned within it simply comes out nearly a copy of the level below, which is the +// right answer: there was nothing there to save. +constexpr double errorshare { 128.0 }; + +// the area of a triangle in space, whichever way it faces +double +surface_area( cooker::corner const &A, cooker::corner const &B, cooker::corner const &C ) { + auto const nx { ( B.y - A.y ) * ( C.z - A.z ) - ( B.z - A.z ) * ( C.y - A.y ) }; + auto const ny { ( B.z - A.z ) * ( C.x - A.x ) - ( B.x - A.x ) * ( C.z - A.z ) }; + auto const nz { ( B.x - A.x ) * ( C.y - A.y ) - ( B.y - A.y ) * ( C.x - A.x ) }; + return 0.5 * std::sqrt( nx * nx + ny * ny + nz * nz ); +} + +// a triangle standing upright has no extent in the plan at all. it still belongs to the ground, +// but it has to be told apart: the tile it goes to cannot be decided by cutting it in two +double +planar_extent( cooker::corner const &A, cooker::corner const &B, cooker::corner const &C ) { + return std::max( + std::max( { A.x, B.x, C.x } ) - std::min( { A.x, B.x, C.x } ), + std::max( { A.z, B.z, C.z } ) - std::min( { A.z, B.z, C.z } ) ); +} + +// The error of collapsing a vertex away, as Garland and Heckbert measure it: the sum of the +// squared distances to the planes of the triangles that met there. Kept as the ten distinct +// entries of the symmetric matrix. +struct quadric { + double m[ 10 ] {}; + + void add_plane( double const A, double const B, double const C, double const D, double const Weight ) { + m[ 0 ] += Weight * A * A; m[ 1 ] += Weight * A * B; m[ 2 ] += Weight * A * C; m[ 3 ] += Weight * A * D; + m[ 4 ] += Weight * B * B; m[ 5 ] += Weight * B * C; m[ 6 ] += Weight * B * D; + m[ 7 ] += Weight * C * C; m[ 8 ] += Weight * C * D; + m[ 9 ] += Weight * D * D; + } + + void operator+=( quadric const &Other ) { + for( std::size_t index = 0; index < 10; ++index ) { m[ index ] += Other.m[ index ]; } + } + + double at( double const X, double const Y, double const Z ) const { + return m[ 0 ] * X * X + 2.0 * m[ 1 ] * X * Y + 2.0 * m[ 2 ] * X * Z + 2.0 * m[ 3 ] * X + + m[ 4 ] * Y * Y + 2.0 * m[ 5 ] * Y * Z + 2.0 * m[ 6 ] * Y + + m[ 7 ] * Z * Z + 2.0 * m[ 8 ] * Z + + m[ 9 ]; + } +}; + +// which sides of a tile's square a point lies on, one bit each +std::uint8_t +sides_of( double const X, double const Z, double const West, double const North, double const Side ) { + std::uint8_t sides { 0 }; + if( std::abs( X - West ) < weldepsilon ) { sides |= 1; } + if( std::abs( X - ( West + Side ) ) < weldepsilon ) { sides |= 2; } + if( std::abs( Z - North ) < weldepsilon ) { sides |= 4; } + if( std::abs( Z - ( North + Side ) ) < weldepsilon ){ sides |= 8; } + return sides; +} + +// a vertex of a polygon being cut along the tile grid +struct clipvertex { + double x, y, z; +}; + +clipvertex +between( clipvertex const &From, clipvertex const &To, double const Share ) { + return { + From.x + ( To.x - From.x ) * Share, + From.y + ( To.y - From.y ) * Share, + From.z + ( To.z - From.z ) * Share }; +} + +// Sutherland and Hodgman against one side of the tile's square. Axis 0 is x and 1 is z; the +// coordinate cut against is set to the bound itself rather than to what the division gives, so +// that the two tiles sharing that border end up with a vertex in exactly the same place. +void +clip_side( std::vector &Polygon, int const Axis, double const Bound, bool const Keepabove ) { + + if( Polygon.empty() ) { return; } + + auto const coordinate = [ Axis ]( clipvertex const &Vertex ) { + return ( Axis == 0 ) ? Vertex.x : Vertex.z; }; + auto const inside = [ & ]( clipvertex const &Vertex ) { + return Keepabove ? ( coordinate( Vertex ) >= Bound - weldepsilon ) + : ( coordinate( Vertex ) <= Bound + weldepsilon ); }; + + std::vector kept; + kept.reserve( Polygon.size() + 2 ); + for( std::size_t index = 0; index < Polygon.size(); ++index ) { + auto const &here { Polygon[ index ] }; + auto const &next { Polygon[ ( index + 1 ) % Polygon.size() ] }; + auto const hereinside { inside( here ) }; + if( true == hereinside ) { kept.push_back( here ); } + if( hereinside == inside( next ) ) { continue; } + auto const span { coordinate( next ) - coordinate( here ) }; + if( std::abs( span ) < weldepsilon ) { continue; } + auto crossing { between( here, next, ( Bound - coordinate( here ) ) / span ) }; + if( Axis == 0 ) { crossing.x = Bound; } else { crossing.z = Bound; } + kept.push_back( crossing ); + } + Polygon.swap( kept ); +} + +} // anonymous namespace + +cooker::~cooker() { + + if( true == m_scratch.is_open() ) { m_scratch.close(); } + if( false == m_scratchpath.empty() ) { + std::error_code error; + std::filesystem::remove( m_scratchpath, error ); + } +} + +bool +cooker::begin( std::string const &Scratchpath, std::size_t const Tilebudget ) { + + m_scratchpath = Scratchpath; + m_tilebudget = std::max( 256, Tilebudget ); + m_scratch.open( m_scratchpath, std::ios::binary | std::ios::in | std::ios::out | std::ios::trunc ); + return m_scratch.good(); +} + +std::uint16_t +cooker::material_index( std::string_view const Material ) { + + auto const found { m_materiallookup.find( std::string { Material } ) }; + if( found != m_materiallookup.end() ) { return found->second; } + auto const index { static_cast( m_materials.size() ) }; + m_materials.emplace_back( Material ); + m_materiallookup.emplace( std::string { Material }, index ); + return index; +} + +// The scenery's texture coordinates are not kept: ground textures tile in world space, so a +// coordinate follows from where a vertex is. What has to be kept is the scale the scenery drew the +// material at, and that is measured here - the ratio of world distance to texture distance along +// a triangle's edges. Edges too short, or with no texture movement along them, say nothing. +void +cooker::measure_repeat( std::uint16_t const Material, corner const ( &Corners )[ 3 ] ) { + + if( Material >= m_repeatcounts.size() ) { + m_repeatsums.resize( Material + 1, 0.0 ); + m_repeatsquares.resize( Material + 1, 0.0 ); + m_repeatcounts.resize( Material + 1, 0 ); + } + for( std::size_t index = 0; index < 3; ++index ) { + auto const &from { Corners[ index ] }; + auto const &to { Corners[ ( index + 1 ) % 3 ] }; + auto const world { + std::sqrt( ( to.x - from.x ) * ( to.x - from.x ) + ( to.z - from.z ) * ( to.z - from.z ) ) }; + auto const texture { + std::sqrt( ( to.u - from.u ) * ( to.u - from.u ) + ( to.v - from.v ) * ( to.v - from.v ) ) }; + if( ( world < 0.5 ) || ( texture < 1e-4 ) ) { continue; } + auto const ratio { world / texture }; + m_repeatsums[ Material ] += ratio; + m_repeatsquares[ Material ] += ratio * ratio; + ++m_repeatcounts[ Material ]; + } +} + +void +cooker::add( corner const &A, corner const &B, corner const &C, std::string_view const Material ) { + + auto const id { static_cast( m_added ) }; + ++m_added; + + if( ( surface_area( A, B, C ) < nullarea ) || ( planar_extent( A, B, C ) < weldepsilon ) ) { + // no surface, or standing on a single point in the plan: nothing to lay, and the scenery + // goes on drawing it itself + m_refusals.push_back( id ); + return; + } + + corner const corners[ 3 ] { A, B, C }; + if( m_edges.size() < edgesample ) { + for( std::size_t index = 0; index < 3; ++index ) { + auto const &from { corners[ index ] }; + auto const &to { corners[ ( index + 1 ) % 3 ] }; + auto const length { + std::sqrt( ( to.x - from.x ) * ( to.x - from.x ) + ( to.z - from.z ) * ( to.z - from.z ) ) }; + if( length > 0.0 ) { m_edges.push_back( length ); } + } + } + spilled record {}; + record.material = material_index( Material ); + measure_repeat( record.material, corners ); + for( std::size_t index = 0; index < 3; ++index ) { + record.corners[ index * 3 + 0 ] = corners[ index ].x; + record.corners[ index * 3 + 1 ] = corners[ index ].y; + record.corners[ index * 3 + 2 ] = corners[ index ].z; + } + m_scratch.write( reinterpret_cast( &record ), sizeof( record ) ); + + for( auto const &vertex : corners ) { + if( false == m_anything ) { + m_anything = true; + m_minx = m_maxx = vertex.x; + m_minz = m_maxz = vertex.z; + m_lowest = m_highest = vertex.y; + continue; + } + m_minx = std::min( m_minx, vertex.x ); m_maxx = std::max( m_maxx, vertex.x ); + m_minz = std::min( m_minz, vertex.z ); m_maxz = std::max( m_maxz, vertex.z ); + m_lowest = std::min( m_lowest, vertex.y ); m_highest = std::max( m_highest, vertex.y ); + } +} + +// The root square starts on a multiple of the finest tile side and doubles until it holds +// everything, so that where a tile's borders fall follows from the terrain alone and a second +// cook of the same scenery puts them in the same place. +void +cooker::measure() { + + // the middle edge length of the sample, and the tile that suits it, rounded to a power of two so + // that the same scenery always comes out the same way + if( false == m_edges.empty() ) { + auto const middle { m_edges.begin() + m_edges.size() / 2 }; + std::nth_element( m_edges.begin(), middle, m_edges.end() ); + auto const wanted { std::clamp( tileedges * *middle, smallesttileside, largesttileside ) }; + m_finesttile = std::exp2( std::round( std::log2( wanted ) ) ); + m_finesttile = std::clamp( m_finesttile, smallesttileside, largesttileside ); + m_edges.clear(); + m_edges.shrink_to_fit(); + } + + m_west = std::floor( m_minx / m_finesttile ) * m_finesttile; + m_north = std::floor( m_minz / m_finesttile ) * m_finesttile; + auto const needed { std::max( m_maxx - m_west, m_maxz - m_north ) }; + m_side = m_finesttile; + m_levels = 1; + while( ( m_side < needed ) && ( m_levels < maxlevels ) ) { + m_side *= 2.0; + ++m_levels; + } + // a height range of zero would make every height quantize to nothing + if( m_highest - m_lowest < 1.0 ) { m_highest = m_lowest + 1.0; } +} + +void +cooker::square( tile_address const &Tile, double &West, double &North, double &Side ) const { + + Side = m_side / static_cast( 1u << Tile.level ); + West = m_west + Tile.x * Side; + North = m_north + Tile.z * Side; +} + +bool +cooker::finish( sink const &Sink, terrain_table &Table ) { + + std::sort( m_refusals.begin(), m_refusals.end() ); + m_report.refused = m_refusals.size(); + + if( ( false == m_anything ) || ( false == m_scratch.good() ) ) { + return false; + } + m_scratch.flush(); + measure(); + + // which of the spilled triangles touch which tile of the finest level. a triangle can span + // several, and each of them cuts its own piece out of it + auto const finest { m_levels - 1 }; + auto const acrossfinest { static_cast( 1u << finest ) }; + auto const records { m_scratch.tellp() / static_cast( sizeof( spilled ) ) }; + m_scratch.seekg( 0 ); + for( std::streamoff index = 0; index < records; ++index ) { + spilled record {}; + m_scratch.read( reinterpret_cast( &record ), sizeof( record ) ); + if( false == m_scratch.good() ) { return false; } + auto minx { record.corners[ 0 ] }, maxx { minx }; + auto minz { record.corners[ 2 ] }, maxz { minz }; + for( std::size_t at = 0; at < 3; ++at ) { + minx = std::min( minx, record.corners[ at * 3 + 0 ] ); maxx = std::max( maxx, record.corners[ at * 3 + 0 ] ); + minz = std::min( minz, record.corners[ at * 3 + 2 ] ); maxz = std::max( maxz, record.corners[ at * 3 + 2 ] ); + } + auto const first = [ this ]( double const Value, double const Origin ) { + return static_cast( std::floor( ( Value - Origin ) / m_finesttile ) ); }; + auto const fromx { std::clamp( first( minx, m_west ), 0, acrossfinest - 1 ) }; + auto const tox { std::clamp( first( maxx, m_west ), 0, acrossfinest - 1 ) }; + auto const fromz { std::clamp( first( minz, m_north ), 0, acrossfinest - 1 ) }; + auto const toz { std::clamp( first( maxz, m_north ), 0, acrossfinest - 1 ) }; + for( auto x = fromx; x <= tox; ++x ) { + for( auto z = fromz; z <= toz; ++z ) { + m_placement.emplace_back( + tile_key( { finest, x, z } ), static_cast( index ) ); + } + } + } + std::sort( m_placement.begin(), m_placement.end() ); + + m_errors.assign( m_levels, 0.0 ); + cook( { 0, 0, 0 }, Sink ); + m_report.levels = m_levels; + + Table.rules = cook_rules; + Table.west = m_west; + Table.north = m_north; + Table.side = m_side; + Table.lowest = m_lowest; + Table.highest = m_highest; + Table.levels = m_levels; + Table.errors.assign( m_errors.begin(), m_errors.end() ); + Table.materials.clear(); + for( std::size_t index = 0; index < m_materials.size(); ++index ) { + // a material the measurement never caught falls back to something plausible rather than + // to zero, which would divide by nothing in the shader + auto const caught { ( index < m_repeatcounts.size() ) && ( m_repeatcounts[ index ] > 0 ) }; + auto const repeat { + caught ? m_repeatsums[ index ] / static_cast( m_repeatcounts[ index ] ) : 4.0 }; + Table.materials.push_back( { m_materials[ index ], static_cast( repeat ) } ); + } + + m_scratch.close(); + std::error_code error; + std::filesystem::remove( m_scratchpath, error ); + m_scratchpath.clear(); + return true; +} + +cooker::mesh +cooker::cook( tile_address const &Tile, sink const &Sink ) { + + auto result { + ( Tile.level + 1 == m_levels ) ? lay( Tile ) : merge( Tile, Sink ) }; + if( true == result.empty() ) { return result; } + + if( Tile.level + 1 < m_levels ) { + // The finest level is the ground as the scenery drew it and is left alone. A level above it + // holds what four tiles below it held, and is collapsed back to what one of them held: the + // cost of a level is then a quarter of the level below, and the whole pyramid costs a third + // more than the finest alone. A count fixed in advance cannot do that - a tile that never + // reaches it would be copied rather than simplified - so it only serves as a ceiling. + double west { 0.0 }, north { 0.0 }, side { 0.0 }; + square( Tile, west, north, side ); + auto const merged { result.indices.size() / 3 }; + simplify( result, std::clamp( merged / 4, smallesttile, m_tilebudget ), side / errorshare ); + } + + m_errors[ Tile.level ] = std::max( m_errors[ Tile.level ], result.error ); + + std::vector bytes; + encode( Tile, result, bytes ); + Sink( Tile, bytes ); + + ++m_report.tiles; + m_report.triangles += result.indices.size() / 3; + m_report.vertices += result.vertices.size(); + m_report.bytes += bytes.size(); + if( ( false == m_quiet ) && ( m_report.tiles % 200 == 0 ) ) { + std::printf( "\rCooking terrain: %zu tiles", m_report.tiles ); + std::fflush( stdout ); + } + return result; +} + +namespace { + +// Welds vertices that land in the same place and belong to the same material. Everything the cut +// produces is either a vertex of the source or a point on a tile border worked out the same way +// from both sides, so the tolerance only has to cover the last bits of the arithmetic. +class welder { + +public: + explicit welder( std::vector &Vertices ) : m_vertices( Vertices ) {} + + std::uint32_t operator()( cooker::meshvertex const &Vertex ) { + auto const key { make_key( Vertex ) }; + auto const found { m_known.find( key ) }; + if( found != m_known.end() ) { return found->second; } + auto const index { static_cast( m_vertices.size() ) }; + m_vertices.push_back( Vertex ); + m_known.emplace( key, index ); + return index; + } + +private: + struct key { + std::int64_t x, y, z; + std::uint16_t material; + bool operator==( key const & ) const = default; + }; + struct keyhash { + std::size_t operator()( key const &Key ) const noexcept { + std::size_t hash { 1469598103934665603ull }; + auto const mix = [ &hash ]( std::int64_t const Value ) { + hash ^= static_cast( Value ); + hash *= 1099511628211ull; }; + mix( Key.x ); mix( Key.y ); mix( Key.z ); + mix( Key.material ); + return hash; + } + }; + + static key make_key( cooker::meshvertex const &Vertex ) { + auto const grid = []( double const Value ) { + return static_cast( std::llround( Value / weldepsilon ) ); }; + return { grid( Vertex.x ), grid( Vertex.y ), grid( Vertex.z ), Vertex.material }; + } + + std::vector &m_vertices; + std::unordered_map m_known; +}; + +} // anonymous namespace + +cooker::mesh +cooker::lay( tile_address const &Tile ) { + + mesh result; + double west { 0.0 }, north { 0.0 }, side { 0.0 }; + square( Tile, west, north, side ); + + auto const wanted { tile_key( Tile ) }; + auto const from { + std::lower_bound( m_placement.begin(), m_placement.end(), + std::pair { wanted, 0 } ) }; + auto const to { + std::upper_bound( m_placement.begin(), m_placement.end(), + std::pair { wanted, std::numeric_limits::max() } ) }; + if( from == to ) { return result; } + + welder weld { result.vertices }; + std::vector polygon; + for( auto entry = from; entry != to; ++entry ) { + + m_scratch.seekg( static_cast( entry->second ) * sizeof( spilled ) ); + spilled record {}; + m_scratch.read( reinterpret_cast( &record ), sizeof( record ) ); + if( false == m_scratch.good() ) { m_scratch.clear(); continue; } + + polygon.clear(); + for( std::size_t at = 0; at < 3; ++at ) { + polygon.push_back( + { record.corners[ at * 3 + 0 ], record.corners[ at * 3 + 1 ], record.corners[ at * 3 + 2 ] } ); + } + clip_side( polygon, 0, west, true ); + clip_side( polygon, 0, west + side, false ); + clip_side( polygon, 1, north, true ); + clip_side( polygon, 1, north + side, false ); + if( polygon.size() < 3 ) { continue; } + + // A wall lying along a tile border is inside both tiles: the cut keeps it whole on either + // side, since it has nothing to cut. Whichever border it sits on, the tile past it is the + // one that keeps it, so that it is laid exactly once. + { + auto onborderonly { false }; + for( int axis = 0; axis < 2; ++axis ) { + auto const at = [ axis ]( clipvertex const &Vertex ) { return axis == 0 ? Vertex.x : Vertex.z; }; + auto low { at( polygon.front() ) }, high { low }; + for( auto const &vertex : polygon ) { + low = std::min( low, at( vertex ) ); + high = std::max( high, at( vertex ) ); + } + auto const upper { ( axis == 0 ? west + side : north + side ) }; + if( ( high - low < weldepsilon ) && ( std::abs( low - upper ) < weldepsilon ) ) { + onborderonly = true; + } + } + if( true == onborderonly ) { continue; } + } + + auto const place = [ & ]( clipvertex const &Vertex ) { + return weld( { Vertex.x, Vertex.y, Vertex.z, record.material, + sides_of( Vertex.x, Vertex.z, west, north, side ) } ); }; + + auto const first { place( polygon[ 0 ] ) }; + for( std::size_t corner = 1; corner + 1 < polygon.size(); ++corner ) { + // the cut keeps the source winding, so fanning the polygon keeps it too + auto const second { place( polygon[ corner ] ) }; + auto const third { place( polygon[ corner + 1 ] ) }; + if( ( first == second ) || ( second == third ) || ( third == first ) ) { continue; } + result.indices.push_back( first ); + result.indices.push_back( second ); + result.indices.push_back( third ); + } + } + return result; +} + +cooker::mesh +cooker::merge( tile_address const &Tile, sink const &Sink ) { + + mesh result; + double west { 0.0 }, north { 0.0 }, side { 0.0 }; + square( Tile, west, north, side ); + + welder weld { result.vertices }; + for( std::int32_t stepz = 0; stepz < 2; ++stepz ) { + for( std::int32_t stepx = 0; stepx < 2; ++stepx ) { + + auto const child { + cook( { Tile.level + 1, Tile.x * 2 + stepx, Tile.z * 2 + stepz }, Sink ) }; + if( true == child.empty() ) { continue; } + result.error = std::max( result.error, child.error ); + + // the borders between the four are no longer borders: only the outer edge of this + // tile has to stay where it is + std::vector here; + here.reserve( child.vertices.size() ); + for( auto vertex : child.vertices ) { + vertex.sides = sides_of( vertex.x, vertex.z, west, north, side ); + here.push_back( weld( vertex ) ); + } + for( auto const index : child.indices ) { + result.indices.push_back( here[ index ] ); + } + } + } + return result; +} + +// Edge collapse, cheapest first. A vertex is only ever collapsed onto one of its neighbours - +// never onto a new point between them - so a vertex that has to stay put, because a neighbouring +// tile shares it, simply is never the one that goes. That is the whole of the crack handling: +// levels meet because the border is literally the same set of vertices in both. +void +cooker::simplify( mesh &Mesh, std::size_t const Budget, double const Allowed ) { + + auto triangles { Mesh.indices.size() / 3 }; + if( triangles <= Budget ) { return; } + + auto &vertices { Mesh.vertices }; + + // The quadrics are built and evaluated about the middle of the mesh rather than about the + // scenery's origin. A plane's own constant is minus the normal times a point on it, so in world + // coordinates it runs to tens or hundreds of thousands, and the quadric's value - a small + // distance squared - comes out as the difference of two such numbers. There is no precision left + // in that. Measured from the middle of the tile the constants are the size of the tile. + std::vector> local( vertices.size() ); + { + std::array middle { 0.0, 0.0, 0.0 }; + for( auto const &vertex : vertices ) { + middle[ 0 ] += vertex.x; middle[ 1 ] += vertex.y; middle[ 2 ] += vertex.z; + } + for( auto &value : middle ) { value /= static_cast( vertices.size() ); } + for( std::size_t index = 0; index < vertices.size(); ++index ) { + local[ index ] = { + vertices[ index ].x - middle[ 0 ], + vertices[ index ].y - middle[ 1 ], + vertices[ index ].z - middle[ 2 ] }; + } + } + + std::vector quadrics( vertices.size() ); + // the worst the ground has been moved at each vertex by a collapse into it, in metres. what a + // level is off by against the level below it; merging four tiles already carried their own + // figures up, so the levels are not counted twice + std::vector moved( vertices.size(), 0.0 ); + std::vector> incident( vertices.size() ); + std::vector gone( triangles, false ); + std::vector version( vertices.size(), 0 ); + std::vector removed( vertices.size(), false ); + + auto const cornerof = [ &Mesh ]( std::size_t const Triangle, std::size_t const Corner ) -> std::uint32_t & { + return Mesh.indices[ Triangle * 3 + Corner ]; }; + + for( std::size_t triangle = 0; triangle < triangles; ++triangle ) { + auto const &a { local[ cornerof( triangle, 0 ) ] }; + auto const &b { local[ cornerof( triangle, 1 ) ] }; + auto const &c { local[ cornerof( triangle, 2 ) ] }; + double nx { ( b[ 1 ] - a[ 1 ] ) * ( c[ 2 ] - a[ 2 ] ) - ( b[ 2 ] - a[ 2 ] ) * ( c[ 1 ] - a[ 1 ] ) }; + double ny { ( b[ 2 ] - a[ 2 ] ) * ( c[ 0 ] - a[ 0 ] ) - ( b[ 0 ] - a[ 0 ] ) * ( c[ 2 ] - a[ 2 ] ) }; + double nz { ( b[ 0 ] - a[ 0 ] ) * ( c[ 1 ] - a[ 1 ] ) - ( b[ 1 ] - a[ 1 ] ) * ( c[ 0 ] - a[ 0 ] ) }; + auto const length { std::sqrt( nx * nx + ny * ny + nz * nz ) }; + if( length < 1e-12 ) { continue; } + auto const area { 0.5 * length }; + nx /= length; ny /= length; nz /= length; + auto const offset { -( nx * a[ 0 ] + ny * a[ 1 ] + nz * a[ 2 ] ) }; + for( std::size_t corner = 0; corner < 3; ++corner ) { + auto const index { cornerof( triangle, corner ) }; + quadrics[ index ].add_plane( nx, ny, nz, offset, area ); + incident[ index ].push_back( static_cast( triangle ) ); + } + } + + // Where the ground ends - the outline of a hole the scenery left, or the edge of the terrain - + // only one triangle meets the edge. Nothing in the quadrics above holds such an edge in + // place, so a plane standing on it, weighted heavily, is added to both its ends: the outline + // then survives into the coarse levels instead of being eaten away. + { + std::map, std::size_t> uses; + for( std::size_t triangle = 0; triangle < triangles; ++triangle ) { + for( std::size_t corner = 0; corner < 3; ++corner ) { + auto const from { cornerof( triangle, corner ) }; + auto const to { cornerof( triangle, ( corner + 1 ) % 3 ) }; + ++uses[ { std::min( from, to ), std::max( from, to ) } ]; + } + } + for( std::size_t triangle = 0; triangle < triangles; ++triangle ) { + auto const &a { local[ cornerof( triangle, 0 ) ] }; + auto const &b { local[ cornerof( triangle, 1 ) ] }; + auto const &c { local[ cornerof( triangle, 2 ) ] }; + double nx { ( b[ 1 ] - a[ 1 ] ) * ( c[ 2 ] - a[ 2 ] ) - ( b[ 2 ] - a[ 2 ] ) * ( c[ 1 ] - a[ 1 ] ) }; + double ny { ( b[ 2 ] - a[ 2 ] ) * ( c[ 0 ] - a[ 0 ] ) - ( b[ 0 ] - a[ 0 ] ) * ( c[ 2 ] - a[ 2 ] ) }; + double nz { ( b[ 0 ] - a[ 0 ] ) * ( c[ 1 ] - a[ 1 ] ) - ( b[ 1 ] - a[ 1 ] ) * ( c[ 0 ] - a[ 0 ] ) }; + auto const facelength { std::sqrt( nx * nx + ny * ny + nz * nz ) }; + if( facelength < 1e-12 ) { continue; } + nx /= facelength; ny /= facelength; nz /= facelength; + for( std::size_t corner = 0; corner < 3; ++corner ) { + auto const from { cornerof( triangle, corner ) }; + auto const to { cornerof( triangle, ( corner + 1 ) % 3 ) }; + auto const use { uses.find( { std::min( from, to ), std::max( from, to ) } ) }; + if( ( use == uses.end() ) || ( use->second != 1 ) ) { continue; } + auto const &head { local[ from ] }; + auto const &tail { local[ to ] }; + double ex { tail[ 0 ] - head[ 0 ] }, ey { tail[ 1 ] - head[ 1 ] }, ez { tail[ 2 ] - head[ 2 ] }; + auto const edgelength { std::sqrt( ex * ex + ey * ey + ez * ez ) }; + if( edgelength < 1e-9 ) { continue; } + ex /= edgelength; ey /= edgelength; ez /= edgelength; + // perpendicular to both the edge and the face it belongs to + auto const px { ey * nz - ez * ny }; + auto const py { ez * nx - ex * nz }; + auto const pz { ex * ny - ey * nx }; + auto const offset { -( px * head[ 0 ] + py * head[ 1 ] + pz * head[ 2 ] ) }; + auto const weight { 100.0 * edgelength * edgelength }; + quadrics[ from ].add_plane( px, py, pz, offset, weight ); + quadrics[ to ].add_plane( px, py, pz, offset, weight ); + } + } + } + + struct candidate { + double cost; + std::uint32_t from, to; + std::uint32_t fromversion, toversion; + bool operator<( candidate const &Other ) const { return cost > Other.cost; } + }; + std::priority_queue queue; + + // the cost of taking From away and letting To stand for both. only what the collapse leaves + // behind is measured; where To sits does not change, so its own quadric is already paid for + auto const cost_of = [ & ]( std::uint32_t const From, std::uint32_t const To ) { + quadric together { quadrics[ From ] }; + together += quadrics[ To ]; + return std::max( 0.0, together.at( local[ To ][ 0 ], local[ To ][ 1 ], local[ To ][ 2 ] ) ); }; + + // How far the ground moves where From used to be, if From were collapsed into To: the height of + // whatever would then lie over that spot, against the height that is there now. Worked out before + // the collapse rather than after it, so that a collapse which moves the ground too far is passed + // over instead of being taken and regretted. The cost above orders the candidates; this is the + // one that says whether a candidate may be taken at all, because it is a distance and the cost is + // not. + auto const drop_of = [ & ]( std::uint32_t const From, std::uint32_t const To ) { + auto const &was { local[ From ] }; + for( auto const triangle : incident[ From ] ) { + if( true == gone[ triangle ] ) { continue; } + std::array corners { + cornerof( triangle, 0 ), cornerof( triangle, 1 ), cornerof( triangle, 2 ) }; + // a triangle holding both ends of the edge disappears with it, and covers nothing after + if( std::find( corners.begin(), corners.end(), To ) != corners.end() ) { continue; } + std::array const *, 3> after { + &local[ corners[ 0 ] ], &local[ corners[ 1 ] ], &local[ corners[ 2 ] ] }; + for( std::size_t corner = 0; corner < 3; ++corner ) { + if( corners[ corner ] == From ) { after[ corner ] = &local[ To ]; } + } + auto const &a { *after[ 0 ] }; auto const &b { *after[ 1 ] }; auto const &c { *after[ 2 ] }; + auto const twice { + ( b[ 0 ] - a[ 0 ] ) * ( c[ 2 ] - a[ 2 ] ) - ( c[ 0 ] - a[ 0 ] ) * ( b[ 2 ] - a[ 2 ] ) }; + if( std::abs( twice ) < 1e-12 ) { continue; } + auto const first { + ( ( b[ 0 ] - was[ 0 ] ) * ( c[ 2 ] - was[ 2 ] ) - ( c[ 0 ] - was[ 0 ] ) * ( b[ 2 ] - was[ 2 ] ) ) / twice }; + auto const second { + ( ( c[ 0 ] - was[ 0 ] ) * ( a[ 2 ] - was[ 2 ] ) - ( a[ 0 ] - was[ 0 ] ) * ( c[ 2 ] - was[ 2 ] ) ) / twice }; + auto const third { 1.0 - first - second }; + if( ( first < -1e-9 ) || ( second < -1e-9 ) || ( third < -1e-9 ) ) { continue; } + return std::abs( first * a[ 1 ] + second * b[ 1 ] + third * c[ 1 ] - was[ 1 ] ); + } + // nothing would lie over that spot: the edge of the ground, or a wall, whose triangles cover + // nothing in the plan at all. what changes there is the height where the vertex went + return std::abs( was[ 1 ] - local[ To ][ 1 ] ); }; + + auto const allowed = [ & ]( std::uint32_t const From, std::uint32_t const To ) { + if( ( true == removed[ From ] ) || ( true == removed[ To ] ) ) { return false; } + // a vertex a neighbouring tile shares stays where it is + if( true == vertices[ From ].locked() ) { return false; } + // and a material boundary stays where it is too, or the ground would change colour by + // level + if( vertices[ From ].material != vertices[ To ].material ) { return false; } + // nothing that turns a triangle over or squashes it flat + for( auto const triangle : incident[ From ] ) { + if( true == gone[ triangle ] ) { continue; } + std::array corners { + cornerof( triangle, 0 ), cornerof( triangle, 1 ), cornerof( triangle, 2 ) }; + if( std::find( corners.begin(), corners.end(), To ) != corners.end() ) { + // this triangle goes when the edge does. it may not be the only one holding a + // piece of the tile's border up: From is not on a border, so the only edge of it + // that can be one is between To and the third corner + for( auto const third : corners ) { + if( ( third == From ) || ( third == To ) ) { continue; } + if( ( vertices[ To ].sides & vertices[ third ].sides ) != 0 ) { return false; } + } + continue; + } + std::array const *, 3> before { + &local[ corners[ 0 ] ], &local[ corners[ 1 ] ], &local[ corners[ 2 ] ] }; + std::array const *, 3> after { before }; + for( std::size_t corner = 0; corner < 3; ++corner ) { + if( corners[ corner ] == From ) { after[ corner ] = &local[ To ]; } + } + // the normal in space, so that this holds for a wall as well as for open ground + auto const normal = []( std::array const *, 3> const &Corners ) { + auto const &a { *Corners[ 0 ] }; auto const &b { *Corners[ 1 ] }; auto const &c { *Corners[ 2 ] }; + return std::array { + ( b[ 1 ] - a[ 1 ] ) * ( c[ 2 ] - a[ 2 ] ) - ( b[ 2 ] - a[ 2 ] ) * ( c[ 1 ] - a[ 1 ] ), + ( b[ 2 ] - a[ 2 ] ) * ( c[ 0 ] - a[ 0 ] ) - ( b[ 0 ] - a[ 0 ] ) * ( c[ 2 ] - a[ 2 ] ), + ( b[ 0 ] - a[ 0 ] ) * ( c[ 1 ] - a[ 1 ] ) - ( b[ 1 ] - a[ 1 ] ) * ( c[ 0 ] - a[ 0 ] ) }; }; + auto const was { normal( before ) }; + auto const now { normal( after ) }; + auto const length { + std::sqrt( now[ 0 ] * now[ 0 ] + now[ 1 ] * now[ 1 ] + now[ 2 ] * now[ 2 ] ) }; + // nothing that squashes a triangle flat, and nothing that turns one over + if( 0.5 * length < nullarea ) { return false; } + if( was[ 0 ] * now[ 0 ] + was[ 1 ] * now[ 1 ] + was[ 2 ] * now[ 2 ] <= 0.0 ) { return false; } + } + // and nothing that moves the ground further than this level is allowed to + if( drop_of( From, To ) > Allowed ) { return false; } + return true; }; + + auto const offer = [ & ]( std::uint32_t const Vertex ) { + for( auto const triangle : incident[ Vertex ] ) { + if( true == gone[ triangle ] ) { continue; } + for( std::size_t corner = 0; corner < 3; ++corner ) { + auto const other { cornerof( triangle, corner ) }; + if( other == Vertex ) { continue; } + for( auto const [ from, to ] : { std::pair { Vertex, other }, std::pair { other, Vertex } } ) { + if( false == allowed( from, to ) ) { continue; } + queue.push( { cost_of( from, to ), from, to, version[ from ], version[ to ] } ); + } + } + } }; + + for( std::uint32_t vertex = 0; vertex < vertices.size(); ++vertex ) { offer( vertex ); } + + while( ( triangles > Budget ) && ( false == queue.empty() ) ) { + + auto const best { queue.top() }; + queue.pop(); + if( ( version[ best.from ] != best.fromversion ) || ( version[ best.to ] != best.toversion ) ) { + continue; // one of the two has moved on since this was worked out + } + if( false == allowed( best.from, best.to ) ) { continue; } + + auto const drop { drop_of( best.from, best.to ) }; + moved[ best.to ] = std::max( { moved[ best.to ], moved[ best.from ], drop } ); + Mesh.error = std::max( Mesh.error, drop ); + + removed[ best.from ] = true; + quadrics[ best.to ] += quadrics[ best.from ]; + + for( auto const triangle : incident[ best.from ] ) { + if( true == gone[ triangle ] ) { continue; } + auto wasthere { false }; + for( std::size_t corner = 0; corner < 3; ++corner ) { + if( cornerof( triangle, corner ) == best.to ) { wasthere = true; } + } + if( true == wasthere ) { + // the triangle had both ends of the edge: it has no area left + gone[ triangle ] = true; + --triangles; + continue; + } + for( std::size_t corner = 0; corner < 3; ++corner ) { + if( cornerof( triangle, corner ) == best.from ) { cornerof( triangle, corner ) = best.to; } + } + incident[ best.to ].push_back( triangle ); + } + incident[ best.from ].clear(); + ++version[ best.from ]; + ++version[ best.to ]; + + offer( best.to ); + } + + // drop what went, and close up the numbering + std::vector indices; + indices.reserve( triangles * 3 ); + for( std::size_t triangle = 0; triangle < gone.size(); ++triangle ) { + if( true == gone[ triangle ] ) { continue; } + for( std::size_t corner = 0; corner < 3; ++corner ) { + indices.push_back( cornerof( triangle, corner ) ); + } + } + std::vector renumber( vertices.size(), std::numeric_limits::max() ); + std::vector kept; + kept.reserve( vertices.size() ); + for( auto &index : indices ) { + if( renumber[ index ] == std::numeric_limits::max() ) { + renumber[ index ] = static_cast( kept.size() ); + kept.push_back( vertices[ index ] ); + } + index = renumber[ index ]; + } + Mesh.vertices.swap( kept ); + Mesh.indices.swap( indices ); +} + +namespace { + +template +void +append( std::vector &Out, Type_ const &Value ) { + auto const *bytes { reinterpret_cast( &Value ) }; + Out.insert( Out.end(), bytes, bytes + sizeof( Type_ ) ); +} + +template +void +append_indices( std::vector &Out, std::vector const &Indices ) { + std::vector narrow( Indices.begin(), Indices.end() ); + std::vector codes; + highwater_encode( narrow, codes ); + for( auto const code : codes ) { append( Out, code ); } +} + +template +void +append_edge( std::vector &Out, std::vector const &Edge ) { + append( Out, static_cast( Edge.size() ) ); + for( auto const index : Edge ) { append( Out, static_cast( index ) ); } +} + +} // anonymous namespace + +// The layout is the format's: the header, then the vertices as zigzag deltas of their quantized +// coordinates, then the triangles high-water-mark encoded, then the four lists of border +// vertices, then extensions. Texture coordinates and materials are ours, and ride in one. +void +cooker::encode( tile_address const &Tile, mesh const &Mesh, std::vector &Out ) const { + + double west { 0.0 }, north { 0.0 }, side { 0.0 }; + square( Tile, west, north, side ); + + // The format's indices are coded against the highest one used so far, which only works when a + // vertex is numbered the first time a triangle reaches for it. So the vertices are put in the + // order the triangles ask for them; anything no triangle asks for is left out. + std::vector order; + std::vector indices; + { + constexpr std::uint32_t unplaced { std::numeric_limits::max() }; + std::vector placed( Mesh.vertices.size(), unplaced ); + order.reserve( Mesh.vertices.size() ); + indices.reserve( Mesh.indices.size() ); + for( auto const index : Mesh.indices ) { + if( placed[ index ] == unplaced ) { + placed[ index ] = static_cast( order.size() ); + order.push_back( index ); + } + indices.push_back( placed[ index ] ); + } + } + + auto lowest { Mesh.vertices[ order.front() ].y }, highest { lowest }; + for( auto const index : order ) { + lowest = std::min( lowest, Mesh.vertices[ index ].y ); + highest = std::max( highest, Mesh.vertices[ index ].y ); + } + // heights are quantized against this tile's own range, as the format says. two tiles sharing + // a border therefore round a shared vertex a fraction of a millimetre differently, which is + // what the format's skirts are for elsewhere and is below anything visible here + auto const heightspan { std::max( 1e-3, highest - lowest ) }; + + file_header header {}; + header.centre_x = west + side * 0.5; + header.centre_y = ( lowest + highest ) * 0.5; + header.centre_z = north + side * 0.5; + header.lowest = static_cast( lowest ); + header.highest = static_cast( highest ); + header.sphere_x = header.centre_x; + header.sphere_y = header.centre_y; + header.sphere_z = header.centre_z; + header.sphere_radius = std::sqrt( 0.5 * side * side + 0.25 * heightspan * heightspan ); + header.horizon_x = header.horizon_y = header.horizon_z = 0.0; + + Out.clear(); + Out.reserve( order.size() * 8 + indices.size() * 2 + sizeof( header ) + 64 ); + append( Out, header ); + append( Out, static_cast( order.size() ) ); + + std::vector quantx( order.size() ); + std::vector quanty( order.size() ); + std::vector quantz( order.size() ); + for( std::size_t index = 0; index < order.size(); ++index ) { + auto const &vertex { Mesh.vertices[ order[ index ] ] }; + quantx[ index ] = quantize( vertex.x, west, side ); + quanty[ index ] = quantize( vertex.y, lowest, heightspan ); + quantz[ index ] = quantize( vertex.z, north, side ); + } + // the deltas between one vertex and the next, so the run is small numbers + auto const deltas = [ &Out ]( std::vector const &Values ) { + std::int32_t previous { 0 }; + for( auto const value : Values ) { + append( Out, zigzag_encode( static_cast( value ) - previous ) ); + previous = static_cast( value ); + } }; + deltas( quantx ); + deltas( quantz ); + deltas( quanty ); + + auto const wide { order.size() > 65536 }; + auto const indexsize { wide ? 4u : 2u }; + // the format wants the indices to start on a boundary of their own size + while( ( Out.size() % indexsize ) != 0 ) { Out.push_back( 0 ); } + + append( Out, static_cast( indices.size() / 3 ) ); + if( true == wide ) { append_indices( Out, indices ); } + else { append_indices( Out, indices ); } + + // the border vertices, which the format expects to be told about by the quantized coordinate + // they share + std::vector edges[ 4 ]; + for( std::size_t index = 0; index < order.size(); ++index ) { + if( quantx[ index ] == 0 ) { edges[ 0 ].push_back( static_cast( index ) ); } + if( quantz[ index ] == 0 ) { edges[ 1 ].push_back( static_cast( index ) ); } + if( quantx[ index ] == quantum_max ) { edges[ 2 ].push_back( static_cast( index ) ); } + if( quantz[ index ] == quantum_max ) { edges[ 3 ].push_back( static_cast( index ) ); } + } + for( auto const &edge : edges ) { + if( true == wide ) { append_edge( Out, edge ); } + else { append_edge( Out, edge ); } + } + + // ours: which ground material each vertex belongs to. the format has nowhere for it + append( Out, static_cast( ext_ground ) ); + append( Out, static_cast( order.size() * sizeof( std::uint16_t ) ) ); + for( auto const index : order ) { + append( Out, Mesh.vertices[ index ].material ); + } +} + +} // namespace quantizedmesh diff --git a/scene/quantizedmeshcooker.h b/scene/quantizedmeshcooker.h new file mode 100644 index 00000000..c5064243 --- /dev/null +++ b/scene/quantizedmeshcooker.h @@ -0,0 +1,176 @@ +/* +This Source Code Form is subject to the +terms of the Mozilla Public License, v. +2.0. If a copy of the MPL was not +distributed with this file, You can +obtain one at +http://mozilla.org/MPL/2.0/. +*/ + +#pragma once + +// Cuts a scenery's ground into Cesium Quantized Mesh tiles. +// +// The source is already a mesh, so this does not triangulate anything: it cuts the triangles it +// is given along the tile grid, welds what coincides, and builds each coarser level by taking +// the four tiles below it and collapsing edges until the tile costs what one tile should. What +// it never does is invent ground - a hole in the scenery stays a hole - or move a vertex that +// sits on a tile's border, which is what lets neighbours drawn at different levels meet. +// +// Triangles standing upright have no ground in them, and are refused rather than laid: they are +// walls, and the scenery goes on drawing them itself. refusals() says which. +// +// Shared by the engine's first-run bake and the offline tool, so the two cannot drift apart. +// Millions of triangles do not fit in memory, so what is added is spilled to a scratch file and +// read back a tile at a time. + +#include +#include +#include +#include +#include +#include +#include + +#include "scene/quantizedmeshformat.h" + +namespace quantizedmesh { + +class cooker { + +public: + // a corner of a source triangle, in world space, with the texture coordinates the scenery + // gave it + struct corner { + double x, y, z; + float u, v; + }; + + // one finished tile, handed over as the bytes of its file + using sink = std::function const & )>; + + cooker() = default; + ~cooker(); + cooker( cooker const & ) = delete; + cooker &operator=( cooker const & ) = delete; + + // Tilebudget is the most triangles a tile of any level may hold. The side of a tile at the + // finest level is not given: it is worked out from how large the triangles turn out to be, since + // a tile much smaller than them would spend most of its triangles on the cuts along its own + // borders. Opens the scratch file; returns false when it cannot be written + bool begin( std::string const &Scratchpath, std::size_t Tilebudget = 8192 ); + // the side of a tile at the finest level, once finish() has worked it out + double finesttile() const { return m_finesttile; } + // stops talking to standard output, for the engine + void quiet() { m_quiet = true; } + + // lays one triangle of ground. the id counts from zero in the order they are added, and is + // what refusals() speaks of + void add( corner const &A, corner const &B, corner const &C, std::string_view Material ); + std::size_t added() const { return m_added; } + + // cuts, welds, simplifies and hands every tile to the sink, then fills the table. the + // scratch file is read several times over and removed afterwards + bool finish( sink const &Sink, terrain_table &Table ); + + // triangles that carry no ground, by the id add() gave them, in ascending order. only + // meaningful once finish() has run + std::vector const &refusals() const { return m_refusals; } + + // what the cook did, for the log + struct report { + std::size_t triangles { 0 }; // laid, after cutting along the grid + std::size_t refused { 0 }; + std::size_t tiles { 0 }; + std::size_t vertices { 0 }; + std::size_t bytes { 0 }; + std::uint32_t levels { 1 }; + }; + report const &result() const { return m_report; } + + // a vertex of a tile's mesh while it is being worked on, in metres + struct meshvertex { + double x, y, z; + std::uint16_t material; + // which of the tile's four sides it sits on, one bit each. a vertex on a side is shared + // with the tile beyond it, so it survives into every level; and an edge whose two ends + // share a side is a piece of the border itself, and survives too + std::uint8_t sides { 0 }; + bool locked() const { return sides != 0; } + }; + + struct mesh { + std::vector vertices; + std::vector indices; + double error { 0.0 }; // metres this mesh departs from the finest one + bool empty() const { return indices.empty(); } + }; + +private: + // a triangle as it waits in the scratch file. fixed size, so a tile's triangles can be read + // back by their place in it. only the corners' positions matter by then: the texture + // coordinates have already been turned into the material's tiling + struct spilled { + double corners[ 9 ]; + std::uint16_t material; + std::uint16_t padding[ 3 ]; + }; + + std::uint16_t material_index( std::string_view Material ); + // how many metres of ground one repeat of a material covers, from the texture coordinates + // the scenery drew it with: the ratio of world distance to texture distance along an edge + void measure_repeat( std::uint16_t Material, corner const ( &Corners )[ 3 ] ); + // the square the root tile covers, from what was added + void measure(); + // cooks a tile and everything under it, depth first, and hands each to the sink. returns the + // tile's own mesh, which its parent merges + mesh cook( tile_address const &Tile, sink const &Sink ); + // the finest level: the triangles of the scratch file that fall on this tile, cut to it + mesh lay( tile_address const &Tile ); + // the four tiles below this one, welded into one mesh + mesh merge( tile_address const &Tile, sink const &Sink ); + // collapses edges until the mesh fits the budget, or until the ground would move further than + // Allowed, whichever comes first. locked vertices are left where they are + void simplify( mesh &Mesh, std::size_t Budget, double Allowed ); + // the bytes of a tile's file + void encode( tile_address const &Tile, mesh const &Mesh, std::vector &Out ) const; + // where a tile's square lies + void square( tile_address const &Tile, double &West, double &North, double &Side ) const; + + std::string m_scratchpath; + std::fstream m_scratch; + std::size_t m_added { 0 }; + bool m_quiet { false }; + + double m_finesttile { 256.0 }; + std::size_t m_tilebudget { 8192 }; + // a sample of how long a triangle's edges are in the plan, which decides the finest tile + std::vector m_edges; + + std::vector m_materials; + std::unordered_map m_materiallookup; + std::vector m_repeatsums; + std::vector m_repeatsquares; + std::vector m_repeatcounts; + + // extent and height range of everything added + double m_minx { 0.0 }, m_maxx { 0.0 }; + double m_minz { 0.0 }, m_maxz { 0.0 }; + double m_lowest { 0.0 }, m_highest { 0.0 }; + bool m_anything { false }; + + // the root square and the depth under it + double m_west { 0.0 }, m_north { 0.0 }, m_side { 0.0 }; + std::uint32_t m_levels { 1 }; + + // which triangles of the scratch file touch which finest tile, sorted by tile. built once, + // walked as the finest level is laid + std::vector> m_placement; + + std::vector m_refusals; + // the worst error any tile of each level came out with + std::vector m_errors; + report m_report; +}; + +} // namespace quantizedmesh diff --git a/scene/quantizedmeshformat.h b/scene/quantizedmeshformat.h index 71baca31..94268f26 100644 --- a/scene/quantizedmeshformat.h +++ b/scene/quantizedmeshformat.h @@ -9,155 +9,191 @@ http://mozilla.org/MPL/2.0/. #pragma once -// Cesium Quantized Mesh format - TIN-based terrain tiles with LOD. -// Replaces regular heightfield with adaptive triangulation. +// Cesium Quantized Mesh, as the terrain tiles are written. +// https://github.com/CesiumGS/quantized-mesh // -// Format: https://github.com/CesiumGS/quantized-mesh -// -// File layout: -// header (88 bytes) -// vertex data (positions, uvs, indices) -// edge indices (for seamless LOD stitching) -// extensions (optional metadata) +// The ground in a MaSzyna scenery is already an irregular mesh: fine along the track where it +// came from a survey, coarse in the open where it was drawn by hand. The cook keeps that mesh +// rather than resampling it - it cuts it into tiles, welds the vertices and builds coarser +// levels of it - so there is nothing to interpolate, nothing to fill in where the ground has +// holes, and no grid step to pick. // -// Advantages over heightfield: -// - Variable density: fine detail where needed (track corridor), coarse elsewhere -// - ~50% fewer vertices for same visual quality on mixed-detail terrain -// - Built-in LOD with proper edge constraints (no T-junctions) -// - Industry-standard format, tooling available +// Tiles form a quadtree, as the format intends: one tile at the root covers the whole terrain, +// and each level splits every tile into four. A distant tile is therefore a large one, and the +// horizon costs tens of tiles rather than thousands. Every vertex on a tile's border survives +// into all of its coarser levels, so two neighbours drawn at different levels share their +// border exactly, and nothing has to be stitched or morphed at draw time. // -// Disadvantages: -// - More complex to generate (requires constrained triangulation) -// - Slightly larger per-tile overhead (index buffer) +// Where this departs from the specification, and why: +// +// - the header's fields are geodetic (ECEF centre, bounding sphere, horizon occlusion point). +// A scenery has a local metric frame and no geographic reference to convert from, so those +// fields carry scenery metres. The geometry reads back correctly anywhere; a Cesium client +// would place the tile wrongly on the globe, which is a trade for something we do not want. +// - the format has no texture coordinates and no materials: its u and v are the vertex's +// horizontal position within the tile. A vertex of ours carries which ground material it +// belongs to, in extension ext_ground below, which is what the extension mechanism is for. +// Texture coordinates are not stored at all: ground textures tile in world space, so the +// coordinate follows from where the vertex is and how many metres one repeat of its material +// covers, which is measured from the scenery once and kept in the terrain's table. +// - tiles are stored inside the scenery's archive, one zstd-compressed entry each, in place of +// the gzip the format expects from an HTTP layer. The bytes of a tile are the format's. +#include #include -#include +#include +#include +#include namespace quantizedmesh { -// "EU07QMSH" - distinguishes from other formats -inline constexpr char magic[ 8 ] { 'E', 'U', '0', '7', 'Q', 'M', 'S', 'H' }; -inline constexpr std::uint32_t version { 1 }; +// u, v and height are quantized to this, not to 65535: the format says so +inline constexpr std::uint16_t quantum_max { 32767 }; + +// what the cook counted as ground, and how it cut and simplified it. tiles cooked under other +// rules are baked again rather than read +inline constexpr std::uint32_t cook_rules { 2 }; #pragma pack( push, 1 ) +// the format's header, unchanged. 88 bytes struct file_header { - char magic[ 8 ]; - std::uint32_t version; - - // Tile bounds in world space (metres) - double center_x, center_y, center_z; // tile centre - double min_x, min_y, min_z; // bounding box min - double max_x, max_y, max_z; // bounding box max - - // Quantization ranges - vertices stored as uint16, decoded to this range - double horizont_error; // maximum geometric error for this LOD - - std::uint32_t vertex_count; - std::uint32_t triangle_count; - - // Offsets into file (all after this header) - std::uint32_t vertex_data_offset; // packed uint16[3] positions - std::uint32_t uv_data_offset; // packed uint16[2] texture coords - std::uint32_t index_data_offset; // uint16 indices (or uint32 if >65535 verts) - std::uint32_t edge_indices_offset; // seamless stitching data - std::uint32_t extension_offset; // optional metadata - - std::uint16_t index_size; // 2 (uint16) or 4 (uint32) - std::uint16_t edge_count_north; - std::uint16_t edge_count_south; - std::uint16_t edge_count_west; - std::uint16_t edge_count_east; - std::uint16_t reserved; // padding for future use + double centre_x, centre_y, centre_z; + float lowest, highest; // the tile's height range, in metres + double sphere_x, sphere_y, sphere_z; // bounding sphere, for culling + double sphere_radius; + double horizon_x, horizon_y, horizon_z; // horizon occlusion point; unused here }; -static_assert( sizeof( file_header ) == 132 ); // Actual size with current layout - -// Edge indices for seamless LOD transitions -// Each edge stores indices of vertices lying on that tile boundary, in order -// Parent LOD uses these to stitch to child tiles without gaps -struct edge_indices { - std::uint16_t count; - std::uint16_t indices[]; // variable length -}; - -// Optional extensions (material splat, metadata, etc.) -struct extension_header { - std::uint32_t extension_id; - std::uint32_t extension_size; -}; - -// Extension IDs -enum extension_id : std::uint32_t { - ext_material_splat = 1, // RGBA8 per-vertex material weights - ext_metadata = 2, // JSON metadata - ext_meshlet = 3, // Meshlet data for mesh shading -}; +static_assert( sizeof( file_header ) == 88 ); #pragma pack( pop ) -// Helper functions +// extension ids. 1, 2 and 4 are taken by the format for normals, the water mask and metadata +enum extension_id : std::uint8_t { + ext_normals = 1, + ext_watermask = 2, + ext_metadata = 4, + // ours: one uint16 per vertex, its material's place in the terrain's table + ext_ground = 64, +}; -// Quantize position to uint16 -inline std::uint16_t quantize( double value, double min, double max ) { - if( max <= min ) return 0; - double const normalized = std::clamp( ( value - min ) / ( max - min ), 0.0, 1.0 ); - return static_cast( normalized * 65535.0 + 0.5 ); +// zigzag, as the format uses it on the deltas between consecutive vertices +inline std::uint16_t +zigzag_encode( std::int32_t const Value ) { + return static_cast( ( Value << 1 ) ^ ( Value >> 31 ) ); } -// Dequantize uint16 to position -inline double dequantize( std::uint16_t encoded, double min, double max ) { - double const normalized = static_cast( encoded ) / 65535.0; - return min + normalized * ( max - min ); +inline std::int32_t +zigzag_decode( std::uint16_t const Stored ) { + return ( Stored >> 1 ) ^ -static_cast( Stored & 1 ); } -// ZigZag encode for delta-compression of indices -inline std::uint32_t zigzag_encode( std::int32_t value ) { - return ( value << 1 ) ^ ( value >> 31 ); +// a quantized coordinate back to metres, against the range it was quantized in +inline double +dequantize( std::uint16_t const Stored, double const Low, double const Span ) { + return Low + ( static_cast( Stored ) / quantum_max ) * Span; } -inline std::int32_t zigzag_decode( std::uint32_t value ) { - return ( value >> 1 ) ^ -static_cast( value & 1 ); +inline std::uint16_t +quantize( double const Value, double const Low, double const Span ) { + if( Span <= 0.0 ) { return 0; } + auto const share { std::clamp( ( Value - Low ) / Span, 0.0, 1.0 ) }; + return static_cast( share * quantum_max + 0.5 ); } -// Octahedral encoding for normals (optional, saves space) -inline void encode_oct( float nx, float ny, float nz, std::int16_t &out_x, std::int16_t &out_y ) { - float const l1norm = std::abs(nx) + std::abs(ny) + std::abs(nz); - float px = nx / l1norm; - float py = ny / l1norm; - - if( nz < 0.0f ) { - float const old_x = px; - px = ( 1.0f - std::abs(py) ) * ( old_x >= 0.0f ? 1.0f : -1.0f ); - py = ( 1.0f - std::abs(old_x) ) * ( py >= 0.0f ? 1.0f : -1.0f ); +// Indices are stored high-water-mark encoded: a code of zero means the next index never used +// before, and anything else counts back from the highest index used so far. The run of codes is +// small numbers whatever the mesh looks like, which is what makes the tile compress. +template +void +highwater_encode( std::vector const &Indices, std::vector &Out ) { + + Out.clear(); + Out.reserve( Indices.size() ); + std::int64_t highest { 0 }; + for( auto const index : Indices ) { + Out.push_back( static_cast( highest - static_cast( index ) ) ); + if( static_cast( index ) == highest ) { ++highest; } } - - out_x = static_cast( std::clamp( px * 32767.0f, -32767.0f, 32767.0f ) ); - out_y = static_cast( std::clamp( py * 32767.0f, -32767.0f, 32767.0f ) ); } -inline void decode_oct( std::int16_t enc_x, std::int16_t enc_y, float &nx, float &ny, float &nz ) { - float px = static_cast( enc_x ) / 32767.0f; - float py = static_cast( enc_y ) / 32767.0f; - - nz = 1.0f - std::abs(px) - std::abs(py); - - if( nz < 0.0f ) { - float const old_x = px; - px = ( 1.0f - std::abs(py) ) * ( old_x >= 0.0f ? 1.0f : -1.0f ); - py = ( 1.0f - std::abs(old_x) ) * ( py >= 0.0f ? 1.0f : -1.0f ); +// returns false when a code points past what has been used, which a corrupted tile can do +template +bool +highwater_decode( std::vector const &Codes, std::vector &Out ) { + + Out.clear(); + Out.reserve( Codes.size() ); + std::int64_t highest { 0 }; + for( auto const code : Codes ) { + auto const index { highest - static_cast( code ) }; + if( ( index < 0 ) || ( index > highest ) ) { return false; } + Out.push_back( static_cast( index ) ); + if( index == highest ) { ++highest; } } - - float const len = std::sqrt( px*px + py*py + nz*nz ); - nx = px / len; - ny = py / len; - nz = nz / len; + return true; } -// Tile key - same as heightfield for compatibility +// A tile's place in the quadtree. Level 0 is the root, covering the whole terrain; each level +// splits every tile into four. X grows east, Z grows with the scenery's z, so a tile's square +// follows from the terrain's extent and the level alone. +struct tile_address { + std::uint32_t level { 0 }; + std::int32_t x { 0 }, z { 0 }; + + bool operator==( tile_address const & ) const = default; +}; + constexpr std::int64_t -tile_key( std::int32_t const X, std::int32_t const Z ) { - return ( static_cast( X ) << 32 ) ^ static_cast( Z ); +tile_key( tile_address const &Tile ) { + // a level fits in six bits for any terrain worth cooking, and the coordinates cannot exceed + // the level, so this is unique + return ( static_cast( Tile.level ) << 58 ) + ^ ( static_cast( Tile.x ) << 29 ) + ^ static_cast( Tile.z ); } +// where a tile's file sits in the archive, following the format's own layout of level, column +// and row +inline std::string +tile_name( tile_address const &Tile ) { + return std::to_string( Tile.level ) + "/" + std::to_string( Tile.x ) + "/" + + std::to_string( Tile.z ) + ".terrain"; +} + +// the terrain's extent, height range, depth and material table are the same for every tile, so +// they are written once beside them, under this name. the format keeps the same thing in +// layer.json +inline constexpr char const *tablename { "terrain.table" }; + +// what that file says +struct terrain_table { + std::uint32_t rules { 0 }; + // the square the root tile covers. always square, so that a tile at any level is too + double west { 0.0 }, north { 0.0 }, side { 0.0 }; + double lowest { 0.0 }, highest { 0.0 }; + // levels cooked, root included: addresses run from 0 to levels - 1 + std::uint32_t levels { 1 }; + // metres the ground of a tile at each level departs from the finest one. what decides which + // level to draw: a level is good enough while its error covers few enough pixels + std::vector errors; + // one ground texture: its name, and how many metres of ground one repeat of it covers, as + // measured from the texture coordinates the scenery drew it with + struct material { + std::string name; + float repeat { 4.f }; + }; + // in the order the vertices' material indices count + std::vector materials; + + // metres along the side of a tile at this level + double tilesize( std::uint32_t const Level ) const { + return side / static_cast( 1u << Level ); + } + float error( std::uint32_t const Level ) const { + return Level < errors.size() ? errors[ Level ] : 0.f; + } +}; + } // namespace quantizedmesh diff --git a/scene/quantizedmeshreader.h b/scene/quantizedmeshreader.h index b65bdf02..177e6e59 100644 --- a/scene/quantizedmeshreader.h +++ b/scene/quantizedmeshreader.h @@ -9,215 +9,369 @@ http://mozilla.org/MPL/2.0/. #pragma once -// Reads Cesium Quantized Mesh tiles - replaces heightfield reader entirely. -// Used by terrain_tile_loader to stream TIN tiles from disk. +// Reads cooked terrain tiles back, from a scenery's archive or from a directory of loose files. +// A directory is what a tile cooked by hand or by another tool arrives as; the archive is what +// the bake writes. +// +// Everything a tile says about itself is checked before it is believed: a truncated or mangled +// tile is refused rather than handed on, because what comes out of here goes straight into a +// vertex buffer and an index buffer. #include #include #include +#include +#include #include +#include #include -#include +#include "scene/pakformat.h" #include "scene/quantizedmeshformat.h" namespace quantizedmesh { -// Reader for cooked TIN terrain tiles (.qm files) -// Single-threaded, meant to be used from terrain_tile_loader worker thread +// one vertex as the renderer wants it: measured from the terrain's origin, so that the whole +// terrain fits in floats and every tile of it can go into one vertex buffer. there are no texture +// coordinates - the ground tiles in world space, so the shader works them out from the position +#pragma pack( push, 1 ) +struct render_vertex { + float x, y, z; + std::uint16_t material; + std::uint16_t padding; +}; +#pragma pack( pop ) + +static_assert( sizeof( render_vertex ) == 16 ); + class reader { + public: + struct tile_data { + tile_address tile; + std::vector vertices; + std::vector indices; + // what the vertices are measured from, in scenery metres + double originx { 0.0 }, originy { 0.0 }, originz { 0.0 }; + // for culling, in scenery metres + double centrex { 0.0 }, centrey { 0.0 }, centrez { 0.0 }; + double radius { 0.0 }; + float lowest { 0.f }, highest { 0.f }; + }; + reader() = default; ~reader() { close(); } - reader( reader const & ) = delete; reader &operator=( reader const & ) = delete; - - // Open tile directory (contains terrain_X_Z_lodN.qm files) - bool open( std::string const &Directory ); + + // an archive or a directory. reads the terrain's table, without which nothing can be placed + bool open( std::string const &Path ); void close(); - - // Read single LOD of a tile - struct tile_data { - // Dequantized positions - std::vector positions_x; - std::vector positions_y; - std::vector positions_z; - - // Texture coordinates - std::vector uvs_u; - std::vector uvs_v; - - // Triangle indices - std::vector indices; - - // Material per vertex (index into material table) - std::vector materials; - - // Edge indices for stitching - std::vector north_edge; - std::vector south_edge; - std::vector west_edge; - std::vector east_edge; - - // Tile bounds (for positioning) - double center_x, center_y, center_z; - double min_x, min_y, min_z; - double max_x, max_y, max_z; - - double geometric_error; - }; - - bool read_tile( std::int32_t X, std::int32_t Z, std::uint32_t LOD, tile_data &Out ); - - // Check if tile exists - bool contains( std::int32_t X, std::int32_t Z, std::uint32_t LOD ) const; - - std::string const &directory() const { return m_directory; } - + bool ready() const { return m_ready; } + + terrain_table const &table() const { return m_table; } + std::string const &path() const { return m_path; } + + // Origin is what the vertices come back measured from: one point for the whole terrain, so + // that every tile can go into the same vertex buffer and be drawn in one call + bool read_tile( tile_address const &Tile, tile_data &Out, + double Originx, double Originy, double Originz ); + // the tiles the source holds, by tile_key. a tile the cook found nothing for is not there + std::unordered_set const &present() const { return m_present; } + bool contains( tile_address const &Tile ) const { return m_present.contains( tile_key( Tile ) ); } + private: - std::string tile_path( std::int32_t X, std::int32_t Z, std::uint32_t LOD ) const; - - std::string m_directory; + bool entry( std::string const &Name, std::vector &Out ); + bool read_table(); + void find_tiles(); + + std::string m_path; + bool m_archive { false }; + pak::reader m_pak; + terrain_table m_table; + std::unordered_set m_present; + bool m_ready { false }; + std::vector m_buffer; }; -// Implementation +// Implementation. + +inline bool +reader::open( std::string const &Path ) { -inline bool reader::open( std::string const &Directory ) { close(); - m_directory = Directory; - return true; // Lazy - tiles opened on demand -} - -inline void reader::close() { - m_directory.clear(); -} - -inline std::string reader::tile_path( std::int32_t X, std::int32_t Z, std::uint32_t LOD ) const { - char filename[ 256 ]; - std::snprintf( filename, sizeof( filename ), "terrain_%d_%d_lod%u.qm", X, Z, LOD ); - return m_directory + "/" + filename; -} - -inline bool reader::contains( std::int32_t X, std::int32_t Z, std::uint32_t LOD ) const { - auto const path = tile_path( X, Z, LOD ); - FILE *f = std::fopen( path.c_str(), "rb" ); - if( !f ) return false; - std::fclose( f ); + m_path = Path; + std::error_code error; + if( true == std::filesystem::is_directory( Path, error ) ) { + m_archive = false; + } + else if( true == std::filesystem::is_regular_file( Path, error ) ) { + m_archive = true; + if( false == m_pak.open( Path ) ) { close(); return false; } + } + else { + close(); + return false; + } + if( false == read_table() ) { close(); return false; } + find_tiles(); + m_ready = true; return true; } -inline bool reader::read_tile( std::int32_t X, std::int32_t Z, std::uint32_t LOD, tile_data &Out ) { - auto const path = tile_path( X, Z, LOD ); - - FILE *file = std::fopen( path.c_str(), "rb" ); - if( !file ) return false; - - // Read header - file_header header; - if( std::fread( &header, sizeof( header ), 1, file ) != 1 ) { - std::fclose( file ); - return false; +inline void +reader::close() { + + m_pak.close(); + m_table = {}; + m_present.clear(); + m_path.clear(); + m_archive = false; + m_ready = false; +} + +inline bool +reader::entry( std::string const &Name, std::vector &Out ) { + + if( true == m_archive ) { return m_pak.read( Name, Out ); } + + std::ifstream file( std::filesystem::path( m_path ) / Name, std::ios::binary ); + if( false == file.good() ) { return false; } + file.seekg( 0, std::ios::end ); + auto const size { static_cast( file.tellg() ) }; + file.seekg( 0, std::ios::beg ); + Out.resize( size ); + if( size > 0 ) { + file.read( reinterpret_cast( Out.data() ), static_cast( size ) ); } - - // Verify magic - if( std::memcmp( header.magic, magic, sizeof( magic ) ) != 0 ) { - std::fclose( file ); - return false; - } - - if( header.version != version ) { - std::fclose( file ); - return false; - } - - // Store bounds - Out.center_x = header.center_x; - Out.center_y = header.center_y; - Out.center_z = header.center_z; - Out.min_x = header.min_x; - Out.min_y = header.min_y; - Out.min_z = header.min_z; - Out.max_x = header.max_x; - Out.max_y = header.max_y; - Out.max_z = header.max_z; - Out.geometric_error = header.horizont_error; - - // Read quantized positions - std::fseek( file, header.vertex_data_offset, SEEK_SET ); - - std::vector quantized_x( header.vertex_count ); - std::vector quantized_y( header.vertex_count ); - std::vector quantized_z( header.vertex_count ); - - for( std::uint32_t i = 0; i < header.vertex_count; ++i ) { - std::fread( &quantized_x[ i ], sizeof( std::uint16_t ), 1, file ); - std::fread( &quantized_y[ i ], sizeof( std::uint16_t ), 1, file ); - std::fread( &quantized_z[ i ], sizeof( std::uint16_t ), 1, file ); - } - - // Dequantize positions - Out.positions_x.resize( header.vertex_count ); - Out.positions_y.resize( header.vertex_count ); - Out.positions_z.resize( header.vertex_count ); - - for( std::uint32_t i = 0; i < header.vertex_count; ++i ) { - Out.positions_x[ i ] = dequantize( quantized_x[ i ], header.min_x, header.max_x ); - Out.positions_y[ i ] = dequantize( quantized_y[ i ], header.min_y, header.max_y ); - Out.positions_z[ i ] = dequantize( quantized_z[ i ], header.min_z, header.max_z ); - } - - // Read UVs - std::fseek( file, header.uv_data_offset, SEEK_SET ); - - Out.uvs_u.resize( header.vertex_count ); - Out.uvs_v.resize( header.vertex_count ); - - for( std::uint32_t i = 0; i < header.vertex_count; ++i ) { - std::uint16_t u, v; - std::fread( &u, sizeof( std::uint16_t ), 1, file ); - std::fread( &v, sizeof( std::uint16_t ), 1, file ); - Out.uvs_u[ i ] = static_cast( dequantize( u, 0.0, 1.0 ) ); - Out.uvs_v[ i ] = static_cast( dequantize( v, 0.0, 1.0 ) ); - } - - // Read indices - std::fseek( file, header.index_data_offset, SEEK_SET ); - - Out.indices.resize( header.triangle_count * 3 ); - - if( header.index_size == 2 ) { - for( std::uint32_t i = 0; i < header.triangle_count * 3; ++i ) { - std::uint16_t idx; - std::fread( &idx, sizeof( std::uint16_t ), 1, file ); - Out.indices[ i ] = idx; + return static_cast( file.gcount() ) == size; +} + +// the table is text: a terrain outlives the build that cooked it, and a line of it should be +// readable without a tool +inline bool +reader::read_table() { + + std::vector bytes; + if( false == entry( tablename, bytes ) ) { return false; } + std::string const text( bytes.begin(), bytes.end() ); + + auto found { false }; + std::size_t at { 0 }; + while( at < text.size() ) { + auto const end { std::min( text.find( '\n', at ), text.size() ) }; + std::string const line { text.substr( at, end - at ) }; + at = end + 1; + if( line.empty() ) { continue; } + auto const space { line.find( ' ' ) }; + std::string const what { line.substr( 0, space ) }; + std::string const rest { space == std::string::npos ? std::string {} : line.substr( space + 1 ) }; + if( what == "eu07-terrain" ) { found = true; } + else if( what == "rules" ) { m_table.rules = static_cast( std::strtoul( rest.c_str(), nullptr, 10 ) ); } + else if( what == "levels" ) { m_table.levels = static_cast( std::strtoul( rest.c_str(), nullptr, 10 ) ); } + else if( what == "square" ) { + char const *cursor { rest.c_str() }; + m_table.west = std::strtod( cursor, const_cast( &cursor ) ); + m_table.north = std::strtod( cursor, const_cast( &cursor ) ); + m_table.side = std::strtod( cursor, const_cast( &cursor ) ); + } + else if( what == "height" ) { + char const *cursor { rest.c_str() }; + m_table.lowest = std::strtod( cursor, const_cast( &cursor ) ); + m_table.highest = std::strtod( cursor, const_cast( &cursor ) ); + } + else if( what == "material" ) { + // name first, then how many metres one repeat covers. a name can hold spaces, so the + // number is taken off the end + auto const gap { rest.rfind( ' ' ) }; + if( gap == std::string::npos ) { m_table.materials.push_back( { rest, 4.f } ); } + else { + m_table.materials.push_back( + { rest.substr( 0, gap ), static_cast( std::strtod( rest.c_str() + gap + 1, nullptr ) ) } ); + } + } + else if( what == "error" ) { + char const *cursor { rest.c_str() }; + auto const level { std::strtoul( cursor, const_cast( &cursor ), 10 ) }; + auto const metres { std::strtod( cursor, const_cast( &cursor ) ) }; + if( m_table.errors.size() <= level ) { m_table.errors.resize( level + 1, 0.f ); } + m_table.errors[ level ] = static_cast( metres ); } - } else { - std::fread( Out.indices.data(), sizeof( std::uint32_t ), header.triangle_count * 3, file ); } - - // Read edge indices - std::fseek( file, header.edge_indices_offset, SEEK_SET ); - - Out.north_edge.resize( header.edge_count_north ); - Out.south_edge.resize( header.edge_count_south ); - Out.west_edge.resize( header.edge_count_west ); - Out.east_edge.resize( header.edge_count_east ); - - if( header.edge_count_north > 0 ) - std::fread( Out.north_edge.data(), sizeof( std::uint16_t ), header.edge_count_north, file ); - if( header.edge_count_south > 0 ) - std::fread( Out.south_edge.data(), sizeof( std::uint16_t ), header.edge_count_south, file ); - if( header.edge_count_west > 0 ) - std::fread( Out.west_edge.data(), sizeof( std::uint16_t ), header.edge_count_west, file ); - if( header.edge_count_east > 0 ) - std::fread( Out.east_edge.data(), sizeof( std::uint16_t ), header.edge_count_east, file ); - - // Materials - for now, all zeros (placeholder) - Out.materials.resize( header.vertex_count, 0 ); - - std::fclose( file ); + return found + && ( m_table.rules == cook_rules ) + && ( m_table.side > 0.0 ) + && ( m_table.levels >= 1 ); +} + +// Which tiles there are is worked out once. Asking the filesystem, or even the archive's index, +// for a tile that is not there is the commonest thing the renderer does - most of a quadtree is +// empty - and it must not cost anything. +inline void +reader::find_tiles() { + + auto const remember = [ this ]( std::string const &Name ) { + if( false == Name.ends_with( ".terrain" ) ) { return; } + tile_address tile {}; + if( 3 != std::sscanf( Name.c_str(), "%u/%d/%d.terrain", &tile.level, &tile.x, &tile.z ) ) { return; } + if( Name != tile_name( tile ) ) { return; } + m_present.insert( tile_key( tile ) ); }; + + if( true == m_archive ) { + for( auto const &name : m_pak.names() ) { remember( name ); } + return; + } + std::error_code error; + auto const base { std::filesystem::path( m_path ) }; + for( auto const &found : std::filesystem::recursive_directory_iterator( base, error ) ) { + if( false == found.is_regular_file( error ) ) { continue; } + auto const relative { std::filesystem::relative( found.path(), base, error ) }; + if( error ) { continue; } + remember( relative.generic_string() ); + } +} + +inline bool +reader::read_tile( tile_address const &Tile, tile_data &Out, + double const Originx, double const Originy, double const Originz ) { + + if( false == m_ready ) { return false; } + if( Tile.level >= m_table.levels ) { return false; } + if( false == entry( tile_name( Tile ), m_buffer ) ) { return false; } + + auto const *cursor { m_buffer.data() }; + auto const *end { cursor + m_buffer.size() }; + auto const take = [ &cursor, end ]( void *Into, std::size_t const Bytes ) { + if( static_cast( end - cursor ) < Bytes ) { return false; } + std::memcpy( Into, cursor, Bytes ); + cursor += Bytes; + return true; }; + + file_header header {}; + if( false == take( &header, sizeof( header ) ) ) { return false; } + + std::uint32_t vertexcount { 0 }; + if( false == take( &vertexcount, sizeof( vertexcount ) ) ) { return false; } + if( ( vertexcount < 3 ) || ( vertexcount > ( 1u << 24 ) ) ) { return false; } + + // the three runs of zigzag deltas: x, z, then height + std::vector quantized[ 3 ]; + for( auto &run : quantized ) { + run.resize( vertexcount ); + std::int32_t value { 0 }; + for( auto &stored : run ) { + std::uint16_t delta { 0 }; + if( false == take( &delta, sizeof( delta ) ) ) { return false; } + value += zigzag_decode( delta ); + if( ( value < 0 ) || ( value > quantum_max ) ) { return false; } + stored = static_cast( value ); + } + } + + auto const wide { vertexcount > 65536 }; + auto const indexsize { wide ? 4u : 2u }; + while( ( static_cast( cursor - m_buffer.data() ) % indexsize ) != 0 ) { + if( cursor >= end ) { return false; } + ++cursor; + } + + std::uint32_t trianglecount { 0 }; + if( false == take( &trianglecount, sizeof( trianglecount ) ) ) { return false; } + if( ( trianglecount == 0 ) || ( trianglecount > ( 1u << 24 ) ) ) { return false; } + + auto const codes { static_cast( trianglecount ) * 3 }; + auto decoded { false }; + if( true == wide ) { + std::vector raw( codes ); + for( auto &code : raw ) { if( false == take( &code, sizeof( code ) ) ) { return false; } } + decoded = highwater_decode( raw, Out.indices ); + } + else { + std::vector raw( codes ); + for( auto &code : raw ) { if( false == take( &code, sizeof( code ) ) ) { return false; } } + decoded = highwater_decode( raw, Out.indices ); + } + if( false == decoded ) { return false; } + for( auto const index : Out.indices ) { + if( index >= vertexcount ) { return false; } + } + + // the four lists of border vertices. nothing here needs them - a tile's border survives into + // every level of it, so neighbours meet whatever level they are drawn at - but they are part + // of the format and are stepped over + for( int edge = 0; edge < 4; ++edge ) { + std::uint32_t count { 0 }; + if( false == take( &count, sizeof( count ) ) ) { return false; } + if( count > vertexcount ) { return false; } + if( static_cast( end - cursor ) < count * indexsize ) { return false; } + cursor += count * indexsize; + } + + // which material each vertex belongs to, from the extension the cook writes + std::vector ground; + while( cursor < end ) { + std::uint8_t id { 0 }; + std::uint32_t length { 0 }; + if( false == take( &id, sizeof( id ) ) ) { return false; } + if( false == take( &length, sizeof( length ) ) ) { return false; } + if( static_cast( end - cursor ) < length ) { return false; } + if( ( id == ext_ground ) && ( length == vertexcount * sizeof( std::uint16_t ) ) ) { + ground.resize( vertexcount ); + std::memcpy( ground.data(), cursor, length ); + } + cursor += length; + } + if( ground.size() != vertexcount ) { return false; } + + auto const side { m_table.tilesize( Tile.level ) }; + auto const west { m_table.west + Tile.x * side }; + auto const north { m_table.north + Tile.z * side }; + auto const heightspan { std::max( 1e-3, static_cast( header.highest ) - header.lowest ) }; + + Out.tile = Tile; + Out.originx = Originx; + Out.originy = Originy; + Out.originz = Originz; + Out.centrex = header.sphere_x; + Out.centrey = header.sphere_y; + Out.centrez = header.sphere_z; + Out.radius = header.sphere_radius; + Out.lowest = header.lowest; + Out.highest = header.highest; + + Out.vertices.resize( vertexcount ); + for( std::uint32_t index = 0; index < vertexcount; ++index ) { + auto &vertex { Out.vertices[ index ] }; + vertex.x = static_cast( dequantize( quantized[ 0 ][ index ], west, side ) - Out.originx ); + vertex.y = static_cast( dequantize( quantized[ 2 ][ index ], header.lowest, heightspan ) - Out.originy ); + vertex.z = static_cast( dequantize( quantized[ 1 ][ index ], north, side ) - Out.originz ); + vertex.material = ground[ index ]; + vertex.padding = 0; + } return true; } +// writes the table the reader expects, so that the cook and the reader cannot disagree about it +inline std::vector +write_table( terrain_table const &Table ) { + + auto const number = []( double const Value ) { + char text[ 64 ]; + std::snprintf( text, sizeof( text ), "%.6f", Value ); + return std::string { text }; }; + + std::string text { "eu07-terrain 1\n" }; + text += "rules " + std::to_string( Table.rules ) + "\n"; + text += "square " + number( Table.west ) + " " + number( Table.north ) + " " + number( Table.side ) + "\n"; + text += "height " + number( Table.lowest ) + " " + number( Table.highest ) + "\n"; + text += "levels " + std::to_string( Table.levels ) + "\n"; + for( std::size_t level = 0; level < Table.errors.size(); ++level ) { + text += "error " + std::to_string( level ) + " " + number( Table.errors[ level ] ) + "\n"; + } + for( auto const &material : Table.materials ) { + text += "material " + material.name + " " + number( material.repeat ) + "\n"; + } + return { text.begin(), text.end() }; +} + } // namespace quantizedmesh diff --git a/scene/terraincooker_tin.cpp b/scene/terraincooker_tin.cpp deleted file mode 100644 index 5b18f950..00000000 --- a/scene/terraincooker_tin.cpp +++ /dev/null @@ -1,579 +0,0 @@ -/* -This Source Code Form is subject to the -terms of the Mozilla Public License, v. -2.0. If a copy of the MPL was not -distributed with this file, You can -obtain one at -http://mozilla.org/MPL/2.0/. -*/ - -#include "scene/terraincooker_tin.h" -#include "scene/terraincooker.h" - -#include -#include -#include -#include -#include -#include -#include - -namespace terrain { - -namespace { - -// Hash for int64 tile key -struct tile_key_hash { - std::size_t operator()( std::int64_t key ) const noexcept { - return static_cast( key ^ ( key >> 32 ) ); - } -}; - -// Calculate tile coordinates from world position -std::int32_t tile_coord( double world_pos, double tile_size ) { - return static_cast( std::floor( world_pos / tile_size ) ); -} - -std::int64_t make_tile_key( std::int32_t x, std::int32_t z ) { - return ( static_cast( x ) << 32 ) | static_cast( z ); -} - -} // anonymous namespace - -void tin_cooker::configure( - double TileSize, - std::uint32_t LodLevels, - double ErrorMultiplier -) { - m_tilesize = TileSize; - m_lodlevels = std::max( 1u, LodLevels ); - m_errormultiplier = std::max( 1.1, ErrorMultiplier ); -} - -void tin_cooker::rasterize( - vertex const &A, - vertex const &B, - vertex const &C, - std::string_view Material -) { - ++m_triangles_in; - - // Find centroid to determine tile - double const cx = ( A.x + B.x + C.x ) / 3.0; - double const cz = ( A.z + B.z + C.z ) / 3.0; - - std::int32_t const tile_x = tile_coord( cx, m_tilesize ); - std::int32_t const tile_z = tile_coord( cz, m_tilesize ); - - std::int64_t const key = make_tile_key( tile_x, tile_z ); - - auto &tile = m_tiles[ key ]; - tile.triangles.push_back( { { A, B, C }, std::string( Material ) } ); - tile.dirty = true; -} - -void tin_cooker::add_constraint( vertex const &A, vertex const &B ) { - // Add constraint to all tiles it intersects - double const minx = std::min( A.x, B.x ); - double const maxx = std::max( A.x, B.x ); - double const minz = std::min( A.z, B.z ); - double const maxz = std::max( A.z, B.z ); - - std::int32_t const tile_x0 = tile_coord( minx, m_tilesize ); - std::int32_t const tile_z0 = tile_coord( minz, m_tilesize ); - std::int32_t const tile_x1 = tile_coord( maxx, m_tilesize ); - std::int32_t const tile_z1 = tile_coord( maxz, m_tilesize ); - - for( std::int32_t tx = tile_x0; tx <= tile_x1; ++tx ) { - for( std::int32_t tz = tile_z0; tz <= tile_z1; ++tz ) { - std::int64_t const key = make_tile_key( tx, tz ); - auto &tile = m_tiles[ key ]; - tile.constraints.push_back( { A, B } ); - tile.dirty = true; - } - } -} - -std::vector tin_cooker::finish() { - std::vector result; - result.reserve( m_tiles.size() ); - - std::size_t total_tiles = m_tiles.size(); - std::size_t processed = 0; - - for( auto &[ key, accumulator ] : m_tiles ) { - if( !accumulator.dirty || accumulator.triangles.empty() ) { - continue; - } - - ++processed; - if( processed % 100 == 0 || processed == total_tiles ) { - std::printf( "\rTriangulating tiles: %zu/%zu", processed, total_tiles ); - std::fflush( stdout ); - } - - std::int32_t const tile_x = static_cast( key >> 32 ); - std::int32_t const tile_z = static_cast( key & 0xffffffff ); - - tin_tile tile; - tile.x = tile_x; - tile.z = tile_z; - - // Tile bounds - tile.minx = tile_x * m_tilesize; - tile.minz = tile_z * m_tilesize; - tile.maxx = tile.minx + m_tilesize; - tile.maxz = tile.minz + m_tilesize; - tile.miny = std::numeric_limits::max(); - tile.maxy = std::numeric_limits::lowest(); - - // Collect unique vertices from triangles - std::vector points; - points.reserve( accumulator.triangles.size() * 3 ); - - for( auto const &tri : accumulator.triangles ) { - for( auto const &v : tri.v ) { - points.push_back( v ); - tile.miny = std::min( tile.miny, v.y ); - tile.maxy = std::max( tile.maxy, v.y ); - } - } - - // Build material table - for( auto const &tri : accumulator.triangles ) { - get_material_index( tile, tri.material ); - } - - // Generate LODs from finest to coarsest - double current_error = 0.1; // Start with very fine detail - - for( std::uint32_t lod = 0; lod < m_lodlevels; ++lod ) { - tin_lod level; - - if( lod == 0 ) { - // Finest LOD: triangulate all points with constraints - level = triangulate_lod( points, accumulator.constraints, current_error ); - } else { - // Coarser LODs: simplify previous level - level = simplify_lod( tile.lods.back(), current_error ); - } - - level.geometric_error = current_error; - extract_edges( level, tile.minx, tile.minz, tile.maxx, tile.maxz ); - - tile.lods.push_back( std::move( level ) ); - - m_triangles_out += level.indices.size() / 3; - m_vertices_out += level.vertices.size(); - - current_error *= m_errormultiplier; - } - - result.push_back( std::move( tile ) ); - } - - std::printf( "\n" ); - std::printf( "Triangulation complete:\n" ); - std::printf( " Input triangles: %zu\n", m_triangles_in ); - std::printf( " Output triangles: %zu\n", m_triangles_out ); - std::printf( " Output vertices: %zu\n", m_vertices_out ); - std::printf( " Reduction: %.1f%%\n", - m_triangles_in > 0 ? 100.0 * ( 1.0 - static_cast( m_triangles_out ) / m_triangles_in ) : 0.0 ); - - return result; -} - -tin_lod tin_cooker::triangulate_lod( - std::vector const &Points, - std::vector const &Constraints, - double TargetError -) { - tin_lod result; - - if( Points.empty() ) { - return result; - } - - // Convert points to CDT format, removing duplicates - std::vector> cdt_points; - std::vector cdt_edges; - - cdt_points.reserve( Points.size() ); - std::unordered_map point_map; - - auto hash_point = []( vertex const &v ) -> std::uint64_t { - // Hash to 1cm precision - std::uint32_t const x = static_cast( std::round( v.x * 100.0 ) ); - std::uint32_t const z = static_cast( std::round( v.z * 100.0 ) ); - return ( static_cast( x ) << 32 ) | z; - }; - - for( auto const &v : Points ) { - std::uint64_t const h = hash_point( v ); - auto const [ it, inserted ] = point_map.try_emplace( h, static_cast( cdt_points.size() ) ); - - if( inserted ) { - cdt_points.push_back( to_cdt_point( v ) ); - result.vertices.push_back( v ); - } - } - - // Add constraint edges - for( auto const &constraint : Constraints ) { - std::uint64_t const h0 = hash_point( constraint.a ); - std::uint64_t const h1 = hash_point( constraint.b ); - - auto it0 = point_map.find( h0 ); - auto it1 = point_map.find( h1 ); - - if( it0 != point_map.end() && it1 != point_map.end() && it0->second != it1->second ) { - cdt_edges.push_back( CDT::Edge( it0->second, it1->second ) ); - } - } - - // Run CDT - CDT::Triangulation cdt; - cdt.insertVertices( cdt_points ); - - if( !cdt_edges.empty() ) { - cdt.insertEdges( cdt_edges ); - cdt.eraseOuterTrianglesAndHoles(); - } - - // Extract triangles - result.indices.reserve( cdt.triangles.size() * 3 ); - - for( auto const &tri : cdt.triangles ) { - result.indices.push_back( tri.vertices[ 0 ] ); - result.indices.push_back( tri.vertices[ 1 ] ); - result.indices.push_back( tri.vertices[ 2 ] ); - } - - // Assign material indices (simplified - use most common material for now) - result.materials.resize( result.vertices.size(), 0 ); - - return result; -} - -tin_lod tin_cooker::simplify_lod( - tin_lod const &FineLod, - double TargetError -) { - // Edge collapse simplification - // For now: simple vertex removal based on geometric error - - tin_lod result = FineLod; // Start with copy - - if( FineLod.vertices.size() < 100 ) { - // Too small to simplify meaningfully - return result; - } - - // Build adjacency information - std::unordered_map> adjacency; - - for( std::size_t i = 0; i < FineLod.indices.size(); i += 3 ) { - std::uint32_t const i0 = FineLod.indices[ i + 0 ]; - std::uint32_t const i1 = FineLod.indices[ i + 1 ]; - std::uint32_t const i2 = FineLod.indices[ i + 2 ]; - - adjacency[ i0 ].push_back( i1 ); - adjacency[ i0 ].push_back( i2 ); - adjacency[ i1 ].push_back( i0 ); - adjacency[ i1 ].push_back( i2 ); - adjacency[ i2 ].push_back( i0 ); - adjacency[ i2 ].push_back( i1 ); - } - - // Calculate error for each vertex - struct vertex_error { - std::uint32_t index; - double error; - - bool operator>( vertex_error const &other ) const { - return error > other.error; // Min-heap - } - }; - - std::priority_queue, std::greater> error_queue; - std::unordered_set removed; - - for( std::uint32_t i = 0; i < FineLod.vertices.size(); ++i ) { - auto it = adjacency.find( i ); - if( it == adjacency.end() || it->second.empty() ) { - continue; - } - - std::vector neighbours; - for( auto const j : it->second ) { - neighbours.push_back( FineLod.vertices[ j ] ); - } - - double const error = calculate_error( FineLod.vertices[ i ], neighbours ); - - if( error < TargetError ) { - error_queue.push( { i, error } ); - } - } - - // Remove vertices with low error (target: remove ~30%) - std::size_t const target_removals = FineLod.vertices.size() * 3 / 10; - - while( !error_queue.empty() && removed.size() < target_removals ) { - auto const top = error_queue.top(); - error_queue.pop(); - - if( removed.count( top.index ) > 0 ) { - continue; - } - - removed.insert( top.index ); - } - - // Rebuild mesh without removed vertices - std::unordered_map index_remap; - std::vector new_vertices; - std::vector new_materials; - - for( std::uint32_t i = 0; i < FineLod.vertices.size(); ++i ) { - if( removed.count( i ) == 0 ) { - index_remap[ i ] = static_cast( new_vertices.size() ); - new_vertices.push_back( FineLod.vertices[ i ] ); - if( i < FineLod.materials.size() ) { - new_materials.push_back( FineLod.materials[ i ] ); - } - } - } - - result.vertices = std::move( new_vertices ); - result.materials = std::move( new_materials ); - - // Re-triangulate (simple approach: use CDT again) - std::vector> cdt_points; - cdt_points.reserve( result.vertices.size() ); - - for( auto const &v : result.vertices ) { - cdt_points.push_back( to_cdt_point( v ) ); - } - - CDT::Triangulation cdt; - cdt.insertVertices( cdt_points ); - - result.indices.clear(); - result.indices.reserve( cdt.triangles.size() * 3 ); - - for( auto const &tri : cdt.triangles ) { - result.indices.push_back( tri.vertices[ 0 ] ); - result.indices.push_back( tri.vertices[ 1 ] ); - result.indices.push_back( tri.vertices[ 2 ] ); - } - - return result; -} - -double tin_cooker::calculate_error( - vertex const &V, - std::vector const &Neighbours -) { - if( Neighbours.size() < 3 ) { - return std::numeric_limits::max(); - } - - // Calculate maximum distance to plane fitted through neighbours - // Simplified: just use average height difference - - double sum_y = 0.0; - for( auto const &n : Neighbours ) { - sum_y += n.y; - } - double const avg_y = sum_y / Neighbours.size(); - - return std::abs( V.y - avg_y ); -} - -void tin_cooker::extract_edges( - tin_lod &Lod, - double TileMinX, - double TileMinZ, - double TileMaxX, - double TileMaxZ -) { - constexpr double edge_epsilon = 0.01; - - Lod.north_edge.clear(); - Lod.south_edge.clear(); - Lod.west_edge.clear(); - Lod.east_edge.clear(); - - for( std::uint32_t i = 0; i < Lod.vertices.size(); ++i ) { - auto const &v = Lod.vertices[ i ]; - - bool on_edge = false; - - if( std::abs( v.x - TileMinX ) < edge_epsilon ) { - Lod.west_edge.push_back( static_cast( i ) ); - on_edge = true; - } - if( std::abs( v.x - TileMaxX ) < edge_epsilon ) { - Lod.east_edge.push_back( static_cast( i ) ); - on_edge = true; - } - if( std::abs( v.z - TileMinZ ) < edge_epsilon ) { - Lod.south_edge.push_back( static_cast( i ) ); - on_edge = true; - } - if( std::abs( v.z - TileMaxZ ) < edge_epsilon ) { - Lod.north_edge.push_back( static_cast( i ) ); - on_edge = true; - } - } - - // Sort edges by position (west/east by Z, north/south by X) - auto sort_west_east = [ &Lod ]( std::uint16_t a, std::uint16_t b ) { - return Lod.vertices[ a ].z < Lod.vertices[ b ].z; - }; - auto sort_north_south = [ &Lod ]( std::uint16_t a, std::uint16_t b ) { - return Lod.vertices[ a ].x < Lod.vertices[ b ].x; - }; - - std::sort( Lod.west_edge.begin(), Lod.west_edge.end(), sort_west_east ); - std::sort( Lod.east_edge.begin(), Lod.east_edge.end(), sort_west_east ); - std::sort( Lod.north_edge.begin(), Lod.north_edge.end(), sort_north_south ); - std::sort( Lod.south_edge.begin(), Lod.south_edge.end(), sort_north_south ); -} - -std::uint8_t tin_cooker::get_material_index( tin_tile &Tile, std::string const &Material ) { - auto it = Tile.material_lookup.find( Material ); - - if( it != Tile.material_lookup.end() ) { - return it->second; - } - - std::uint8_t const index = static_cast( Tile.materials.size() ); - Tile.materials.push_back( Material ); - Tile.material_lookup[ Material ] = index; - - return index; -} - -bool tin_cooker::write_tile( - tin_tile const &Tile, - std::uint32_t LodIndex, - std::string const &Path, - bool Compress -) { - if( LodIndex >= Tile.lods.size() ) { - return false; - } - - auto const &lod = Tile.lods[ LodIndex ]; - - // Calculate bounding box centre - double const cx = ( Tile.minx + Tile.maxx ) * 0.5; - double const cy = ( Tile.miny + Tile.maxy ) * 0.5; - double const cz = ( Tile.minz + Tile.maxz ) * 0.5; - - // Build header - quantizedmesh::file_header header{}; - std::memcpy( header.magic, quantizedmesh::magic, sizeof( header.magic ) ); - header.version = quantizedmesh::version; - - header.center_x = cx; - header.center_y = cy; - header.center_z = cz; - - header.min_x = Tile.minx; - header.min_y = Tile.miny; - header.min_z = Tile.minz; - - header.max_x = Tile.maxx; - header.max_y = Tile.maxy; - header.max_z = Tile.maxz; - - header.horizont_error = lod.geometric_error; - header.vertex_count = static_cast( lod.vertices.size() ); - header.triangle_count = static_cast( lod.indices.size() / 3 ); - - header.index_size = lod.vertices.size() > 65535 ? 4 : 2; - header.edge_count_north = static_cast( lod.north_edge.size() ); - header.edge_count_south = static_cast( lod.south_edge.size() ); - header.edge_count_west = static_cast( lod.west_edge.size() ); - header.edge_count_east = static_cast( lod.east_edge.size() ); - - // Calculate offsets - std::uint32_t offset = sizeof( header ); - - header.vertex_data_offset = offset; - offset += header.vertex_count * 3 * sizeof( std::uint16_t ); - - header.uv_data_offset = offset; - offset += header.vertex_count * 2 * sizeof( std::uint16_t ); - - header.index_data_offset = offset; - offset += header.triangle_count * 3 * header.index_size; - - header.edge_indices_offset = offset; - std::uint32_t const edge_data_size = - ( header.edge_count_north + header.edge_count_south + - header.edge_count_west + header.edge_count_east ) * sizeof( std::uint16_t ); - offset += edge_data_size; - - header.extension_offset = offset; - - // Write file - std::ofstream file( Path, std::ios::binary ); - if( !file.good() ) { - return false; - } - - file.write( reinterpret_cast( &header ), sizeof( header ) ); - - // Write quantized vertex positions - for( auto const &v : lod.vertices ) { - std::uint16_t const qx = quantizedmesh::quantize( v.x, Tile.minx, Tile.maxx ); - std::uint16_t const qy = quantizedmesh::quantize( v.y, Tile.miny, Tile.maxy ); - std::uint16_t const qz = quantizedmesh::quantize( v.z, Tile.minz, Tile.maxz ); - - file.write( reinterpret_cast( &qx ), sizeof( qx ) ); - file.write( reinterpret_cast( &qy ), sizeof( qy ) ); - file.write( reinterpret_cast( &qz ), sizeof( qz ) ); - } - - // Write UV coordinates (from texture coords in vertex) - for( auto const &v : lod.vertices ) { - std::uint16_t const u = quantizedmesh::quantize( v.u, 0.0, 1.0 ); - std::uint16_t const uv = quantizedmesh::quantize( v.v, 0.0, 1.0 ); - - file.write( reinterpret_cast( &u ), sizeof( u ) ); - file.write( reinterpret_cast( &uv ), sizeof( uv ) ); - } - - // Write indices - for( auto const idx : lod.indices ) { - if( header.index_size == 2 ) { - std::uint16_t const idx16 = static_cast( idx ); - file.write( reinterpret_cast( &idx16 ), sizeof( idx16 ) ); - } else { - file.write( reinterpret_cast( &idx ), sizeof( idx ) ); - } - } - - // Write edge indices - for( auto const idx : lod.north_edge ) { - file.write( reinterpret_cast( &idx ), sizeof( idx ) ); - } - for( auto const idx : lod.south_edge ) { - file.write( reinterpret_cast( &idx ), sizeof( idx ) ); - } - for( auto const idx : lod.west_edge ) { - file.write( reinterpret_cast( &idx ), sizeof( idx ) ); - } - for( auto const idx : lod.east_edge ) { - file.write( reinterpret_cast( &idx ), sizeof( idx ) ); - } - - file.close(); - return file.good(); -} - -} // namespace terrain diff --git a/scene/terraincooker_tin.h b/scene/terraincooker_tin.h deleted file mode 100644 index 7512a8a3..00000000 --- a/scene/terraincooker_tin.h +++ /dev/null @@ -1,194 +0,0 @@ -/* -This Source Code Form is subject to the -terms of the Mozilla Public License, v. -2.0. If a copy of the MPL was not -distributed with this file, You can -obtain one at -http://mozilla.org/MPL/2.0/. -*/ - -#pragma once - -// Triangulates terrain into TIN with multiple LODs using Constrained Delaunay. -// Input: triangle soup from scenery (same as heightfield cooker) -// Output: quantized mesh tiles with proper edge constraints for seamless LOD - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "scene/quantizedmeshformat.h" -#include "scene/terraincooker.h" // For terrain::vertex definition - -namespace terrain { - -// Triangle with material -struct tin_triangle { - vertex v[ 3 ]; - std::string material; -}; - -// Constraint edge (e.g. track corridor boundary) -struct tin_constraint { - vertex a, b; -}; - -// Single LOD level of a TIN tile -struct tin_lod { - std::vector vertices; - std::vector indices; - std::vector materials; // Per-vertex material indices - - // Edge vertices for seamless stitching (indices into vertices array) - std::vector north_edge; - std::vector south_edge; - std::vector west_edge; - std::vector east_edge; - - double geometric_error; // Maximum error vs. finer LOD -}; - -// Complete TIN tile with all LODs -struct tin_tile { - std::int32_t x, z; // Tile coordinates - - std::vector lods; // Finest to coarsest - - // Bounding box - double minx, miny, minz; - double maxx, maxy, maxz; - - // Material table (shared across LODs) - std::vector materials; - std::unordered_map material_lookup; -}; - -class tin_cooker { -public: - tin_cooker() = default; - - // Configure tile size and LOD parameters - void configure( - double TileSize, // Metres per tile side - std::uint32_t LodLevels = 4, // Number of LOD levels - double ErrorMultiplier = 2.0 // Geometric error growth per LOD - ); - - // Accumulate terrain triangles (same interface as heightfield cooker) - void rasterize( - vertex const &A, - vertex const &B, - vertex const &C, - std::string_view Material - ); - - // Add constraint edges (track corridors, roads, water boundaries) - void add_constraint( vertex const &A, vertex const &B ); - - // Generate all tiles with LODs - std::vector finish(); - - // Export single tile to Quantized Mesh format - static bool write_tile( - tin_tile const &Tile, - std::uint32_t LodIndex, - std::string const &Path, - bool Compress = true - ); - -private: - struct tile_accumulator { - std::vector triangles; - std::vector constraints; - bool dirty { false }; - }; - - // Build single LOD level using CDT - tin_lod triangulate_lod( - std::vector const &Points, - std::vector const &Constraints, - double TargetError - ); - - // Simplify LOD by removing vertices with low geometric error - tin_lod simplify_lod( - tin_lod const &FineLod, - double TargetError - ); - - // Calculate geometric error of removing a vertex - double calculate_error( - vertex const &V, - std::vector const &Neighbours - ); - - // Extract edge vertices for stitching - void extract_edges( tin_lod &Lod, double TileMinX, double TileMinZ, double TileMaxX, double TileMaxZ ); - - // Find material index, adding to table if new - std::uint8_t get_material_index( tin_tile &Tile, std::string const &Material ); - - double m_tilesize { 256.0 }; - std::uint32_t m_lodlevels { 4 }; - double m_errormultiplier { 2.0 }; - - std::unordered_map m_tiles; - - // Statistics - std::size_t m_triangles_in { 0 }; - std::size_t m_triangles_out { 0 }; - std::size_t m_vertices_out { 0 }; -}; - -// Helper: convert CDT point to our vertex -inline vertex from_cdt_point( CDT::V2d const &P, double Y = 0.0 ) { - return vertex{ P.x, Y, P.y }; -} - -// Helper: convert vertex to CDT point -inline CDT::V2d to_cdt_point( vertex const &V ) { - return CDT::V2d::make( V.x, V.z ); -} - -// Helper: calculate distance from point to line segment -inline double point_to_segment_distance( - vertex const &Point, - vertex const &SegmentA, - vertex const &SegmentB -) { - double const dx = SegmentB.x - SegmentA.x; - double const dz = SegmentB.z - SegmentA.z; - double const len_sq = dx * dx + dz * dz; - - if( len_sq < 1e-10 ) { - // Degenerate segment - double const pdx = Point.x - SegmentA.x; - double const pdz = Point.z - SegmentA.z; - return std::sqrt( pdx * pdx + pdz * pdz ); - } - - // Project point onto line - double const t = std::clamp( - ( ( Point.x - SegmentA.x ) * dx + ( Point.z - SegmentA.z ) * dz ) / len_sq, - 0.0, 1.0 - ); - - double const proj_x = SegmentA.x + t * dx; - double const proj_z = SegmentA.z + t * dz; - - double const pdx = Point.x - proj_x; - double const pdy = Point.y - ( SegmentA.y + t * ( SegmentB.y - SegmentA.y ) ); - double const pdz = Point.z - proj_z; - - return std::sqrt( pdx * pdx + pdy * pdy + pdz * pdz ); -} - -} // namespace terrain diff --git a/shaders/terrainclipmap_tin.frag b/shaders/terrainclipmap_tin.frag deleted file mode 100644 index ff55d87f..00000000 --- a/shaders/terrainclipmap_tin.frag +++ /dev/null @@ -1,23 +0,0 @@ -// TIN Terrain Fragment Shader -// Minimal material lookup, no blending for now - -#version 330 core - -// Input from vertex shader -in vec2 vUV; -flat in uint vMaterial; - -// Uniforms -uniform sampler2DArray ground; // Material texture array - -// Output -out vec4 fragColor; - -void main() { - // Sample material texture - // TODO: Add blending between materials for smooth transitions - vec3 texCoord = vec3(vUV, float(vMaterial)); - vec4 color = texture(ground, texCoord); - - fragColor = color; -} diff --git a/shaders/terrainclipmap_tin.vert b/shaders/terrainclipmap_tin.vert deleted file mode 100644 index 04bf63de..00000000 --- a/shaders/terrainclipmap_tin.vert +++ /dev/null @@ -1,30 +0,0 @@ -// TIN Terrain Vertex Shader -// Replaces texture-based heightfield with vertex attributes - -#version 330 core - -// Vertex attributes from VBO -layout(location = 0) in vec3 aPosition; // Relative to tile origin -layout(location = 1) in vec2 aUV; -layout(location = 2) in uint aMaterial; - -// Uniforms -uniform vec3 tile_offset; // Tile origin relative to camera -uniform mat4 projectionMatrix; -uniform mat4 modelViewMatrix; - -// Output to fragment shader -out vec2 vUV; -flat out uint vMaterial; - -void main() { - // Position is tile-relative, add offset to get camera-relative - vec3 worldPos = aPosition + tile_offset; - - // Transform to clip space - gl_Position = projectionMatrix * modelViewMatrix * vec4(worldPos, 1.0); - - // Pass through to fragment shader - vUV = aUV; - vMaterial = aMaterial; -} diff --git a/shaders/terrainmesh.frag b/shaders/terrainmesh.frag new file mode 100644 index 00000000..8669b8e1 --- /dev/null +++ b/shaders/terrainmesh.frag @@ -0,0 +1,56 @@ +// Terrain mesh, fragment stage. +// +// One triangle carries one material: the cook only welds vertices that share it, so nothing has to +// be blended here and a material boundary follows the edges the scenery drew it with. The texture +// coordinate is the world position over the material's tiling, so the ground keeps the scale it was +// authored at whatever level the tile is drawn at. +// +// The mesh carries no normals - that would be four more bytes on every vertex - so the surface +// normal comes from how the view position changes across the triangle, which is its face normal. + +in vec4 f_pos; +in vec2 f_ground; +flat in uint f_material; + +#include +#include +#include + +uniform sampler2DArray ground; +// per material: rgb the colour to show while it has no texture, a the metres one texture repeat +// covers - negative while there is no texture layer for it yet +uniform sampler2D materialtable; + +layout(location = 0) out vec4 out_color; +#if MOTIONBLUR_ENABLED +layout(location = 1) out vec4 out_motion; +#endif + +void main() +{ + vec4 entry = texelFetch( materialtable, ivec2( int( f_material ), 0 ), 0 ); + vec3 albedo = + ( entry.a < 0.0 ) + ? entry.rgb + : texture( ground, vec3( f_ground / max( entry.a, 0.01 ), float( f_material ) ) ).rgb; + + vec3 normal = normalize( cross( dFdx( f_pos.xyz ), dFdy( f_pos.xyz ) ) ); + // towards the camera, whichever way the derivatives came out + if( dot( normal, -f_pos.xyz ) < 0.0 ) { normal = -normal; } + + // a plain lambert term against the first directional light, so that relief reads + float lambert = 0.35; + if( lights_count >= 1U && lights[ 0 ].type == LIGHT_DIR ) { + lambert = 0.35 + 0.65 * max( dot( normal, normalize( -lights[ 0 ].dir ) ), 0.0 ); + } + vec3 col = albedo * lambert; + +#if POSTFX_ENABLED + out_color = vec4( apply_fog( col ), 1.0 ); +#else + out_color = tonemap( vec4( apply_fog( col ), 1.0 ) ); +#endif +#if MOTIONBLUR_ENABLED + out_motion = vec4( 0.0 ); +#endif +} diff --git a/shaders/terrainmesh.vert b/shaders/terrainmesh.vert new file mode 100644 index 00000000..210529a9 --- /dev/null +++ b/shaders/terrainmesh.vert @@ -0,0 +1,32 @@ +// Terrain mesh, vertex stage. +// +// A vertex is a position and the ground material it belongs to, nothing else. The position is +// measured from the middle of the terrain rather than from the scenery's origin, so that the whole +// of a map fits in floats and every resident tile can live in one vertex buffer - which is what +// lets a frame of terrain be one draw call. terrainorigin is where that middle is relative to the +// camera, terrainworld where it is in the world. +// +// There are no texture coordinates: ground textures tile in world space, so the coordinate follows +// from where the vertex is and from how many metres one repeat of its material covers, which the +// cook measured from the scenery and left in the material table. + +#include + +layout(location = 0) in vec3 in_position; +layout(location = 1) in uint in_material; + +uniform vec3 terrainorigin; // the terrain's middle, relative to the camera +uniform vec2 terrainworld; // the terrain's middle, in the world + +out vec4 f_pos; +out vec2 f_ground; // world position on the ground, for texturing +flat out uint f_material; + +void main() +{ + vec3 local = in_position + terrainorigin; + f_ground = terrainworld + in_position.xz; + f_material = in_material; + f_pos = modelview * vec4( local, 1.0 ); + gl_Position = projection * f_pos; +} diff --git a/simulation/cookdeps.cpp b/simulation/cookdeps.cpp index 4a723372..3b01535f 100644 --- a/simulation/cookdeps.cpp +++ b/simulation/cookdeps.cpp @@ -94,10 +94,13 @@ write( std::string const &Path, std::vector const &Files ) { entry item; item.path = file; if( false == hash_of( file, item.size, item.hash ) ) { - // a file that was read a moment ago and cannot be now; the artifact is recorded as - // depending on its absence, which any later read will contradict + // a file the scenery names and that is not there. all zeroes records that it was not + // there, so that a later read agrees with the manifest instead of forcing a recook item.size = 0; item.hash = 0; + item.time = 0; + text << item.size << ' ' << item.time << ' ' << std::hex << item.hash << std::dec << ' ' << item.path << '\n'; + continue; } item.time = static_cast( last_modified( file ) ); text << item.size << ' ' << item.time << ' ' << std::hex << item.hash << std::dec << ' ' << item.path << '\n'; @@ -127,7 +130,16 @@ current( std::string const &Path ) { } std::uintmax_t size { 0 }; std::uint64_t hash { 0 }; - if( ( false == hash_of( item.path, size, hash ) ) || ( size != item.size ) || ( hash != item.hash ) ) { + if( false == hash_of( item.path, size, hash ) ) { + // not there now. a scenery may name an include that does not exist, and if it did not + // exist when the manifest was written either then nothing has changed. an empty file + // that does exist hashes to the starting value rather than to zero, so the two cannot + // be confused + if( ( item.size == 0 ) && ( item.hash == 0 ) && ( item.time == 0 ) ) { continue; } + WriteLog( "Cooked data: \"" + item.path + "\" is gone since \"" + Path + "\" was written" ); + return false; + } + if( ( size != item.size ) || ( hash != item.hash ) ) { WriteLog( "Cooked data: \"" + item.path + "\" has changed since \"" + Path + "\" was written" ); return false; } diff --git a/simulation/simulationstateserializer.cpp b/simulation/simulationstateserializer.cpp index b620ff72..5c1066ed 100644 --- a/simulation/simulationstateserializer.cpp +++ b/simulation/simulationstateserializer.cpp @@ -725,16 +725,17 @@ state_serializer::deserialize_node( cParser &Input, scene::scratch_data &Scratch // the bake sees the node exactly where it is drawn. placing is done on a copy: // the region applies the same placement to the shape itself when inserting it auto const &data { shape.data() }; - auto const material { - data.material != null_handle - ? GfxRenderer->Material( data.material )->GetName() - : "none" }; + auto const *entry { + data.material != null_handle ? GfxRenderer->Material( data.material ) : nullptr }; + auto const material { entry != nullptr ? entry->GetName() : "none" }; + // a texture with an alpha channel is not ground; the bake wants to know + auto const translucent { ( entry != nullptr ) && entry->is_translucent() }; // the old switch trackbeds are dropped from the scenery when generated ones // replace them, so they are no part of the ground either if( false == ( Global.CreateSwitchTrackbeds && scene::is_switch_trackbed( material ) ) ) { auto placed { data.vertices }; scene::place_in_world( placed, Scratchpad ); - terrainbake::add( decision.key, placed, material ); + terrainbake::add( decision.key, placed, material, translucent ); } } if( decision.plan.what == terrainbake::verdict::filter ) { diff --git a/simulation/terrainbake.cpp b/simulation/terrainbake.cpp index b3f21f16..4295ebfa 100644 --- a/simulation/terrainbake.cpp +++ b/simulation/terrainbake.cpp @@ -9,14 +9,11 @@ http://mozilla.org/MPL/2.0/. module; #include -#include -#include #include #include #include #include #include -#include #include #include #include @@ -24,286 +21,176 @@ module; #include #include #include "utilities/Globals_macros.h" -#include "scene/terraincooker.h" -#include "scene/heightfieldreader.h" +#include "scene/pakformat.h" +#include "scene/quantizedmeshcooker.h" +#include "scene/quantizedmeshreader.h" module eu07.simulation.terrainbake; import eu07.utilities.logs; import eu07.utilities.globals; import eu07.utilities.utilities; -import eu07.simulation.loadprofile; +import eu07.model.vertex; import eu07.simulation.cookdeps; namespace simulation::terrainbake { namespace { -// vertices kept from the start of the scenery to tell the grid step from -constexpr std::size_t samplegoal { 200000 }; +// the most triangles a tile of any level is allowed. the finest level is the ground as the scenery +// drew it and is not cut down, so this is a ceiling for the coarser ones rather than a target +constexpr std::size_t tilebudget { 8192 }; -// Terrain comes in two kinds: modelled by hand with triangles hundreds of metres long, or taken -// from an elevation model with small triangles along the line. One grid step for both either -// loses the detail or samples a field of large triangles every metre. So a bake writes several -// heightfields at steps a factor apart. A triangle's own field is the coarsest that still puts -// samplesperedge samples along its longest edge, and it is laid into that field and every -// coarser one: the coarsest field then holds all the ground, as the layers of a level of -// detail scheme do, and a triangle at the edge of a finer field - next to ground that field -// does not hold - can still be replaced by a coarser one lying under it. Where two fields -// hold the same ground the finer one wins the depth test. A flat triangle sampled coarsely is -// the same plane; what a coarser step gives up is the exact line of the creases between -// triangles, which a hand modelled terrain does not have to spare anyway. -constexpr std::uint32_t fieldcount { 3 }; -constexpr double fieldstride { 4.0 }; -constexpr double samplesperedge { 16.0 }; - -// the node table written beside a baked heightfield -constexpr char tablemagic[ 8 ] { 'E', 'U', '0', '7', 'H', 'F', 'N', 'D' }; -// 2: nodes keyed by file, line, include parameters and placement rather than by their text -constexpr std::uint32_t tableversion { 2 }; +// the note written beside the tiles: which nodes they stand for +constexpr char notemagic[ 8 ] { 'E', 'U', '0', '7', 'T', 'N', 'O', 'D' }; +constexpr std::uint32_t noteversion { 1 }; +// a node with no mask: the tiles stand for every triangle of it constexpr std::uint32_t allshown { 0xffffffffu }; -struct table_header { +struct note_header { char magic[ 8 ]; std::uint32_t version; - std::uint32_t selection; + std::uint32_t rules; std::uint64_t count; std::uint64_t maskbytes; }; -struct table_record { +struct note_record { std::uint64_t key; std::uint32_t triangles; - // offset into the mask blob, one bit per triangle set where it is still drawn; allshown - // when the heightfield shows every triangle and there is no mask + // where this node's bits start in the mask blob, one per triangle, set where the scenery + // still draws it. allshown when none of them is drawn any more std::uint32_t mask; }; -// one triangle as it waits in the spill file for the passes: its corners in world space with -// their texture coordinates, the node it came from, which holds its material, and its number -struct spilled { - double corners[ 15 ]; - std::uint32_t node; - std::uint32_t id; -}; - -// one node as baked: where its triangles sit in the cooker's numbering +// one node as it went into the cook: where its triangles sit in the cook's numbering struct baked_node { - std::uint64_t key; - std::uint32_t first; - std::uint32_t count; - // kept once per node rather than once per spilled triangle - std::string material; + std::uint64_t key { 0 }; + std::uint32_t first { 0 }; + std::uint32_t count { 0 }; }; struct { - // baking. triangles go to a spill file as the scenery is parsed; the heightfield is made - // from it once everything is in + // cooking bool collecting { false }; - std::ofstream spill; - std::string spillpath; - std::vector sample; + quantizedmesh::cooker cooker; std::vector nodes; std::uint32_t triangles { 0 }; - // per triangle, once baked: whether the heightfield shows it - std::vector shown; + std::size_t objects { 0 }; + std::size_t objecttriangles { 0 }; std::chrono::steady_clock::time_point started; - // planning - std::unordered_map table; + // reading a cook from an earlier run + std::unordered_map note; std::vector masks; std::size_t skipped { 0 }, filtered { 0 }, drawn { 0 }; } state; -// where a field is written: the finest beside the scenery as .ehf, coarser ones as -// ..ehf -std::string -heightfield_path( std::string const &Sceneryfile, std::uint32_t const Field = 0 ) { +// beside the scenery, under its name: the same place the heightfield used to be cooked to +std::string archive_path( std::string const &Sceneryfile ) { return scenery_sidecar( Sceneryfile, "_terrain.pak" ); } +std::string note_path( std::string const &Sceneryfile ) { return scenery_sidecar( Sceneryfile, "_terrain.nodes" ); } +std::string manifest_path( std::string const &Sceneryfile ) { return scenery_sidecar( Sceneryfile, "_terrain.deps" ); } +std::string scratch_path( std::string const &Sceneryfile ) { return scenery_sidecar( Sceneryfile, "_terrain.cook" ); } - return scenery_sidecar( Sceneryfile, Field == 0 ? std::string( ".ehf" ) : "." + std::to_string( Field ) + ".ehf" ); -} - -std::string -table_path( std::string const &Heightfield ) { - - return Heightfield + ".nodes"; -} - -std::string -manifest_path( std::string const &Heightfield ) { - - return Heightfield + ".deps"; -} - -// throws away every file of a bake, so the next start makes a new one void discard( std::string const &Sceneryfile ) { std::error_code error; - for( std::uint32_t field = 0; field < fieldcount; ++field ) { - std::filesystem::remove( heightfield_path( Sceneryfile, field ), error ); + for( auto const &path : { + archive_path( Sceneryfile ), note_path( Sceneryfile ), + manifest_path( Sceneryfile ), scratch_path( Sceneryfile ) } ) { + std::filesystem::remove( path, error ); } - std::filesystem::remove( table_path( heightfield_path( Sceneryfile ) ), error ); - std::filesystem::remove( manifest_path( heightfield_path( Sceneryfile ) ), error ); } bool -load_table( std::string const &Path ) { +load_note( std::string const &Path ) { - std::ifstream input( Path, std::ios::binary ); - if( false == input.good() ) { return false; } + std::ifstream file( Path, std::ios::binary ); + if( false == file.good() ) { return false; } + note_header header {}; + file.read( reinterpret_cast( &header ), sizeof( header ) ); + if( false == file.good() ) { return false; } + if( 0 != std::memcmp( header.magic, notemagic, sizeof( notemagic ) ) ) { return false; } + if( header.version != noteversion ) { return false; } + if( header.rules != quantizedmesh::cook_rules ) { return false; } - table_header header {}; - input.read( reinterpret_cast( &header ), sizeof( header ) ); - if( ( false == static_cast( input ) ) - || ( 0 != std::memcmp( header.magic, tablemagic, sizeof( tablemagic ) ) ) - || ( header.version != tableversion ) - || ( header.selection != heightfield::selection_rules ) ) { - return false; + state.note.reserve( header.count ); + for( std::uint64_t index = 0; index < header.count; ++index ) { + note_record record {}; + file.read( reinterpret_cast( &record ), sizeof( record ) ); + if( false == file.good() ) { state.note.clear(); return false; } + state.note.emplace( record.key, record ); } - - std::vector records( header.count ); - input.read( reinterpret_cast( records.data() ), static_cast( header.count * sizeof( table_record ) ) ); state.masks.resize( header.maskbytes ); - input.read( reinterpret_cast( state.masks.data() ), static_cast( header.maskbytes ) ); - if( false == static_cast( input ) ) { return false; } - - state.table.reserve( records.size() ); - for( auto const &record : records ) { - state.table.emplace( record.key, record ); + if( header.maskbytes > 0 ) { + file.read( reinterpret_cast( state.masks.data() ), static_cast( header.maskbytes ) ); } + if( false == file.good() ) { state.note.clear(); state.masks.clear(); return false; } return true; } -void -write_table( std::string const &Path ) { +bool +save_note( std::string const &Path, std::vector const &Refusals ) { - // a key met more than once is the same line of the same file, included with the same - // parameters at the same place - the same surface: a triangle counts as shown if any of - // the copies is - std::unordered_map> shown; - for( auto const &node : state.nodes ) { - auto &entry { shown[ node.key ] }; - entry.resize( node.count, false ); - for( std::uint32_t index = 0; index < node.count; ++index ) { - if( state.shown[ node.first + index ] != 0 ) { entry[ index ] = true; } - } - } - - std::vector records; + // the cook refuses a triangle that has no ground in it - a wall - and the scenery goes on + // drawing that one. the note says so per node, as a bit each + std::vector records; std::vector masks; + records.reserve( state.nodes.size() ); + + auto refusal { Refusals.begin() }; for( auto const &node : state.nodes ) { - auto const lookup { shown.find( node.key ) }; - if( lookup == shown.end() ) { continue; } // written already - auto const &entry { lookup->second }; - std::uint32_t count { 0 }; - for( auto const flag : entry ) { count += ( flag ? 1 : 0 ); } - if( count > 0 ) { - table_record record { node.key, node.count, allshown }; - if( count < node.count ) { - record.mask = static_cast( masks.size() ); - auto const bytes { ( node.count + 7u ) / 8u }; - masks.resize( masks.size() + bytes, 0u ); - for( std::uint32_t index = 0; index < node.count; ++index ) { - if( false == entry[ index ] ) { - masks[ record.mask + index / 8u ] |= static_cast( 1u << ( index % 8u ) ); - } - } - } - records.emplace_back( record ); + while( ( refusal != Refusals.end() ) && ( *refusal < node.first ) ) { ++refusal; } + auto const last { node.first + node.count }; + auto const firstrefusal { refusal }; + std::size_t refused { 0 }; + for( auto walk = refusal; ( walk != Refusals.end() ) && ( *walk < last ); ++walk ) { ++refused; } + if( refused == 0 ) { + records.push_back( { node.key, node.count, allshown } ); + continue; } - shown.erase( lookup ); + if( refused == node.count ) { + // nothing of this node is in the tiles: it is drawn as written, and a node the note + // says nothing about is exactly that + continue; + } + auto const at { static_cast( masks.size() ) }; + masks.resize( masks.size() + ( node.count + 7 ) / 8, 0 ); + for( auto walk = firstrefusal; ( walk != Refusals.end() ) && ( *walk < last ); ++walk ) { + auto const bit { *walk - node.first }; + masks[ at + bit / 8 ] |= static_cast( 1u << ( bit % 8 ) ); + } + records.push_back( { node.key, node.count, at } ); } - table_header header {}; - std::memcpy( header.magic, tablemagic, sizeof( tablemagic ) ); - header.version = tableversion; - header.selection = heightfield::selection_rules; + std::ofstream file( Path, std::ios::binary | std::ios::trunc ); + if( false == file.good() ) { return false; } + note_header header {}; + std::memcpy( header.magic, notemagic, sizeof( notemagic ) ); + header.version = noteversion; + header.rules = quantizedmesh::cook_rules; header.count = records.size(); header.maskbytes = masks.size(); - - std::ofstream output( Path, std::ios::binary | std::ios::trunc ); - output.write( reinterpret_cast( &header ), sizeof( header ) ); - output.write( reinterpret_cast( records.data() ), static_cast( records.size() * sizeof( table_record ) ) ); - output.write( reinterpret_cast( masks.data() ), static_cast( masks.size() ) ); - - std::size_t full { 0 }; - for( auto const &record : records ) { full += ( record.mask == allshown ? 1 : 0 ); } - WriteLog( - "Terrain bake: " + std::to_string( records.size() ) + " of " + std::to_string( state.nodes.size() ) - + " triangle nodes are shown by the heightfield, " + std::to_string( full ) + " of them entirely" ); + file.write( reinterpret_cast( &header ), sizeof( header ) ); + for( auto const &record : records ) { + file.write( reinterpret_cast( &record ), sizeof( record ) ); + } + if( false == masks.empty() ) { + file.write( reinterpret_cast( masks.data() ), static_cast( masks.size() ) ); + } + return file.good(); } -// the field a triangle belongs to, given the finest step -std::uint32_t -field_of( terrain::vertex const &A, terrain::vertex const &B, terrain::vertex const &C, double const Basestep ) { - - auto const longest { std::max( { terrain::planar_length( A, B ), terrain::planar_length( B, C ), terrain::planar_length( C, A ) } ) }; - std::uint32_t field { 0 }; - auto step { Basestep }; - while( ( field + 1 < fieldcount ) && ( longest / ( step * fieldstride ) >= samplesperedge ) ) { - step *= fieldstride; - ++field; - } - return field; -} - -using fields = std::array, fieldcount>; - -// one pass over the spill file: lays every triangle into fresh cookers for its own field and -// the coarser ones, wherever Keep has the bit of that field set -fields -lay( std::string const &Spillpath, double const Basestep, std::vector const &Keep, - std::vector *Ownfield = nullptr ) { - - fields result; - auto step { Basestep }; - for( auto &cooker : result ) { - cooker = std::make_unique(); - if( false == loadprofile::enabled() ) { - // the per-material survival table is what a bad bake is diagnosed from, so it is - // printed when the load is being measured anyway - cooker->quiet(); - } - cooker->configure( step, true ); - step *= fieldstride; - } - - std::ifstream input( Spillpath, std::ios::binary ); - std::vector chunk( 4096 ); - while( input ) { - input.read( reinterpret_cast( chunk.data() ), static_cast( chunk.size() * sizeof( spilled ) ) ); - auto const count { static_cast( input.gcount() ) / sizeof( spilled ) }; - for( std::size_t index = 0; index < count; ++index ) { - auto const &triangle { chunk[ index ] }; - if( Keep[ triangle.id ] == 0 ) { continue; } - auto const corner = [ &triangle ]( std::size_t const Corner ) { - auto const *values { triangle.corners + Corner * 5 }; - return terrain::vertex { values[ 0 ], values[ 1 ], values[ 2 ], values[ 3 ], values[ 4 ] }; }; - auto const a { corner( 0 ) }, b { corner( 1 ) }, c { corner( 2 ) }; - auto const own { field_of( a, b, c, Basestep ) }; - if( Ownfield != nullptr ) { ( *Ownfield )[ triangle.id ] = static_cast( own ); } - for( auto field { own }; field < fieldcount; ++field ) { - if( ( Keep[ triangle.id ] & ( 1u << field ) ) == 0 ) { continue; } - result[ field ]->rasterize( a, b, c, state.nodes[ triangle.node ].material, triangle.id ); - } - } - } - return result; -} - -} // anonymous namespace - -namespace { - +// the key a node is known by. what it is made of, where it stands, and where it is written: the +// manifest vouches for the content of the file, so its name is enough here std::uint64_t node_key( std::string const &File, std::size_t const Line, std::vector const &Parameters, std::string_view const Type, glm::dvec3 const &Offset, glm::vec3 const &Rotation ) { - // fnv-1a. stable across runs and builds, which a std::hash is not required to be std::uint64_t hash { 0xcbf29ce484222325ull }; auto const mix { [ &hash ]( void const *Data, std::size_t const Size ) { - auto const *bytes { static_cast( Data ) }; + auto const *bytes { static_cast( Data ) }; for( std::size_t index = 0; index < Size; ++index ) { hash ^= bytes[ index ]; hash *= 0x100000001b3ull; @@ -313,7 +200,6 @@ node_key( mix( Text.data(), Text.size() ); mix( "\n", 1 ); } }; - // the file's content is vouched for by the bake's manifest, so its name is enough here text( File ); auto const line { static_cast( Line ) }; mix( &line, sizeof( line ) ); @@ -324,81 +210,16 @@ node_key( return hash; } -} // anonymous namespace - -void -begin( std::string const &Sceneryfile ) { - - state = {}; - - if( false == Global.terrain_heightfields.empty() ) { - // the scenery names its own, which always wins over anything baked beside it - return; - } - if( false == Global.bake_terrain ) { - // neither bake nor pick one up: the triangles as written - return; - } - - auto const path { heightfield_path( Sceneryfile ) }; - std::error_code error; - if( true == FileExists( table_path( path ) ) ) { - // a bake is a cache: one this build cannot read, one cooked under older rules about - // what counts as terrain, or one missing a part, is thrown away and baked again - // rather than used - std::vector baked; - auto usable { - ( true == cookdeps::current( manifest_path( path ) ) ) - && ( true == load_table( table_path( path ) ) ) }; - for( std::uint32_t field = 0; ( field < fieldcount ) && ( true == usable ); ++field ) { - auto const fieldpath { heightfield_path( Sceneryfile, field ) }; - if( false == FileExists( fieldpath ) ) { continue; } - heightfield::reader probe; - usable = ( true == probe.open( fieldpath ) ) && ( probe.selection() == heightfield::selection_rules ); - baked.push_back( fieldpath ); - } - if( ( true == usable ) && ( false == baked.empty() ) ) { - for( auto const &fieldpath : baked ) { - Global.terrain_heightfields.push_back( fieldpath ); - } - WriteLog( - "Terrain bake: using " + std::to_string( baked.size() ) + " heightfields baked earlier, " - + std::to_string( state.table.size() ) + " triangle nodes replaced by them" ); - return; - } - WriteLog( "Terrain bake: the heightfields beside this scenery are out of date, baking them again" ); - state.table.clear(); - state.masks.clear(); - discard( Sceneryfile ); - } - - // the triangles wait on disk beside the heightfield rather than in memory: a large scenery - // holds millions of them, and the bake reads them several times - state.spillpath = path + ".bake"; - state.spill.open( state.spillpath, std::ios::binary | std::ios::trunc ); - if( false == state.spill.good() ) { - ErrorLog( "Terrain bake: cannot write \"" + state.spillpath + "\", not baking" ); - return; - } - WriteLog( "Terrain bake: no heightfield for this scenery, baking one. This load will be slow; the next will not" ); - state.collecting = true; - state.started = std::chrono::steady_clock::now(); -} - -namespace { - -bool -planning() { - - return false == state.table.empty(); -} +bool planning() { return false == state.note.empty(); } +// collecting, and not overruled by a terrain the scenery named part way in +bool baking() { return state.collecting && Global.terrain_heightfields.empty(); } node_plan plan( std::uint64_t const Key ) { node_plan result; - auto const lookup { state.table.find( Key ) }; - if( lookup == state.table.end() ) { + auto const lookup { state.note.find( Key ) }; + if( lookup == state.note.end() ) { ++state.drawn; return result; } @@ -413,19 +234,74 @@ plan( std::uint64_t const Key ) { result.drawn.resize( record.triangles, false ); for( std::uint32_t index = 0; index < record.triangles; ++index ) { auto const byte { record.mask + index / 8u }; - result.drawn[ index ] = ( byte < state.masks.size() ) && ( ( state.masks[ byte ] >> ( index % 8u ) ) & 1u ) != 0; + result.drawn[ index ] = + ( byte < state.masks.size() ) && ( ( ( state.masks[ byte ] >> ( index % 8u ) ) & 1u ) != 0 ); } return result; } -// collecting for a bake, and not overruled by a heightfield the scenery named part way in -bool -baking() { +} // anonymous namespace - return state.collecting && Global.terrain_heightfields.empty(); +void +begin( std::string const &Sceneryfile ) { + + state.collecting = false; + state.nodes.clear(); + state.triangles = 0; + state.objects = 0; + state.objecttriangles = 0; + state.note.clear(); + state.masks.clear(); + state.skipped = state.filtered = state.drawn = 0; + + if( false == Global.terrain_heightfields.empty() ) { + // the scenery names a terrain of its own, which wins over anything cooked beside it + return; + } + if( false == Global.bake_terrain ) { + // neither cook nor pick one up: the triangles as written + return; + } + + auto const archive { archive_path( Sceneryfile ) }; + if( true == FileExists( note_path( Sceneryfile ) ) ) { + // a cook is a cache. one this build cannot read, one made under other rules about what + // counts as ground, or one made from a scenery that has changed since, is thrown away + // and cooked again rather than used + quantizedmesh::reader probe; + auto const usable { + ( true == cookdeps::current( manifest_path( Sceneryfile ) ) ) + && ( true == probe.open( archive ) ) + && ( true == load_note( note_path( Sceneryfile ) ) ) }; + if( true == usable ) { + Global.terrain_heightfields.push_back( archive ); + WriteLog( + "Terrain cook: using tiles cooked earlier, " + std::to_string( probe.table().levels ) + + " levels over " + std::to_string( static_cast( probe.table().side ) ) + " m, " + + std::to_string( state.note.size() ) + " triangle nodes stood for" ); + return; + } + WriteLog( "Terrain cook: the tiles beside this scenery are out of date, cooking them again" ); + state.note.clear(); + state.masks.clear(); + discard( Sceneryfile ); + } + + if( false == state.cooker.begin( scratch_path( Sceneryfile ), tilebudget ) ) { + ErrorLog( "Terrain cook: cannot write \"" + scratch_path( Sceneryfile ) + "\", not cooking" ); + return; + } + state.cooker.quiet(); + WriteLog( "Terrain cook: no tiles for this scenery, cooking them. This load will be slow; the next will not" ); + state.collecting = true; + state.started = std::chrono::steady_clock::now(); } -} // anonymous namespace +bool +collecting() { + + return state.collecting; +} node_decision examine( @@ -435,47 +311,61 @@ examine( node_decision decision; if( ( true == baking() ) || ( true == planning() ) ) { decision.key = node_key( File, Line, Parameters, Type, Offset, Rotation ); - decision.bake = baking(); + // only what the scenery drew in world space: a node under origin, rotate or scale is an + // object standing on the ground rather than the ground itself + decision.bake = baking() && Worldspace; if( false == decision.bake ) { decision.plan = plan( decision.key ); decision.skip = ( decision.plan.what == verdict::skip ); } return decision; } - // a heightfield the scenery names itself, cooked by hand, replaces the world space - // triangles outright, as it always has + // a terrain the scenery names itself replaces the world space triangles outright, as it + // always has decision.skip = ( true == Worldspace ) && ( false == Global.terrain_heightfields.empty() ); return decision; } +// What is ground and what merely stands on it. +// +// The cook lays a triangle whichever way it faces: the side of an embankment or a cutting stands +// upright and is ground all the same, and leaving those out is what opens a hole along every bank. +// So orientation cannot be the test. Two things the scenery says about a node serve instead. +// +// A node under origin, rotate or scale is an object the scenery placed, not ground it drew, and +// examine() keeps those out. And a material with an alpha channel is a tree, a bush, a fence or a +// catenary mast: the ground is opaque. That matters twice over - the terrain is drawn without an +// alpha test, so a billboard would come out as a full sheet, and every level above the finest is one +// simplified surface, which cannot be fitted through the ground and through what stands on it at +// once. void -add( std::uint64_t const Key, std::vector const &Vertices, std::string_view const Material ) { +add( + std::uint64_t const Key, std::vector const &Vertices, std::string_view const Material, + bool const Translucent ) { if( false == state.collecting ) { return; } auto const count { static_cast( Vertices.size() / 3 ) }; - auto const node { static_cast( state.nodes.size() ) }; - state.nodes.push_back( { Key, state.triangles, count, std::string { Material } } ); + if( count == 0 ) { return; } + if( true == Translucent ) { + // no record of it, and a node the note says nothing about is one the scenery draws as written + ++state.objects; + state.objecttriangles += count; + return; + } + state.nodes.push_back( { Key, state.triangles, count } ); - spilled triangle {}; - triangle.node = node; + auto const corner { + []( world_vertex const &Vertex ) { + return quantizedmesh::cooker::corner { + Vertex.position.x, Vertex.position.y, Vertex.position.z, + Vertex.texture.x, Vertex.texture.y }; } }; for( std::uint32_t index = 0; index < count; ++index ) { - for( std::size_t corner = 0; corner < 3; ++corner ) { - auto const &source { Vertices[ index * 3u + corner ] }; - auto *values { triangle.corners + corner * 5 }; - values[ 0 ] = source.position.x; - values[ 1 ] = source.position.y; - values[ 2 ] = source.position.z; - values[ 3 ] = source.texture.x; - values[ 4 ] = source.texture.y; - // the grid step is a property of the whole terrain; the first vertices that turn up - // are enough to tell it - if( state.sample.size() < samplegoal ) { - state.sample.push_back( { values[ 0 ], values[ 1 ], values[ 2 ], values[ 3 ], values[ 4 ] } ); - } - } - triangle.id = state.triangles + index; - state.spill.write( reinterpret_cast( &triangle ), sizeof( triangle ) ); + state.cooker.add( + corner( Vertices[ index * 3u + 0 ] ), + corner( Vertices[ index * 3u + 1 ] ), + corner( Vertices[ index * 3u + 2 ] ), + Material ); } state.triangles += count; } @@ -485,117 +375,86 @@ finish( std::string const &Sceneryfile, std::vector const &Included if( true == planning() ) { WriteLog( - "Terrain bake: " + std::to_string( state.skipped ) + " triangle nodes left to the heightfield, " - + std::to_string( state.filtered ) + " drawn in part, " + std::to_string( state.drawn ) + " drawn as written" ); - // an include added or taken away since the bake: nodes of a new file were drawn as - // geometry this time, as none of their keys is known, but the next start bakes again - if( false == cookdeps::lists( manifest_path( heightfield_path( Sceneryfile ) ), Included ) ) { - WriteLog( "Terrain bake: the scenery includes different files than when it was baked; baking again on the next start" ); + "Terrain cook: " + std::to_string( state.skipped ) + " triangle nodes left to the tiles, " + + std::to_string( state.filtered ) + " drawn in part, " + std::to_string( state.drawn ) + + " drawn as written" ); + // an include added or taken away since the cook: the nodes of a new file were drawn as + // geometry this time, since none of their keys is known, but the next start cooks again + if( false == cookdeps::lists( manifest_path( Sceneryfile ), Included ) ) { + WriteLog( "Terrain cook: the scenery includes different files than when it was cooked; cooking again on the next start" ); discard( Sceneryfile ); } + return; } if( false == state.collecting ) { return; } + state.collecting = false; if( false == Global.terrain_heightfields.empty() ) { - // the scenery named its own heightfield part way through, and that one wins - WriteLog( "Terrain bake: the scenery names its own heightfield, nothing baked" ); - state.spill.close(); - std::error_code error; - std::filesystem::remove( state.spillpath, error ); - state = {}; + WriteLog( "Terrain cook: the scenery names a terrain of its own, nothing cooked" ); return; } - - state.spill.close(); - auto const cleanup = []() { - std::error_code error; - std::filesystem::remove( state.spillpath, error ); - state = {}; - }; if( state.triangles == 0 ) { - WriteLog( "Terrain bake: the scenery holds no triangles, nothing baked" ); - cleanup(); + WriteLog( "Terrain cook: no ground in this scenery, nothing cooked" ); return; } - auto const step { terrain::detect_step( state.sample ) }; - WriteLog( "Terrain bake: grid step " + std::to_string( step ) + " m, from " + std::to_string( state.sample.size() ) + " sampled vertices" ); + // the cook itself, apart from the load that fed it: what a second start saves is the cook + auto const cookstarted { std::chrono::steady_clock::now() }; + auto const archive { archive_path( Sceneryfile ) }; + pak::writer writer; + if( false == writer.open( archive ) ) { + ErrorLog( "Terrain cook: cannot write \"" + archive + "\"" ); + return; + } - // The heightfield should hold the triangles it replaces and nothing else: anything it - // holds that is also drawn as geometry is drawn twice, and a roof or a bridge deck left in - // it bends the ground. But a replaced triangle is only drawn whole if every cell around it - // is, and those cells have corners owned by neighbours that are not replaced - the edge of - // the ground, a slope too steep, a patch too small. So the first pass lays everything and - // decides what is replaced; the second lays the replaced triangles together with the - // neighbours that support them, and that is what gets written. The replaced triangles - // stay replaced in it: their supports are all there, and they have fewer rivals than - // before. The supports are drawn both ways, and the depth offset lets the geometry win. - std::vector ownfield( state.triangles, 0u ); - auto first { lay( state.spillpath, step, std::vector( state.triangles, 0xffu ), &ownfield ) }; - // per triangle, the fields it goes into on the second pass: those that show it, and those - // where it supports a triangle they show - std::vector keep( state.triangles, 0u ); - state.shown.assign( state.triangles, 0u ); - for( std::uint32_t field = 0; field < fieldcount; ++field ) { - first[ field ]->settle(); - for( std::uint32_t id = 0; id < state.triangles; ++id ) { - if( true == first[ field ]->represented( id ) ) { - state.shown[ id ] = 1u; - keep[ id ] |= static_cast( 1u << field ); - } - } + auto written { true }; + quantizedmesh::terrain_table table; + auto const sink { + [ &writer, &written ]( quantizedmesh::tile_address const &Tile, std::vector const &Bytes ) { + written = writer.add( quantizedmesh::tile_name( Tile ), Bytes ) && written; } }; + + if( ( false == state.cooker.finish( sink, table ) ) || ( false == written ) ) { + ErrorLog( "Terrain cook: cooking failed, no tiles written" ); + writer.close(); + discard( Sceneryfile ); + return; } - // a triangle no field shows, counted by the reason its own field gives - terrain::cooker::refusals refused; - for( std::uint32_t id = 0; id < state.triangles; ++id ) { - if( state.shown[ id ] == 0 ) { first[ ownfield[ id ] ]->refusal_of( id, refused ); } + if( false == writer.add( quantizedmesh::tablename, quantizedmesh::write_table( table ) ) ) { + written = false; } + if( ( false == writer.close() ) || ( false == written ) ) { + ErrorLog( "Terrain cook: writing \"" + archive + "\" failed" ); + discard( Sceneryfile ); + return; + } + + auto const &report { state.cooker.result() }; + if( false == save_note( note_path( Sceneryfile ), state.cooker.refusals() ) ) { + ErrorLog( "Terrain cook: cannot write \"" + note_path( Sceneryfile ) + "\"" ); + discard( Sceneryfile ); + return; + } + if( false == cookdeps::write( manifest_path( Sceneryfile ), Included ) ) { + discard( Sceneryfile ); + return; + } + + auto const now { std::chrono::steady_clock::now() }; + auto const cooking { std::chrono::duration( now - cookstarted ).count() }; + auto const collecting { std::chrono::duration( now - state.started ).count() }; WriteLog( - "Terrain bake: not replaced - " + std::to_string( refused.upright ) + " standing on edge, " - + std::to_string( refused.unreached ) + " too small, " + std::to_string( refused.outvoted ) + " under other surfaces, " - + std::to_string( refused.bordering ) + " next to a hole" ); - std::vector support( state.triangles, 0u ); - for( std::uint32_t field = 0; field < fieldcount; ++field ) { - std::fill( support.begin(), support.end(), 0u ); - first[ field ]->mark_support( support ); - first[ field ].reset(); - for( std::uint32_t id = 0; id < state.triangles; ++id ) { - if( support[ id ] != 0 ) { keep[ id ] |= static_cast( 1u << field ); } - } - } - auto const shown { static_cast( std::count( state.shown.begin(), state.shown.end(), 1u ) ) }; - std::size_t supports { 0 }; - for( std::uint32_t id = 0; id < state.triangles; ++id ) { - if( ( keep[ id ] != 0 ) && ( state.shown[ id ] == 0 ) ) { ++supports; } - } + "Terrain cook: " + std::to_string( state.triangles ) + " triangles of ground in " + + std::to_string( state.nodes.size() ) + " nodes, " + std::to_string( state.objecttriangles ) + + " triangles in " + std::to_string( state.objects ) + " nodes left standing on it, into " + + std::to_string( report.tiles ) + " tiles of " + std::to_string( static_cast( state.cooker.finesttile() ) ) + + " m over " + std::to_string( report.levels ) + " levels, " + + std::to_string( report.triangles ) + " triangles and " + std::to_string( report.vertices ) + + " vertices written, " + std::to_string( report.refused ) + " triangles left to the scenery, " + + std::to_string( writer.bytes() / 1024 ) + " kB, cooked in " + + std::to_string( static_cast( cooking ) ) + " s of a " + + std::to_string( static_cast( collecting ) ) + " s load" ); - auto const second { lay( state.spillpath, step, keep ) }; - std::string written; - std::error_code error; - std::uintmax_t bytes { 0 }; - for( std::uint32_t field = 0; field < fieldcount; ++field ) { - auto const path { heightfield_path( Sceneryfile, field ) }; - std::filesystem::remove( path, error ); - if( true == second[ field ]->empty() ) { continue; } - second[ field ]->finish( path, {} ); - auto const size { std::filesystem::file_size( path, error ) }; - bytes += ( error ? 0 : size ); - written += - ( written.empty() ? "" : ", " ) + ( "\"" + path + "\" at " ) - + std::to_string( step * std::pow( fieldstride, field ) ) + " m"; - } - write_table( table_path( heightfield_path( Sceneryfile ) ) ); - cookdeps::write( manifest_path( heightfield_path( Sceneryfile ) ), Included ); - - auto const seconds { - std::chrono::duration( std::chrono::steady_clock::now() - state.started ).count() }; - WriteLog( - "Terrain bake: wrote " + written + "; " + std::to_string( shown ) + " of " + std::to_string( state.triangles ) - + " triangles replaced and " + std::to_string( supports ) + " more supporting them, " - + std::to_string( seconds ) + " s, " + std::to_string( bytes / 1048576 ) + " MB" ); - - // the triangles of this load were all drawn; the table applies from the next one - cleanup(); + Global.terrain_heightfields.push_back( archive ); } } // namespace simulation::terrainbake diff --git a/simulation/terrainbake.cppm b/simulation/terrainbake.cppm index fe40ca75..c0cde266 100644 --- a/simulation/terrainbake.cppm +++ b/simulation/terrainbake.cppm @@ -20,66 +20,54 @@ import eu07.model.vertex; export { -// Baking a scenery's heightfield on its first run, and deciding on every later run which -// triangles it replaces. +// Cooking a scenery's terrain into mesh tiles on its first run, and leaving out the triangles +// the tiles stand for on every run after that. // -// A cooked heightfield is what makes a large scenery affordable - l204 loads in 25 seconds -// against 219, in 1.9 GB against 9.1 - but until now it had to be produced by hand with the -// terraincook tool and named in the scenery text. This bakes it the first time the scenery is -// opened and picks it up by itself afterwards. -// -// Nothing here guesses what terrain is. Every triangle the scenery draws is laid into the -// heightfield where it is drawn, and afterwards each one is asked whether the heightfield still -// shows it. A tree, a wall or a gantry stands on its edge and never reaches a sample; a bridge -// deck loses its samples to the ground beneath; those are drawn as they always were. Only -// triangles the heightfield really shows stop being drawn on their own, so nothing can vanish -// from the scenery - the worst a bad bake can do is draw something twice. -// -// A bake is used only while every file the scenery included when it was made still has the -// content it had (cookdeps): the scenario file itself is not among them, since the launcher -// rewrites it on every start and the terrain lives in what it includes. Within a bake that -// holds, a triangle node is recognised by the file and line it is written at, the parameters -// its include was given, and the origin and rotation in force. -// -// The rasterising is scene/terraincooker.h, the same code the tool uses. +// The first load lays every triangle of ground into the cook and writes the tiles beside the +// scenery, along with a note of which nodes went into them and of what the scenery looked like +// at the time. Later loads read that note: a node the tiles stand for is not built at all, one +// the tiles stand for in part is built without those triangles, and if anything the scenery is +// made of has changed since, the lot is cooked again. namespace simulation::terrainbake { -// what to do with one triangle node enum class verdict { - draw, // the heightfield does not show it: import and draw it as written - skip, // the heightfield shows all of it: do not even import it - filter // the heightfield shows some of it: import it and draw only the rest + draw, // the tiles do not stand for this node: build it as written + skip, // the tiles stand for all of it + filter // the tiles stand for part of it; build the rest }; struct node_plan { verdict what { verdict::draw }; - // for filter: one entry per triangle in import order, true where it is still drawn + // per triangle of the node, whether the scenery still draws it. only for verdict::filter std::vector drawn; }; - -// called before the scenery is parsed. picks up a heightfield baked earlier together with its -// node table, or arms the bake when there is none -void begin( std::string const &Sceneryfile ); -// what the load is to do with one triangle node, decided in one place: whether it is read at -// all, whether it goes into a bake, and which of its triangles are still drawn struct node_decision { - bool skip { false }; // step over it without importing - bool bake { false }; // hand it to add() once imported + bool skip { false }; + bool bake { false }; std::uint64_t key { 0 }; node_plan plan; }; -// Worldspace says whether the node stands in world coordinates, which is all that decides it -// when the scenery names a hand-cooked heightfield of its own +// picks up tiles cooked earlier, or arms the cook when there are none +void begin( std::string const &Sceneryfile ); +// whether triangles are being collected for a cook +bool collecting(); + +// what to do with one triangle node of the scenery node_decision examine( std::string const &File, std::size_t const Line, std::vector const &Parameters, std::string_view const Type, glm::dvec3 const &Offset, glm::vec3 const &Rotation, bool const Worldspace ); -// one triangle node, in world space, as it is about to be drawn -void add( std::uint64_t const Key, std::vector const &Vertices, std::string_view const Material ); -// called once the scenery has been parsed, with every file it included. writes the -// heightfields, their node table and the manifest of what they were made from; or, when a -// bake was used, checks the scenery still includes what that bake was made from + +// hands a node's triangles to the cook, where they are placed in the world. a material with an +// alpha channel is not ground - it is a tree, a bush, a fence or a catenary mast - and is left to +// the scenery to draw +void add( + std::uint64_t const Key, std::vector const &Vertices, std::string_view const Material, + bool const Translucent ); + +// writes the tiles, or reports on what the tiles cooked earlier stood for. Included is every +// file the scenery read, which is what tells a later load whether it is still the same scenery void finish( std::string const &Sceneryfile, std::vector const &Included ); } // namespace simulation::terrainbake diff --git a/tools/terraincook/CMakeLists.txt b/tools/terraincook/CMakeLists.txt index d9f5608a..cc4f01b8 100644 --- a/tools/terraincook/CMakeLists.txt +++ b/tools/terraincook/CMakeLists.txt @@ -13,19 +13,28 @@ if(NOT zstd_FOUND) add_library(zstd::libzstd ALIAS PkgConfig::ZSTD) endif() -find_package(CDT CONFIG REQUIRED) - -add_executable(terraincook +add_executable(terraincook main.cpp - ../scene/terraincooker_tin.cpp + ../../scene/quantizedmeshcooker.cpp ) +# what the cook promises: a tile reads back as it was written, neighbouring tiles share their border +# whatever level each is drawn at, a hole stays a hole and a wall is laid once +add_executable(meshtest + meshtest.cpp + ../../scene/quantizedmeshcooker.cpp +) +target_include_directories(meshtest PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../..") +target_link_libraries(meshtest PRIVATE zstd::libzstd) +if(NOT MSVC) + target_compile_options(meshtest PRIVATE -O2 -Wall) +endif() +add_test(NAME meshtest COMMAND meshtest) +enable_testing() + # the on-disk format lives with the engine, so the tool and the reader cannot drift apart target_include_directories(terraincook PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../..") -target_link_libraries(terraincook PRIVATE - zstd::libzstd - CDT::CDT -) +target_link_libraries(terraincook PRIVATE zstd::libzstd) if(NOT MSVC) target_compile_options(terraincook PRIVATE -O2 -Wall) diff --git a/tools/terraincook/main.cpp b/tools/terraincook/main.cpp index 5704ec65..44411ccb 100644 --- a/tools/terraincook/main.cpp +++ b/tools/terraincook/main.cpp @@ -33,8 +33,8 @@ http://mozilla.org/MPL/2.0/. #include "scene/heightfieldformat.h" #include "scene/heightfieldreader.h" #include "scene/terraincooker.h" -#include "scene/terraincooker_tin.h" -#include "scene/quantizedmeshformat.h" +#include "scene/quantizedmeshcooker.h" +#include "scene/quantizedmeshreader.h" namespace { @@ -570,18 +570,14 @@ main( int argc, char *argv[] ) { std::filesystem::path outputdirectory { "." }; double step { 0.0 }; bool cooking { false }; - double tin_tilesize { 256.0 }; - std::uint32_t tin_lodlevels { 4 }; - bool compress { true }; + std::filesystem::path archive { "terrain.pak" }; for( auto index { 1 }; index < argc; ++index ) { std::string const argument { argv[ index ] }; if( argument == "-step" && index + 1 < argc ) { step = std::strtod( argv[ ++index ], nullptr ); continue; } if( argument == "-out" && index + 1 < argc ) { outputdirectory = argv[ ++index ]; continue; } if( argument == "-cook" ) { cooking = true; continue; } - if( argument == "-tilesize" && index + 1 < argc ) { tin_tilesize = std::strtod( argv[ ++index ], nullptr ); continue; } - if( argument == "-lodlevels" && index + 1 < argc ) { tin_lodlevels = static_cast( std::strtoul( argv[ ++index ], nullptr, 10 ) ); continue; } - if( argument == "-raw" ) { compress = false; continue; } + if( argument == "-archive" && index + 1 < argc ) { archive = argv[ ++index ]; continue; } if( argument == "-tracks" && index + 1 < argc ) { std::filesystem::path const trackpath { argv[ ++index ] }; if( std::filesystem::is_directory( trackpath ) ) { @@ -639,11 +635,16 @@ main( int argc, char *argv[] ) { std::array bandedgesum {}; std::array bandfine {}; - terrain::tin_cooker tin_cooker; + // the same cooker the engine's first-run cook uses, so the two cannot drift apart + quantizedmesh::cooker cooker; if( cooking ) { - tin_cooker.configure( tin_tilesize, tin_lodlevels ); - std::printf( "TIN cooking enabled: tile size %.0f m, %u LOD levels\n", - tin_tilesize, tin_lodlevels ); + auto const scratch { ( outputdirectory / "terrain.cook" ).string() }; + std::filesystem::create_directories( outputdirectory ); + if( false == cooker.begin( scratch ) ) { + std::cerr << "cannot write " << scratch << "\n"; + return 1; + } + std::printf( "cooking terrain mesh tiles into %s\n", ( outputdirectory / archive ).string().c_str() ); } statistics stats; @@ -663,7 +664,13 @@ main( int argc, char *argv[] ) { parse( content, [ & ]( vertex const &A, vertex const &B, vertex const &C, std::string_view Material ) { - if( cooking ) { tin_cooker.rasterize( A, B, C, Material ); } + if( cooking ) { + auto const corner = []( vertex const &Vertex ) { + return quantizedmesh::cooker::corner { + Vertex.x, Vertex.y, Vertex.z, + static_cast( Vertex.u ), static_cast( Vertex.v ) }; }; + cooker.add( corner( A ), corner( B ), corner( C ), Material ); + } ++stats.triangles; stats.vertices += 3; vertex const points[ 3 ] { A, B, C }; @@ -876,31 +883,42 @@ main( int argc, char *argv[] ) { writepreview( outputdirectory / "overlay.pgm", overlaypreview, previewwidth, previewheight, -1.0f ); if( cooking ) { - std::filesystem::create_directories( outputdirectory ); - std::printf( "\n-- generating TIN tiles\n" ); - - auto tiles = tin_cooker.finish(); - - std::printf( "Writing %zu tiles...\n", tiles.size() ); - std::size_t written = 0; - - for( auto const &tile : tiles ) { - for( std::uint32_t lod = 0; lod < tile.lods.size(); ++lod ) { - char filename[ 256 ]; - std::snprintf( filename, sizeof( filename ), "terrain_%d_%d_lod%u.qm", - tile.x, tile.z, lod ); - - std::filesystem::path const filepath = outputdirectory / filename; - - if( terrain::tin_cooker::write_tile( tile, lod, filepath.string(), compress ) ) { - ++written; - } else { - std::fprintf( stderr, "Failed to write %s\n", filename ); - } - } + std::printf( "\n-- cooking terrain mesh tiles\n" ); + + pak::writer writer; + auto const archivepath { ( outputdirectory / archive ).string() }; + if( false == writer.open( archivepath ) ) { + std::cerr << "cannot write " << archivepath << "\n"; + return 1; + } + auto written { true }; + quantizedmesh::terrain_table table; + auto const sink { + [ &writer, &written ]( + quantizedmesh::tile_address const &Tile, std::vector const &Bytes ) { + written = writer.add( quantizedmesh::tile_name( Tile ), Bytes ) && written; } }; + if( ( false == cooker.finish( sink, table ) ) || ( false == written ) ) { + std::cerr << "cooking failed\n"; + return 1; + } + written = writer.add( quantizedmesh::tablename, quantizedmesh::write_table( table ) ) && written; + if( ( false == writer.close() ) || ( false == written ) ) { + std::cerr << "cannot finish " << archivepath << "\n"; + return 1; + } + + auto const &report { cooker.result() }; + std::printf( " %zu triangles laid in %zu tiles of %.0f m over %u levels\n", + report.triangles, report.tiles, cooker.finesttile(), report.levels ); + std::printf( " %zu vertices, %llu kB, %zu triangles refused\n", + report.vertices, static_cast( writer.bytes() / 1024 ), report.refused ); + for( std::uint32_t level = 0; level < table.levels; ++level ) { + std::printf( " level %u: tile %.0f m, off by %.3f m\n", + level, table.tilesize( level ), table.error( level ) ); + } + for( auto const &material : table.materials ) { + std::printf( " material %-40s one repeat every %6.2f m\n", material.name.c_str(), material.repeat ); } - - std::printf( "Wrote %zu quantized mesh files to %s\n", written, outputdirectory.string().c_str() ); } std::printf( "\n previews written to %s (%zux%zu, 1 px = %.0f m)\n", diff --git a/tools/terraincook/meshtest.cpp b/tools/terraincook/meshtest.cpp new file mode 100644 index 00000000..1777b27e --- /dev/null +++ b/tools/terraincook/meshtest.cpp @@ -0,0 +1,221 @@ +// What the cook promises, checked on a terrain made up for the purpose: a tile reads back as it was +// written, two tiles sharing a border agree on it whatever level each is drawn at, a hole the scenery +// left stays a hole, a wall is ground and is laid exactly once, and the tiling of a material is +// measured off the source rather than guessed. +// +// Run it with no arguments; it writes its files to a temporary directory and removes them. +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "scene/quantizedmeshcooker.h" +#include "scene/quantizedmeshreader.h" + +namespace qm = quantizedmesh; + +int failures = 0; +void check( bool ok, char const *what ) { + if( !ok ) { std::printf( "FAIL: %s\n", what ); ++failures; } +} + +int main() { + auto const directory = std::filesystem::temp_directory_path() / "eu07-meshtest"; + std::error_code ignored; + std::filesystem::create_directories( directory, ignored ); + std::string const scratch = ( directory / "terrain.cook" ).string(); + std::string const archive = ( directory / "terrain.pak" ).string(); + + qm::cooker cooker; + cooker.quiet(); + check( cooker.begin( scratch, 2000 ), "begin" ); + + auto const height = []( double x, double z ) { + return 20.0 * std::sin( x / 300.0 ) * std::cos( z / 250.0 ) + 0.01 * x; + }; + // a 1200 x 1200 m field on a 10 m grid, with a square hole and one big triangle + double const step = 10.0; + int const cells = 120; + for( int ix = 0; ix < cells; ++ix ) { + for( int iz = 0; iz < cells; ++iz ) { + double const x0 = -600.0 + ix * step, x1 = x0 + step; + double const z0 = -600.0 + iz * step, z1 = z0 + step; + // a hole in the middle, as a scenery leaves where water or a building is + if( ( x0 >= -100.0 ) && ( x1 <= 100.0 ) && ( z0 >= -100.0 ) && ( z1 <= 100.0 ) ) { continue; } + char const *material = ( x0 < 0.0 ) ? "grass" : "gravel"; + auto const corner = [&]( double x, double z ) { + return qm::cooker::corner{ x, height( x, z ), z, + static_cast( x / 25.0 ), static_cast( z / 25.0 ) }; + }; + cooker.add( corner( x0, z0 ), corner( x1, z0 ), corner( x1, z1 ), material ); + cooker.add( corner( x0, z0 ), corner( x1, z1 ), corner( x0, z1 ), material ); + } + } + // an upright wall, as the side of an embankment is. it is ground too, and must be laid - once, + // although it lies exactly along the border between two tiles (z = -512) + for( int piece = 0; piece < 4; ++piece ) { + double const xa = 100.0 + piece * 20.0, xb = xa + 20.0; + cooker.add( { xa, 0.0, -512.0, 0.f, 0.f }, { xb, 0.0, -512.0, 1.f, 0.f }, { xb, 8.0, -512.0, 1.f, 1.f }, "wall" ); + cooker.add( { xa, 0.0, -512.0, 0.f, 0.f }, { xb, 8.0, -512.0, 1.f, 1.f }, { xa, 8.0, -512.0, 0.f, 1.f }, "wall" ); + } + // one big triangle spanning many tiles, as 3ds Max terrain has + cooker.add( { -600.0, -5.0, 700.0, 0.f, 0.f }, { 600.0, -5.0, 700.0, 24.f, 0.f }, { 0.0, -5.0, 1100.0, 12.f, 16.f }, "grass" ); + + pak::writer writer; + check( writer.open( archive ), "pak open" ); + qm::terrain_table table; + std::map written; + auto const sink = [&]( qm::tile_address const &tile, std::vector const &bytes ) { + written.emplace( qm::tile_key( tile ), tile ); + check( writer.add( qm::tile_name( tile ), bytes ), "pak add" ); + }; + check( cooker.finish( sink, table ), "finish" ); + auto const tabletext = qm::write_table( table ); + check( writer.add( qm::tablename, tabletext ), "table add" ); + check( writer.close(), "pak close" ); + + auto const &report = cooker.result(); + std::printf( "added %zu, laid %zu triangles in %zu tiles, %u levels, %zu vertices, %zu kB, refused %zu\n", + cooker.added(), report.triangles, report.tiles, report.levels, report.vertices, + report.bytes / 1024, report.refused ); + check( cooker.refusals().empty(), "nothing refused: a wall is ground too" ); + check( report.levels >= 3, "quadtree deeper than the root" ); + + qm::reader reader; + check( reader.open( archive ), "reader open" ); + check( reader.table().materials.size() == 3, "grass, gravel and the wall" ); + for( auto const &material : reader.table().materials ) { + std::printf( " material %s, one repeat every %.2f m\n", material.name.c_str(), material.repeat ); + } + for( auto const &material : reader.table().materials ) { + if( material.name != "wall" ) { check( std::abs( material.repeat - 25.0 ) < 0.5, "the tiling was measured off the source" ); } + } + check( reader.table().levels == report.levels, "levels agree" ); + for( std::uint32_t level = 0; level < reader.table().levels; ++level ) { + std::printf( " level %u error %.3f m, tile %.0f m\n", level, + reader.table().error( level ), reader.table().tilesize( level ) ); + } + check( reader.table().error( reader.table().levels - 1 ) == 0.f, "the finest level is the ground as drawn" ); + check( reader.table().error( 0 ) > reader.table().error( reader.table().levels - 2 ), "a coarser level is further off" ); + + // every tile reads back, and its vertices lie inside its own square + // per level and border point, the heights each tile put there. with walls in the mesh one + // point can carry several heights, so every height of one tile has to find a match in the + // other rather than the lists being compared one to one + std::map, std::map>> bordery; + std::size_t read = 0, trianglesread = 0; + std::map perlevel; + for( auto const &[ key, tile ] : written ) { + qm::reader::tile_data data; + if( !reader.read_tile( tile, data, 0.0, 0.0, 0.0 ) ) { std::printf( "FAIL: read %s\n", qm::tile_name( tile ).c_str() ); ++failures; continue; } + ++read; + trianglesread += data.indices.size() / 3; + perlevel[ tile.level ] += data.indices.size() / 3; + auto const side = reader.table().tilesize( tile.level ); + auto const west = reader.table().west + tile.x * side; + auto const north = reader.table().north + tile.z * side; + for( auto const &v : data.vertices ) { + double const x = data.originx + v.x, z = data.originz + v.z, y = data.originy + v.y; + check( x >= west - 0.01 && x <= west + side + 0.01, "vertex inside its tile in x" ); + check( z >= north - 0.01 && z <= north + side + 0.01, "vertex inside its tile in z" ); + check( y >= data.lowest - 0.01 && y <= data.highest + 0.01, "vertex inside its height range" ); + // remember heights on tile borders, to compare with the neighbour later + auto const onborder = std::abs( x - west ) < 0.01 || std::abs( x - west - side ) < 0.01 + || std::abs( z - north ) < 0.01 || std::abs( z - north - side ) < 0.01; + if( onborder ) { + auto const gx = static_cast( std::llround( x * 100.0 ) ); + auto const gz = static_cast( std::llround( z * 100.0 ) ); + bordery[ { tile.level, gx, gz } ][ key ].push_back( y ); + } + } + } + std::printf( "read %zu tiles, %zu triangles\n", read, trianglesread ); + for( auto const &[ level, count ] : perlevel ) { std::printf( " level %u: %zu triangles\n", level, count ); } + + // a vertex two tiles of the same level share must decode to the same height in both + // where two tiles meet, a point on the border can carry more than one height - a wall standing + // on the ground is two surfaces at one place - and only one of them belongs to the neighbour. + // What has to agree, and is what keeps the ground from cracking, is the single-surface case + double worst = 0.0; + std::size_t shared = 0, manysurfaced = 0; + for( auto const &[ where, pertile ] : bordery ) { + if( pertile.size() < 2 ) { continue; } + ++shared; + auto simple = true; + for( auto const &[ which, heights ] : pertile ) { if( heights.size() != 1 ) { simple = false; } } + if( false == simple ) { ++manysurfaced; continue; } + auto const first = pertile.begin()->second.front(); + for( auto const &[ which, heights ] : pertile ) { + worst = std::max( worst, std::abs( heights.front() - first ) ); + } + } + std::printf( "%zu of %zu shared border points carry more than one surface\n", manysurfaced, shared ); + std::printf( "%zu border points shared by two tiles\n", shared ); + std::printf( "worst height disagreement on a shared border: %.4f mm\n", worst * 1000.0 ); + check( worst < 0.01, "shared border vertices agree within a centimetre" ); + + // the coarse levels must actually be coarser + if( perlevel.size() >= 2 ) { + auto const finest = perlevel.rbegin()->second; + auto const next = std::next( perlevel.rbegin() )->second; + check( next < finest, "a coarser level has fewer triangles" ); + } + + // the wall along the tile border must be there, and exactly once: 8 triangles, not 16 + { + std::size_t wallpieces = 0; + std::set> wallcentres; + for( auto const &[ key, tile ] : written ) { + if( tile.level + 1 != reader.table().levels ) { continue; } + qm::reader::tile_data data; + if( !reader.read_tile( tile, data, 0.0, 0.0, 0.0 ) ) { continue; } + for( std::size_t i = 0; i + 2 < data.indices.size(); i += 3 ) { + bool wall = true; + for( int c = 0; c < 3; ++c ) { + auto const &v = data.vertices[ data.indices[ i + c ] ]; + if( std::abs( data.originz + v.z + 512.0 ) > 0.01 ) { wall = false; } + } + if( wall ) { + ++wallpieces; + double cx = 0.0, cy = 0.0; + for( int c = 0; c < 3; ++c ) { + cx += data.originx + data.vertices[ data.indices[ i + c ] ].x; + cy += data.originy + data.vertices[ data.indices[ i + c ] ].y; + } + wallcentres.insert( { std::llround( cx * 100.0 ), std::llround( cy * 100.0 ) } ); + } + } + } + std::printf( "wall triangles on the tile border: %zu (%zu distinct)\n", wallpieces, wallcentres.size() ); + check( wallpieces > 0, "the wall on the border was laid" ); + check( wallcentres.size() == wallpieces, "no piece of the wall was laid twice" ); + } + + // a hole stays a hole: nothing may be drawn in the middle of the map at the finest level + std::size_t inhole = 0; + for( auto const &[ key, tile ] : written ) { + if( tile.level + 1 != reader.table().levels ) { continue; } + qm::reader::tile_data data; + if( !reader.read_tile( tile, data, 0.0, 0.0, 0.0 ) ) { continue; } + for( std::size_t i = 0; i + 2 < data.indices.size(); i += 3 ) { + double cx = 0.0, cz = 0.0; + for( int c = 0; c < 3; ++c ) { + cx += data.originx + data.vertices[ data.indices[ i + c ] ].x; + cz += data.originz + data.vertices[ data.indices[ i + c ] ].z; + } + cx /= 3.0; cz /= 3.0; + if( cx > -95.0 && cx < 95.0 && cz > -95.0 && cz < 95.0 ) { ++inhole; } + } + } + std::printf( "triangles inside the hole: %zu\n", inhole ); + check( inhole == 0, "the hole was not filled in" ); + + std::filesystem::remove_all( directory, ignored ); + std::printf( failures ? "\n%d checks failed\n" : "\nall checks passed\n", failures ); + return failures ? 1 : 0; +} diff --git a/tools/terraincook/seamcheck.py b/tools/terraincook/seamcheck.py deleted file mode 100755 index ccf1aea6..00000000 --- a/tools/terraincook/seamcheck.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -"""Checks that terrain tiles at neighbouring levels meet without gaps. - -The renderer picks each tile's level from its distance at the last rescan, and morphs every -vertex towards the next level as the camera moves (rendering/terrainclipmap.vert). Two -neighbours at levels L and L+1 share their edge exactly when, all along it, the finer tile -has finished morphing and the coarser one has not started. This walks random camera -positions, lets the camera drift up to a rescan's distance from where the levels were -chosen, and counts every edge vertex where that does not hold. - -The constants mirror rendering/terrainstatus.cppm; change them together. Exits non-zero -when a gap is found. -""" - -import math -import random -import sys - -# finest ranges tried, in tile sides: terrain_finest_tiles is the minimum, and the screen -# can push it further (terrain_finest_range) -FINEST_TILES = (4.0, 5.7, 11.3) -RESCAN_TILES = 0.25 # terrain_rescan_tiles -MORPH_BAND = 0.2 # terrain_morph_band -LEVELS = 6 - - -def check(tilesize, step, trials, reach, finest_tiles): - finest = finest_tiles * tilesize - - def level_range(level): - return finest * 2 ** level - - def level_for(distance): - if distance <= finest: - return 0 - return min(int(math.log2(distance / finest)) + 1, LEVELS - 1) - - def nearest(x, z, tilex, tilez): - nx = min(max(x, tilex * tilesize), (tilex + 1) * tilesize) - nz = min(max(z, tilez * tilesize), (tilez + 1) * tilesize) - return math.hypot(nx - x, nz - z) - - def morph(level, distance): - if level + 1 >= LEVELS: - return 0.0 - end = level_range(level) - RESCAN_TILES * tilesize - begin = end - MORPH_BAND * level_range(level) - return min(max((distance - begin) / (end - begin), 0.0), 1.0) - - gaps = 0 - example = "" - for _ in range(trials): - scanx, scanz = random.uniform(-5000, 5000), random.uniform(-5000, 5000) - drift, heading = random.uniform(0, RESCAN_TILES * tilesize), random.uniform(0, 2 * math.pi) - camx, camz = scanx + drift * math.cos(heading), scanz + drift * math.sin(heading) - basex, basez = math.floor(camx / tilesize), math.floor(camz / tilesize) - for tilex in range(basex - reach, basex + reach): - for tilez in range(basez - reach, basez + reach): - here = level_for(nearest(scanx, scanz, tilex, tilez)) - for stepx, stepz in ((1, 0), (0, 1)): - there = level_for(nearest(scanx, scanz, tilex + stepx, tilez + stepz)) - if here == there: - continue - if abs(here - there) > 1: - gaps += 1 - continue - fine, coarse = min(here, there), max(here, there) - count = int(tilesize / step) >> fine - for index in range(count + 1): - along = index * tilesize / count - if stepx: - x, z = (tilex + 1) * tilesize, tilez * tilesize + along - else: - x, z = tilex * tilesize + along, (tilez + 1) * tilesize - distance = math.hypot(x - camx, z - camz) - # an odd vertex of the finer tile is not on the coarser grid and has - # to have finished its move; the coarser edge must not be moving yet - if (index % 2 == 1 and morph(fine, distance) < 1.0) or morph(coarse, distance) > 0.0: - gaps += 1 - if not example: - example = (f"camera {camx:.0f},{camz:.0f}, levels {fine}/{coarse}, " - f"distance {distance:.0f} m") - return gaps, example - - -def main(): - random.seed(1) - failed = False - for finest_tiles in FINEST_TILES: - for tilesize, step, trials, reach in ((128, 1.0, 150, 60), (256, 2.0, 150, 60), (12800, 100.0, 30, 8)): - gaps, example = check(tilesize, step, trials, reach, finest_tiles) - print(f"finest {finest_tiles} tiles, tile {tilesize} m, grid {step} m: {gaps} gaps" - + (f" (e.g. {example})" if gaps else "")) - failed = failed or gaps > 0 - return 1 if failed else 0 - - -if __name__ == "__main__": - sys.exit(main())