World Streaming Part 1

CMakeLists.txt

@@ -168,6 +168,7 @@ set(SOURCES
 "input/editorkeyboardinput.cpp"
 "editor/editorSettings.cpp"
 "editor/editorTerrain.cpp"
+"editor/editorTerrainStreamer.cpp"
 "application/editormode.cpp"
 "input/editormouseinput.cpp"
 "application/editoruilayer.cpp"

application/application.cpp

@@ -593,6 +593,11 @@ int eu07_application::run()
 
 		// -------------------------------------------------------------------
 
+		// keep streamed terrain loaded around the active camera in every mode (editor + driver),
+		// once the simulation is live (avoid streaming while the scenery is still loading)
+		if (simulation::is_ready && EditorTerrain.active())
+			EditorTerrain.update(Global.pCamera.Pos);
+
 		m_taskqueue.update();
 		opengl_texture::reset_unit_cache();
 

application/editormode.cpp

@@ -274,9 +274,9 @@ void editor_mode::snap_to_ground(scene::basic_node *node)
         }
 
     // editable terrain patches keep their heightmap on the CPU, so query them directly
-    for (auto const &terrain : m_terrains)
+    for (editor_terrain *terrain : active_terrains())
     {
-        if (!terrain || !terrain->contains(origin.x, origin.z))
+        if (!terrain->contains(origin.x, origin.z))
             continue;
         double const y = terrain->height_at(origin.x, origin.z);
         if (y <= origin.y + epsilon && y > bestY)
@@ -668,10 +668,43 @@ bool editor_mode::update()
 
     simulation::is_ready = true;
 
+    // note: the streamer is advanced centrally in the application main loop (so it runs in every
+    // mode), using the published Global.pCamera; nothing to do here
+
     // continuous terrain sculpting while the left mouse button is held in sculpt mode
     if (m_terrain_sculpt && mouseHold)
         handle_terrain_sculpt(deltarealtime);
 
+    // debounced auto mesh simplification: once sculpting has settled for a short while, simplify
+    // any chunk that was edited. holding the brush keeps the timer reset so we don't churn mid-stroke.
+    if (m_terrain_auto_optimize)
+    {
+        auto const terrains = active_terrains();
+        bool any_dirty = false;
+        for (editor_terrain *terrain : terrains)
+            if (terrain->dirty())
+            {
+                any_dirty = true;
+                break;
+            }
+
+        if (!any_dirty || (m_terrain_sculpt && mouseHold))
+        {
+            m_terrain_idle = 0.0; // actively editing (or nothing pending): hold off
+        }
+        else
+        {
+            m_terrain_idle += deltarealtime;
+            if (m_terrain_idle >= 0.5) // settle time
+            {
+                for (editor_terrain *terrain : terrains)
+                    if (terrain->dirty())
+                        terrain->optimize(m_terrain_simplify_error);
+                m_terrain_idle = 0.0;
+            }
+        }
+    }
+
     // --- ImGuizmo: in-viewport transform gizmo for the selected node ---
     render_gizmo();
 
@@ -733,7 +766,11 @@ void editor_mode::render_terrain_ui()
         glm::dvec3 const center(Camera.Pos.x, static_cast<double>(m_terrain_baseheight), Camera.Pos.z);
         auto terrain = std::make_unique<editor_terrain>();
         if (terrain->create(center, m_terrain_cells, m_terrain_cellsize, std::string(m_terrain_texture)))
+        {
+            if (m_terrain_auto_optimize)
+                terrain->optimize(m_terrain_simplify_error);
             m_terrains.push_back(std::move(terrain));
+        }
         else
             WriteLog("Editor: failed to create terrain", logtype::generic);
     }
@@ -749,6 +786,64 @@ void editor_mode::render_terrain_ui()
                         static_cast<int>(m_terrain_chunks * m_terrain_cells * m_terrain_cellsize),
                         m_terrain_chunks * m_terrain_chunks);
 
+    if (ImGui::Checkbox("Chunk edit mode (LMB add neighbour / Shift = delete)", &m_chunk_edit))
+        if (m_chunk_edit)
+            m_terrain_sculpt = false; // mutually exclusive with sculpting
+    ImGui::Text("Grid chunks: %zu", m_grid_chunks.size());
+
+    ImGui::Separator();
+    ImGui::TextUnformatted("Streaming (open world, follows camera)");
+    ImGui::SetNextItemWidth(120.0f);
+    ImGui::InputInt("Stream radius", &m_stream_radius);
+    m_stream_radius = std::clamp(m_stream_radius, 0, 16);
+    ImGui::Checkbox("Persist edits to disk (16-bit)", &m_stream_persist);
+    bool streaming = m_streamer.active();
+    if (ImGui::Checkbox("Stream terrain", &streaming))
+    {
+        if (streaming)
+        {
+            // per-scenery chunk folder so chunks from different sceneries don't collide
+            std::string scenery = Global.SceneryFile;
+            auto const slash = scenery.find_last_of("/\\");
+            if (slash != std::string::npos)
+                scenery = scenery.substr(slash + 1);
+            auto const dot = scenery.find_last_of('.');
+            if (dot != std::string::npos)
+                scenery = scenery.substr(0, dot);
+            if (scenery.empty())
+                scenery = "default";
+            m_streamer.directory("editor_terrain/" + scenery);
+
+            m_streamer.configure(m_terrain_cells, m_terrain_cellsize, m_stream_radius,
+                                 m_terrain_baseheight, std::string(m_terrain_texture));
+            m_streamer.simplify(m_terrain_auto_optimize, m_terrain_simplify_error);
+            m_streamer.persist(m_stream_persist);
+
+            // hand the authored grid chunks over to streaming: persist them to disk, then drop the
+            // in-memory meshes so the streamer owns residency (it loads them back within the radius)
+            for (auto &entry : m_grid_chunks)
+                if (entry.second)
+                    m_streamer.save_chunk(entry.first.first, entry.first.second, *entry.second);
+            for (auto &entry : m_grid_chunks)
+                if (entry.second)
+                    entry.second->destroy();
+            m_grid_chunks.clear();
+        }
+        else
+        {
+            m_streamer.clear(); // saves modified chunks before dropping them
+        }
+        m_streamer.active(streaming);
+    }
+    if (m_streamer.active())
+    {
+        // radius / simplify / persist are safe to tweak live; chunk size/base are fixed at toggle
+        m_streamer.radius(m_stream_radius);
+        m_streamer.simplify(m_terrain_auto_optimize, m_terrain_simplify_error);
+        m_streamer.persist(m_stream_persist);
+        ImGui::Text("Resident chunks: %zu  (dir: %s)", m_streamer.resident(), m_streamer.directory().c_str());
+    }
+
     ImGui::Text("Patches: %zu", m_terrains.size());
 
     // capture: sample the selected model's geometry into an editable patch and remove the original
@@ -762,10 +857,13 @@ void editor_mode::render_terrain_ui()
         ImGui::TextDisabled("Capture: select a model instance first");
     }
 
-    if (!m_terrains.empty())
+    std::vector<editor_terrain *> const terrains = active_terrains();
+    if (!terrains.empty())
     {
         ImGui::Separator();
-        ImGui::Checkbox("Sculpt mode (LMB raise / Shift = lower)", &m_terrain_sculpt);
+        if (ImGui::Checkbox("Sculpt mode (LMB raise / Shift = lower)", &m_terrain_sculpt))
+            if (m_terrain_sculpt)
+                m_chunk_edit = false; // mutually exclusive with chunk editing
         ImGui::SetNextItemWidth(120.0f);
         ImGui::InputFloat("Brush radius", &m_terrain_brush_radius);
         if (m_terrain_brush_radius < 0.5f)
@@ -773,14 +871,17 @@ void editor_mode::render_terrain_ui()
         ImGui::SetNextItemWidth(120.0f);
         ImGui::InputFloat("Brush strength", &m_terrain_brush_strength);
 
-        // one-shot nudge of the most recent patch at its centre, handy for a quick test
+        // one-shot nudge of the most recent manual patch at its centre, handy for a quick test
-        auto &terrain = m_terrains.back();
+        if (!m_terrains.empty())
-        glm::dvec3 const c = terrain->centre();
+        {
-        if (ImGui::Button("Raise centre"))
+            auto &terrain = m_terrains.back();
-            terrain->sculpt(c.x, c.z, m_terrain_brush_radius, m_terrain_brush_strength);
+            glm::dvec3 const c = terrain->centre();
-        ImGui::SameLine();
+            if (ImGui::Button("Raise centre"))
-        if (ImGui::Button("Lower centre"))
+                terrain->sculpt(c.x, c.z, m_terrain_brush_radius, m_terrain_brush_strength);
-            terrain->sculpt(c.x, c.z, m_terrain_brush_radius, -m_terrain_brush_strength);
+            ImGui::SameLine();
+            if (ImGui::Button("Lower centre"))
+                terrain->sculpt(c.x, c.z, m_terrain_brush_radius, -m_terrain_brush_strength);
+        }
 
         ImGui::Separator();
         ImGui::TextUnformatted("Optimize (mesh simplification, all patches)");
@@ -788,23 +889,21 @@ void editor_mode::render_terrain_ui()
         ImGui::InputFloat("Flatness tol (m)", &m_terrain_simplify_error);
         if (m_terrain_simplify_error < 0.01f)
             m_terrain_simplify_error = 0.01f;
+        ImGui::Checkbox("Auto-optimize after sculpt", &m_terrain_auto_optimize);
         if (ImGui::Button("Optimize all"))
-            for (auto &t : m_terrains)
+            for (editor_terrain *t : terrains)
-                if (t)
+                t->optimize(m_terrain_simplify_error);
-                    t->optimize(m_terrain_simplify_error);
         ImGui::SameLine();
         if (ImGui::Button("Full-res all"))
-            for (auto &t : m_terrains)
+            for (editor_terrain *t : terrains)
-                if (t)
+                t->unoptimize();
-                    t->unoptimize();
 
         std::size_t tris = 0, full = 0;
-        for (auto &t : m_terrains)
+        for (editor_terrain *t : terrains)
-            if (t)
+        {
-            {
+            tris += t->triangles();
-                tris += t->triangles();
+            full += t->full_triangles();
-                full += t->full_triangles();
+        }
-            }
         ImGui::Text("Triangles: %zu / %zu", tris, full);
     }
 }
@@ -814,34 +913,119 @@ editor_terrain *editor_mode::terrain_at(double X, double Z)
     for (auto &terrain : m_terrains)
         if (terrain && terrain->contains(X, Z))
             return terrain.get();
-    return nullptr;
+    double const size = chunk_grid_size();
+    auto const it = m_grid_chunks.find({static_cast<int>(std::floor(X / size)), static_cast<int>(std::floor(Z / size))});
+    if (it != m_grid_chunks.end() && it->second && it->second->contains(X, Z))
+        return it->second.get();
+    return m_streamer.terrain_at(X, Z);
+}
+
+std::vector<editor_terrain *> editor_mode::active_terrains()
+{
+    std::vector<editor_terrain *> out;
+    out.reserve(m_terrains.size() + m_grid_chunks.size() + m_streamer.resident());
+    for (auto &terrain : m_terrains)
+        if (terrain)
+            out.push_back(terrain.get());
+    for (auto &entry : m_grid_chunks)
+        if (entry.second)
+            out.push_back(entry.second.get());
+    m_streamer.collect(out);
+    return out;
+}
+
+void editor_mode::add_grid_chunk(int Cx, int Cz)
+{
+    std::pair<int, int> const key{Cx, Cz};
+    if (m_grid_chunks.count(key))
+        return; // already occupied
+
+    double const size = chunk_grid_size();
+    int const cells = std::clamp(m_terrain_cells, 1, 256);
+    glm::dvec3 const center((Cx + 0.5) * size, static_cast<double>(m_terrain_baseheight), (Cz + 0.5) * size);
+
+    auto terrain = std::make_unique<editor_terrain>();
+    if (!terrain->create(center, cells, m_terrain_cellsize, std::string(m_terrain_texture)))
+        return;
+    if (m_terrain_auto_optimize)
+        terrain->optimize(m_terrain_simplify_error);
+    m_grid_chunks[key] = std::move(terrain);
+}
+
+void editor_mode::remove_grid_chunk(int Cx, int Cz)
+{
+    auto const it = m_grid_chunks.find({Cx, Cz});
+    if (it == m_grid_chunks.end())
+        return;
+    if (it->second)
+        it->second->destroy();
+    m_grid_chunks.erase(it);
+}
+
+void editor_mode::handle_chunk_edit_click(bool DeleteMode)
+{
+    // world point under the cursor; must land on existing geometry to give a valid depth
+    glm::dvec3 const world = Camera.Pos + GfxRenderer->Mouse_Position();
+    double const size = chunk_grid_size();
+    int const cx = static_cast<int>(std::floor(world.x / size));
+    int const cz = static_cast<int>(std::floor(world.z / size));
+    bool const streaming = m_streamer.active();
+
+    if (DeleteMode)
+    {
+        if (streaming)
+            m_streamer.remove_chunk(cx, cz);
+        else
+            remove_grid_chunk(cx, cz);
+        return;
+    }
+
+    // if the clicked cell holds a chunk, target the neighbour nearest the clicked edge (the empty
+    // side); otherwise fill the clicked cell
+    bool const occupied = streaming
+                              ? (m_streamer.terrain_at(world.x, world.z) != nullptr)
+                              : (m_grid_chunks.count({cx, cz}) > 0);
+    int tcx = cx, tcz = cz;
+    if (occupied)
+    {
+        double const lx = world.x - cx * size, lz = world.z - cz * size;
+        double const dw = lx, de = size - lx, dn = lz, ds = size - lz;
+        double const nearest = std::min({dw, de, dn, ds});
+        if (nearest == dw)
+            tcx = cx - 1;
+        else if (nearest == de)
+            tcx = cx + 1;
+        else if (nearest == dn)
+            tcz = cz - 1;
+        else
+            tcz = cz + 1;
+    }
+
+    if (streaming)
+        m_streamer.add_chunk(tcx, tcz);
+    else
+        add_grid_chunk(tcx, tcz);
 }
 
 void editor_mode::create_chunked_terrain()
 {
     int const chunks = std::clamp(m_terrain_chunks, 1, 32);
-    int const cells = std::clamp(m_terrain_cells, 1, 256);
+    double const size = chunk_grid_size();
-    float const cellsize = std::max(0.1f, m_terrain_cellsize);
 
-    double const chunkextent = static_cast<double>(cells) * cellsize;
+    // snap the field to the global chunk grid (so it aligns with manual/streamed chunks), centred
-    double const half = 0.5 * chunks * chunkextent;
+    // on the camera's chunk
-    double const x0 = Camera.Pos.x - half; // corner of the whole field
+    int const ccx = static_cast<int>(std::floor(Camera.Pos.x / size));
-    double const z0 = Camera.Pos.z - half;
+    int const ccz = static_cast<int>(std::floor(Camera.Pos.z / size));
+    int const half = chunks / 2;
 
     int created = 0;
-    for (int cz = 0; cz < chunks; ++cz)
+    for (int dz = 0; dz < chunks; ++dz)
-        for (int cx = 0; cx < chunks; ++cx)
+        for (int dx = 0; dx < chunks; ++dx)
         {
-            // adjacent chunks share edges exactly (aligned grids, identical world coords),
+            int const cx = ccx - half + dx, cz = ccz - half + dz;
-            // so a world-space brush keeps the seams consistent
+            if (!m_grid_chunks.count({cx, cz}))
-            glm::dvec3 const center(
-                x0 + (cx + 0.5) * chunkextent,
-                static_cast<double>(m_terrain_baseheight),
-                z0 + (cz + 0.5) * chunkextent);
-            auto terrain = std::make_unique<editor_terrain>();
-            if (terrain->create(center, cells, cellsize, std::string(m_terrain_texture)))
             {
-                m_terrains.push_back(std::move(terrain));
+                add_grid_chunk(cx, cz);
                 ++created;
             }
         }
@@ -849,6 +1033,45 @@ void editor_mode::create_chunked_terrain()
     WriteLog("Editor: created chunked terrain with " + std::to_string(created) + " chunks", logtype::generic);
 }
 
+void editor_mode::save_scene_with_terrain()
+{
+    // commit authored terrain so the scenery streams it on load. if not already streaming, hand the
+    // manual grid chunks over to the streamer (same as toggling Stream terrain on)
+    if (!m_streamer.active())
+    {
+        std::string scenery = Global.SceneryFile;
+        auto const slash = scenery.find_last_of("/\\");
+        if (slash != std::string::npos)
+            scenery = scenery.substr(slash + 1);
+        auto const dot = scenery.find_last_of('.');
+        if (dot != std::string::npos)
+            scenery = scenery.substr(0, dot);
+        if (scenery.empty())
+            scenery = "default";
+
+        m_streamer.directory("editor_terrain/" + scenery);
+        m_streamer.configure(m_terrain_cells, m_terrain_cellsize, m_stream_radius, m_terrain_baseheight,
+                             std::string(m_terrain_texture));
+        m_streamer.simplify(m_terrain_auto_optimize, m_terrain_simplify_error);
+        m_streamer.persist(true);
+
+        for (auto &entry : m_grid_chunks)
+            if (entry.second)
+                m_streamer.save_chunk(entry.first.first, entry.first.second, *entry.second);
+        for (auto &entry : m_grid_chunks)
+            if (entry.second)
+                entry.second->destroy();
+        m_grid_chunks.clear();
+        m_streamer.active(true);
+    }
+
+    m_streamer.flush(); // save resident edited chunks to disk
+
+    // export scenery; the exported .scm now carries an `editorterrain` directive (streamer is active)
+    simulation::State.export_as_text(Global.SceneryFile);
+    WriteLog("Editor: saved scene + terrain", logtype::generic);
+}
+
 void editor_mode::handle_terrain_sculpt(double Deltatime)
 {
     // world point under the cursor (Mouse_Position is camera-relative, like the brush placement uses)
@@ -861,9 +1084,8 @@ void editor_mode::handle_terrain_sculpt(double Deltatime)
     double const signedrate = (Global.shiftState ? -rate : rate);
     // apply to every chunk the brush touches; each patch clips to its own bounds, so a stroke
     // crossing a chunk boundary edits both and shared-edge vertices stay in sync
-    for (auto &terrain : m_terrains)
+    for (editor_terrain *terrain : active_terrains())
-        if (terrain)
+        terrain->sculpt(world.x, world.z, m_terrain_brush_radius, signedrate);
-            terrain->sculpt(world.x, world.z, m_terrain_brush_radius, signedrate);
 }
 
 void editor_mode::capture_terrain()
@@ -954,8 +1176,8 @@ void editor_mode::capture_terrain()
 
 void editor_mode::render_gizmo()
 {
-    // the transform gizmo is suppressed while sculpting terrain, so the brush owns the mouse
+    // the transform gizmo is suppressed while editing terrain, so the brush/chunk tool owns the mouse
-    if (!m_gizmo_enabled || m_terrain_sculpt)
+    if (!m_gizmo_enabled || m_terrain_sculpt || m_chunk_edit)
     {
         m_gizmo_using = false;
         return;
@@ -1316,6 +1538,15 @@ void editor_mode::on_mouse_button(int const Button, int const Action, int const
         return;
     }
 
+    // in chunk-edit mode the left button adds a neighbouring chunk (Shift = delete the clicked one)
+    if (m_chunk_edit && Button == GLFW_MOUSE_BUTTON_LEFT)
+    {
+        if (is_press(Action))
+            handle_chunk_edit_click(Global.shiftState);
+        m_input.mouse.button(Button, Action);
+        return;
+    }
+
     // in terrain sculpt mode the left button paints the terrain instead of picking nodes
     if (m_terrain_sculpt && Button == GLFW_MOUSE_BUTTON_LEFT)
     {

application/editormode.h

@@ -16,6 +16,7 @@ http://mozilla.org/MPL/2.0/.
 #include "scene/sceneeditor.h"
 #include "scene/scenenode.h"
 #include "editor/editorTerrain.hpp"
+#include "editor/editorTerrainStreamer.hpp"
 
 #include <memory>
 
@@ -146,14 +147,27 @@ class editor_mode : public application_mode
 	void render_terrain_ui();
 	// creates a large terrain as a grid of adjacent chunks (each its own editable patch)
 	void create_chunked_terrain();
+	// manual grid-aligned chunks: add/remove single chunks for fine control
+	float chunk_grid_size() const { return m_terrain_cells * m_terrain_cellsize; }
+	void add_grid_chunk(int Cx, int Cz);
+	void remove_grid_chunk(int Cx, int Cz);
+	// handles a click in chunk-edit mode (add a neighbour, or Shift = delete the clicked chunk)
+	void handle_chunk_edit_click(bool DeleteMode);
+	// commits authored terrain to disk, enables streaming, and exports the scenery (Ctrl+S)
+	void save_scene_with_terrain();
 	// raises/lowers terrain under the cursor while the left mouse button is held in sculpt mode
 	void handle_terrain_sculpt(double Deltatime);
 	// returns the terrain patch (if any) whose footprint covers the given world point
 	editor_terrain *terrain_at(double X, double Z);
+	// gathers every active terrain patch: manually-created ones plus streamed chunks
+	std::vector<editor_terrain *> active_terrains();
 	// samples the selected model instance's geometry into a new editable terrain patch, then removes it
 	void capture_terrain();
 	std::vector<std::unique_ptr<editor_terrain>> m_terrains;
+	// grid-aligned manual chunks, keyed by (cx,cz) on the global chunk grid
+	std::map<std::pair<int, int>, std::unique_ptr<editor_terrain>> m_grid_chunks;
 	bool m_terrain_sculpt{false};     // when true, LMB sculpts terrain instead of picking
+	bool m_chunk_edit{false};         // when true, LMB adds/removes whole chunks
 	int m_terrain_cells{32};          // grid resolution (quads per side)
 	int m_terrain_chunks{4};          // chunks per side for a chunked terrain
 	float m_terrain_cellsize{2.0f};   // metres per quad
@@ -161,8 +175,16 @@ class editor_mode : public application_mode
 	float m_terrain_brush_radius{12.0f};
 	float m_terrain_brush_strength{4.0f}; // metres per second while held (one-shot for the buttons)
 	float m_terrain_simplify_error{0.5f}; // flatness tolerance (m) for mesh simplification
+	bool m_terrain_auto_optimize{false};  // auto-simplify edited chunks after sculpting settles
+	double m_terrain_idle{0.0};           // seconds since the last sculpt edit (debounce timer)
 	char m_terrain_texture[128]{""};  // optional ground texture name
 
+	// streaming terrain that follows the camera (open-world); the editor shares the single
+	// simulation-level instance so authored terrain also renders in the driver / other modes
+	terrain_streamer &m_streamer{EditorTerrain};
+	int m_stream_radius{2};
+	bool m_stream_persist{true}; // save edited chunks to disk and load them back
+
 	// hierarchy management
 	void add_to_hierarchy(scene::basic_node *node);
 	void remove_from_hierarchy(scene::basic_node *node);

editor/editorTerrain.cpp

@@ -237,6 +237,7 @@ void editor_terrain::optimize(float ErrorMetres)
 {
 	m_simplify = true;
 	m_simplify_error = (ErrorMetres > 0.0f ? ErrorMetres : 0.01f);
+	m_dirty = false;
 	regenerate(true);
 }
 
@@ -246,6 +247,28 @@ void editor_terrain::unoptimize()
 	regenerate(false);
 }
 
+void editor_terrain::destroy()
+{
+	if (m_section != nullptr)
+	{
+		// erase the shape whose geometry handle is ours; other shapes keep their handles (so they
+		// keep rendering), and the geometry GC reclaims our now-undrawn chunk's GPU memory
+		for (auto it = m_section->m_shapes.begin(); it != m_section->m_shapes.end(); ++it)
+		{
+			auto const h = it->data().geometry;
+			if (h.bank == m_geometry.bank && h.chunk == m_geometry.chunk)
+			{
+				m_section->m_shapes.erase(it);
+				break;
+			}
+		}
+	}
+	m_section = nullptr;
+	m_geometry = gfx::geometry_handle{0, 0};
+	m_cells = 0; // mark invalid
+	m_heights.clear();
+}
+
 bool editor_terrain::contains(double X, double Z) const
 {
 	double const x1 = m_x0 + static_cast<double>(m_cells) * m_cellsize;
@@ -298,8 +321,10 @@ bool editor_terrain::sculpt(double X, double Z, double Radius, double Strength)
 	if (changed)
 	{
 		// sculpting edits the full-resolution mesh (fixed vertex count => fast in-place update);
-		// the user can re-run optimize() afterwards to simplify again
+		// mark dirty so it can be auto-simplified once the stroke finishes, and modified for saving
 		m_simplify = false;
+		m_dirty = true;
+		m_modified = true;
 		regenerate(false);
 	}
 	return changed;

editor/editorTerrain.hpp

@@ -55,11 +55,25 @@ class editor_terrain
 	// rebuilds the rendered mesh at full resolution (undoes optimize)
 	void unoptimize();
 
+	// removes the rendered shape from its section (stops drawing it); the renderer's geometry
+	// garbage collector reclaims the GPU memory shortly after. used by terrain streaming to unload.
+	void destroy();
+
 	// horizontal centre and extent, handy for the UI / camera framing
 	glm::dvec3 centre() const;
 	float extent() const { return m_cells * m_cellsize; }
 	bool valid() const { return m_cells > 0; }
 	bool optimized() const { return m_simplify; }
+	// set by sculpt() when the mesh changed and is currently full-resolution; cleared by optimize().
+	// used to auto-simplify only the chunks that were actually edited, once a stroke finishes.
+	bool dirty() const { return m_dirty; }
+	// set by sculpt() and never auto-cleared; used by streaming to know a chunk needs saving to disk
+	bool modified() const { return m_modified; }
+	void clear_modified() { m_modified = false; }
+
+	// direct access to the editable heightmap (row-major world Y, (cells+1)^2), for save/load
+	std::vector<float> const &heights() const { return m_heights; }
+	int cells() const { return m_cells; }
 	// rendered triangle count (drops after optimize)
 	std::size_t triangles() const { return m_vertexcount / 3; }
 	// full-resolution triangle count, for reference
@@ -94,4 +108,6 @@ class editor_terrain
 
 	bool m_simplify{false};        // whether the rendered mesh is currently simplified
 	float m_simplify_error{0.5f};  // flatness tolerance used by optimize()
+	bool m_dirty{false};           // edited since the last optimize (full-res, awaiting simplification)
+	bool m_modified{false};        // edited since load/save (for streaming persistence)
 };

editor/editorTerrainStreamer.cpp

@@ -0,0 +1,289 @@
+/*
+This Source Code Form is subject to the
+terms of the Mozilla Public License, v.
+2.0. If a copy of the MPL was not
+distributed with this file, You can
+obtain one at
+http://mozilla.org/MPL/2.0/.
+*/
+
+#include "stdafx.h"
+#include "editor/editorTerrainStreamer.hpp"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <filesystem>
+#include <fstream>
+#include <string>
+
+// simulation-level streamer instance (see header)
+terrain_streamer EditorTerrain;
+
+void terrain_streamer::configure(int Cells, float CellSize, int Radius, float BaseHeight, std::string const &Texture)
+{
+	m_cells = std::max(1, Cells);
+	m_cellsize = std::max(0.1f, CellSize);
+	m_radius = std::max(0, Radius);
+	m_baseheight = BaseHeight;
+	m_texture = Texture;
+}
+
+terrain_streamer::chunk_key terrain_streamer::key_at(double X, double Z) const
+{
+	double const size = chunk_world_size();
+	return {static_cast<int>(std::floor(X / size)), static_cast<int>(std::floor(Z / size))};
+}
+
+glm::dvec3 terrain_streamer::chunk_centre(int Cx, int Cz) const
+{
+	double const size = chunk_world_size();
+	return glm::dvec3((Cx + 0.5) * size, static_cast<double>(m_baseheight), (Cz + 0.5) * size);
+}
+
+std::string terrain_streamer::chunk_path(int Cx, int Cz) const
+{
+	return m_dir + "/chunk_" + std::to_string(Cx) + "_" + std::to_string(Cz) + ".etc";
+}
+
+bool terrain_streamer::chunk_on_disk(chunk_key const &Key)
+{
+	auto const it = m_known.find(Key);
+	if (it != m_known.end())
+		return (it->second & cf_exists_on_disk) != 0;
+
+	std::error_code ec;
+	bool const exists = std::filesystem::exists(chunk_path(Key.first, Key.second), ec);
+	m_known[Key] = exists ? cf_exists_on_disk : 0;
+	return exists;
+}
+
+// 16-bit chunk file: 'ETC1' | uint16 cells | float baseY | float step | (cells+1)^2 * uint16
+// where worldY = baseY + raw * step (per-chunk auto-scaled to fit the height range losslessly-ish)
+bool terrain_streamer::load_heights(int Cx, int Cz, std::vector<float> &Out) const
+{
+	std::ifstream f(chunk_path(Cx, Cz), std::ios::binary);
+	if (!f)
+		return false;
+
+	char magic[4] = {};
+	f.read(magic, 4);
+	if (f.gcount() != 4 || magic[0] != 'E' || magic[1] != 'T' || magic[2] != 'C' || magic[3] != '1')
+		return false;
+
+	std::uint16_t cells = 0;
+	float baseY = 0.0f, step = 0.0f;
+	f.read(reinterpret_cast<char *>(&cells), sizeof(cells));
+	f.read(reinterpret_cast<char *>(&baseY), sizeof(baseY));
+	f.read(reinterpret_cast<char *>(&step), sizeof(step));
+	if (!f || static_cast<int>(cells) != m_cells)
+		return false; // resolution changed since save: ignore and regenerate
+
+	std::size_t const n = static_cast<std::size_t>(cells + 1) * (cells + 1);
+	Out.resize(n);
+	for (std::size_t i = 0; i < n; ++i)
+	{
+		std::uint16_t raw = 0;
+		f.read(reinterpret_cast<char *>(&raw), sizeof(raw));
+		if (!f)
+			return false;
+		Out[i] = baseY + static_cast<float>(raw) * step;
+	}
+	return true;
+}
+
+void terrain_streamer::save_heights(int Cx, int Cz, editor_terrain const &Terrain)
+{
+	auto const &h = Terrain.heights();
+	if (h.empty())
+		return;
+
+	float minh = h[0], maxh = h[0];
+	for (float const v : h)
+	{
+		minh = std::min(minh, v);
+		maxh = std::max(maxh, v);
+	}
+	float const step = (maxh > minh) ? (maxh - minh) / 65535.0f : 0.0f;
+
+	std::error_code ec;
+	std::filesystem::create_directories(m_dir, ec);
+	std::ofstream f(chunk_path(Cx, Cz), std::ios::binary | std::ios::trunc);
+	if (!f)
+		return;
+
+	char const magic[4] = {'E', 'T', 'C', '1'};
+	std::uint16_t const cells = static_cast<std::uint16_t>(Terrain.cells());
+	f.write(magic, 4);
+	f.write(reinterpret_cast<char const *>(&cells), sizeof(cells));
+	f.write(reinterpret_cast<char const *>(&minh), sizeof(minh));
+	f.write(reinterpret_cast<char const *>(&step), sizeof(step));
+	for (float const v : h)
+	{
+		std::uint16_t const raw = (step > 0.0f)
+		                              ? static_cast<std::uint16_t>(std::lround((v - minh) / step))
+		                              : std::uint16_t{0};
+		f.write(reinterpret_cast<char const *>(&raw), sizeof(raw));
+	}
+}
+
+void terrain_streamer::update(glm::dvec3 const &CameraPos)
+{
+	if (!m_active)
+		return;
+
+	chunk_key const camera = key_at(CameraPos.x, CameraPos.z);
+
+	// load any missing chunk inside the radius (one build per frame keeps the hitch small)
+	bool built = false;
+	for (int dz = -m_radius; dz <= m_radius && !built; ++dz)
+		for (int dx = -m_radius; dx <= m_radius && !built; ++dx)
+		{
+			chunk_key const key{camera.first + dx, camera.second + dz};
+			if (m_chunks.count(key))
+				continue;
+
+			// only stream chunks that were actually authored (exist on disk); empty space stays empty
+			std::vector<float> loaded;
+			if (!chunk_on_disk(key) || !load_heights(key.first, key.second, loaded))
+				continue;
+
+			glm::dvec3 const centre = chunk_centre(key.first, key.second);
+			double const size = chunk_world_size();
+			double const x0 = key.first * size, z0 = key.second * size; // chunk corner
+			int const cells = m_cells;
+			float const cs = m_cellsize;
+			editor_terrain::height_sampler sampler =
+			    [&loaded, x0, z0, cells, cs](double X, double Z, double &OutY) -> bool {
+				int ix = static_cast<int>(std::lround((X - x0) / cs));
+				int iz = static_cast<int>(std::lround((Z - z0) / cs));
+				ix = std::clamp(ix, 0, cells);
+				iz = std::clamp(iz, 0, cells);
+				OutY = loaded[static_cast<std::size_t>(iz) * (cells + 1) + ix];
+				return true;
+			};
+
+			auto terrain = std::make_unique<editor_terrain>();
+			if (terrain->create(centre, m_cells, m_cellsize, m_texture, sampler))
+			{
+				if (m_auto_optimize)
+					terrain->optimize(m_simplify_error);
+				m_chunks.emplace(key, std::move(terrain));
+				built = true; // amortise: at most one new chunk per frame
+			}
+		}
+
+	// unload chunks that drifted outside the radius
+	for (auto it = m_chunks.begin(); it != m_chunks.end();)
+	{
+		int const dx = it->first.first - camera.first;
+		int const dz = it->first.second - camera.second;
+		if (std::abs(dx) > m_radius || std::abs(dz) > m_radius)
+		{
+			if (it->second)
+			{
+				// persist edits before dropping the chunk, so they survive the round-trip
+				if (m_persist && it->second->modified())
+				{
+					save_heights(it->first.first, it->first.second, *it->second);
+					m_known[it->first] |= cf_exists_on_disk;
+				}
+				it->second->destroy();
+			}
+			it = m_chunks.erase(it);
+		}
+		else
+			++it;
+	}
+}
+
+void terrain_streamer::clear()
+{
+	for (auto &entry : m_chunks)
+		if (entry.second)
+		{
+			if (m_persist && entry.second->modified())
+			{
+				save_heights(entry.first.first, entry.first.second, *entry.second);
+				m_known[entry.first] |= cf_exists_on_disk;
+			}
+			entry.second->destroy();
+		}
+	m_chunks.clear();
+}
+
+void terrain_streamer::collect(std::vector<editor_terrain *> &Out) const
+{
+	for (auto const &entry : m_chunks)
+		if (entry.second)
+			Out.push_back(entry.second.get());
+}
+
+editor_terrain *terrain_streamer::terrain_at(double X, double Z) const
+{
+	auto const it = m_chunks.find(key_at(X, Z));
+	if (it != m_chunks.end() && it->second && it->second->contains(X, Z))
+		return it->second.get();
+	return nullptr;
+}
+
+void terrain_streamer::save_chunk(int Cx, int Cz, editor_terrain const &Terrain)
+{
+	save_heights(Cx, Cz, Terrain);
+	m_known[{Cx, Cz}] |= cf_exists_on_disk;
+}
+
+void terrain_streamer::add_chunk(int Cx, int Cz)
+{
+	chunk_key const key{Cx, Cz};
+	if (m_chunks.count(key))
+		return; // already resident
+
+	glm::dvec3 const centre = chunk_centre(Cx, Cz);
+	auto terrain = std::make_unique<editor_terrain>();
+	if (!terrain->create(centre, m_cells, m_cellsize, m_texture)) // flat
+		return;
+
+	// persist immediately so it becomes part of the authored, streamable world
+	save_heights(Cx, Cz, *terrain);
+	m_known[key] |= cf_exists_on_disk;
+
+	if (m_auto_optimize)
+		terrain->optimize(m_simplify_error);
+	m_chunks.emplace(key, std::move(terrain));
+}
+
+void terrain_streamer::reset()
+{
+	// do NOT call editor_terrain::destroy() here: on a scenery reload the sections those chunks
+	// referenced are already gone. just drop our bookkeeping; the old region freed the shapes.
+	m_chunks.clear();
+	m_known.clear();
+	m_active = false;
+}
+
+void terrain_streamer::flush()
+{
+	for (auto &entry : m_chunks)
+		if (entry.second && entry.second->modified())
+		{
+			save_heights(entry.first.first, entry.first.second, *entry.second);
+			m_known[entry.first] |= cf_exists_on_disk;
+			entry.second->clear_modified();
+		}
+}
+
+void terrain_streamer::remove_chunk(int Cx, int Cz)
+{
+	chunk_key const key{Cx, Cz};
+	auto const it = m_chunks.find(key);
+	if (it != m_chunks.end())
+	{
+		if (it->second)
+			it->second->destroy();
+		m_chunks.erase(it);
+	}
+	std::error_code ec;
+	std::filesystem::remove(chunk_path(Cx, Cz), ec);
+	m_known[key] = 0; // no longer on disk
+}

editor/editorTerrainStreamer.hpp

@@ -0,0 +1,116 @@
+/*
+This Source Code Form is subject to the
+terms of the Mozilla Public License, v.
+2.0. If a copy of the MPL was not
+distributed with this file, You can
+obtain one at
+http://mozilla.org/MPL/2.0/.
+*/
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <vector>
+#include <string>
+#include <glm/glm.hpp>
+
+#include "editor/editorTerrain.hpp"
+
+// Streams editable terrain chunks around the camera for an effectively unbounded world.
+//
+// Phase 1: keeps a ring of resident chunks (each an editor_terrain mesh) within a radius of the
+// camera; chunks entering the radius are built, chunks leaving it are destroyed (the renderer's
+// geometry GC reclaims their GPU memory). Heights are flat + a gentle procedural roll so streaming
+// is observable. Phase 2 will add 16-bit on-disk chunk paging, per-chunk flag bits and persistence.
+class terrain_streamer
+{
+  public:
+	using chunk_key = std::pair<int, int>; // (cx, cz) chunk coordinate
+
+	terrain_streamer() = default;
+
+	void active(bool State) { m_active = State; }
+	bool active() const { return m_active; }
+
+	// per-chunk configuration (applied to chunks built from now on)
+	void configure(int Cells, float CellSize, int Radius, float BaseHeight, std::string const &Texture);
+	void simplify(bool Auto, float Error) { m_auto_optimize = Auto; m_simplify_error = Error; }
+	// safe to change while resident (only affects how far chunks load/unload)
+	void radius(int Radius) { m_radius = Radius < 0 ? 0 : Radius; }
+	// persist edited chunks to disk on unload, and load them back instead of regenerating
+	void persist(bool Enable) { m_persist = Enable; }
+	void directory(std::string const &Dir)
+	{
+		if (!Dir.empty() && Dir != m_dir)
+		{
+			m_dir = Dir;
+			m_known.clear(); // existence cache is per-folder; drop it when the folder changes
+		}
+	}
+	std::string const &directory() const { return m_dir; }
+
+	// loads/unloads chunks so the resident set matches the radius around CameraPos
+	void update(glm::dvec3 const &CameraPos);
+	// drops all resident chunks (e.g. when streaming is switched off), saving edits first
+	void clear();
+	// hard reset used when a new scenery loads: drops everything WITHOUT touching scene sections
+	// (the old region is being torn down, so its section pointers are already dangling)
+	void reset();
+
+	// appends raw pointers to every resident chunk (for sculpt / snap / optimize routing)
+	void collect(std::vector<editor_terrain *> &Out) const;
+	// resident chunk whose footprint covers (X,Z), or nullptr
+	editor_terrain *terrain_at(double X, double Z) const;
+
+	// authoring: create a flat chunk at (Cx,Cz), make it resident and persist it to disk
+	void add_chunk(int Cx, int Cz);
+	// authoring: delete the chunk at (Cx,Cz) from memory and disk
+	void remove_chunk(int Cx, int Cz);
+	// persist an externally-authored chunk (e.g. a manual grid chunk being handed over to streaming)
+	void save_chunk(int Cx, int Cz, editor_terrain const &Terrain);
+	// chunk coordinate covering (X,Z)
+	chunk_key key_for(double X, double Z) const { return key_at(X, Z); }
+
+	std::size_t resident() const { return m_chunks.size(); }
+	float chunk_world_size() const { return m_cells * m_cellsize; }
+	int cells() const { return m_cells; }
+	float cellsize() const { return m_cellsize; }
+	int radius() const { return m_radius; }
+
+	// writes every resident, edited chunk to disk (without unloading) - used on save
+	void flush();
+
+  private:
+	// per-chunk state bits cached so we don't stat the filesystem repeatedly
+	enum chunk_flag : uint8_t
+	{
+		cf_exists_on_disk = 1u << 0, // a saved 16-bit chunk file exists
+	};
+
+	chunk_key key_at(double X, double Z) const;
+	glm::dvec3 chunk_centre(int Cx, int Cz) const;
+	std::string chunk_path(int Cx, int Cz) const;
+	bool chunk_on_disk(chunk_key const &Key);                          // cached existence test
+	bool load_heights(int Cx, int Cz, std::vector<float> &Out) const;  // reads a 16-bit chunk file
+	void save_heights(int Cx, int Cz, editor_terrain const &Terrain);  // writes a 16-bit chunk file
+
+	bool m_active{false};
+	int m_cells{32};
+	float m_cellsize{2.0f};
+	int m_radius{2};            // chunks loaded around the camera (Chebyshev radius)
+	float m_baseheight{0.0f};
+	std::string m_texture;
+	bool m_auto_optimize{true};
+	float m_simplify_error{0.5f};
+	bool m_persist{true};
+	std::string m_dir{"editor_terrain"}; // folder for 16-bit chunk files
+
+	std::map<chunk_key, std::unique_ptr<editor_terrain>> m_chunks;
+	std::map<chunk_key, std::uint8_t> m_known; // cached per-chunk flag bits
+};
+
+// single, simulation-level streamer instance shared by the editor (authoring) and the scenery
+// loader (so streamed terrain renders in every mode, including the driver). loaded from the
+// `editorterrain` scenery directive and updated each frame with the active camera.
+extern terrain_streamer EditorTerrain;

simulation/simulationstateserializer.cpp

@@ -27,6 +27,7 @@ http://mozilla.org/MPL/2.0/.
 #include "application/application.h"
 #include "rendering/renderer.h"
 #include "utilities/Logs.h"
+#include "editor/editorTerrainStreamer.hpp"
 
 namespace simulation {
 
@@ -35,6 +36,10 @@ state_serializer::deserialize_begin( std::string const &Scenariofile ) {
 
     crashreport_add_info("scenario", Scenariofile);
 
+    // drop any streamed editor terrain from a previously loaded scenery before the old region (and
+    // its sections, which those chunks referenced) is destroyed below
+    EditorTerrain.reset();
+
     // TODO: move initialization to separate routine so we can reuse it
     SafeDelete( Region );
     Region = new scene::basic_region();
@@ -101,6 +106,7 @@ state_serializer::deserialize_begin( std::string const &Scenariofile ) {
 	            { "time",        &state_serializer::deserialize_time },
 	            { "trainset",    &state_serializer::deserialize_trainset },
 	            { "terrain",     &state_serializer::deserialize_terrain },
+	            { "editorterrain", &state_serializer::deserialize_editorterrain },
 	            { "endtrainset", &state_serializer::deserialize_endtrainset } };
 
 	for( auto &function : functionlist ) {
@@ -803,6 +809,31 @@ state_serializer::deserialize_terrain(cParser &Input, scene::scratch_data &Scrat
 
 }
 
+void
+state_serializer::deserialize_editorterrain(cParser &Input, scene::scratch_data &Scratchpad)
+{
+	// editor-authored streaming terrain. format:
+	//   editorterrain <folder> <cells> <cellsize> <radius> endeditorterrain
+	// the global streamer loads its 16-bit chunk files around the camera in every mode
+	std::string folder;
+	int cells = 32;
+	float cellsize = 2.0f;
+	int radius = 4;
+	Input.getTokens(4);
+	Input >> folder >> cells >> cellsize >> radius;
+	skip_until(Input, "endeditorterrain");
+
+	if (!folder.empty() && cells > 0 && cellsize > 0.0f)
+	{
+		EditorTerrain.directory(folder);
+		EditorTerrain.configure(cells, cellsize, (radius < 0 ? 0 : radius), 0.0f, std::string());
+		EditorTerrain.active(true);
+		WriteLog("Editor terrain stream enabled: " + folder + " (cells " + std::to_string(cells)
+		             + ", cellsize " + std::to_string(cellsize) + ", radius " + std::to_string(radius) + ")",
+		         logtype::generic);
+	}
+}
+
 void
 state_serializer::deserialize_endtrainset( cParser &Input, scene::scratch_data &Scratchpad ) {
 
@@ -1192,6 +1223,16 @@ state_serializer::export_as_text(std::string const &Scenariofile) const {
 	scmfile << "// sounds\n";
 	Region->export_as_text( scmfile );
 
+	// editor-authored streaming terrain: emit a directive pointing at its 16-bit chunk folder so the
+	// scenery streams it on load (in every mode)
+	if( EditorTerrain.active() ) {
+		scmfile << "// editor terrain\neditorterrain "
+		        << EditorTerrain.directory() << ' '
+		        << EditorTerrain.cells() << ' '
+		        << EditorTerrain.cellsize() << ' '
+		        << EditorTerrain.radius() << " endeditorterrain\n";
+	}
+
 	scmfile << "// modified objects\ninclude " << filename << "_export_dirty.scm\n";
 
 	std::ofstream ctrfile { absfilename + ".ctr" };
@@ -1239,6 +1280,16 @@ state_serializer::export_as_text(std::string const &Scenariofile) const {
 	scmfile << "// sounds\n";
 	Region->export_as_text( scmfile );
 
+	// editor-authored streaming terrain: emit a directive pointing at its 16-bit chunk folder so the
+	// scenery streams it on load (in every mode)
+	if( EditorTerrain.active() ) {
+		scmfile << "// editor terrain\neditorterrain "
+		        << EditorTerrain.directory() << ' '
+		        << EditorTerrain.cells() << ' '
+		        << EditorTerrain.cellsize() << ' '
+		        << EditorTerrain.radius() << " endeditorterrain\n";
+	}
+
 	scmfile << "// modified objects\ninclude " << filename << "_export_dirty.scm\n";
 
 	std::ofstream ctrfile { absfilename + ".ctr" };

simulation/simulationstateserializer.h

@@ -73,6 +73,7 @@ private:
     void deserialize_time( cParser &Input, scene::scratch_data &Scratchpad );
     void deserialize_trainset( cParser &Input, scene::scratch_data &Scratchpad );
     void deserialize_terrain( cParser &Input, scene::scratch_data &Scratchpad );
+    void deserialize_editorterrain( cParser &Input, scene::scratch_data &Scratchpad );
     void deserialize_endtrainset( cParser &Input, scene::scratch_data &Scratchpad );
     TTrack * deserialize_path( cParser &Input, scene::scratch_data &Scratchpad, scene::node_data const &Nodedata );
     TTraction * deserialize_traction( cParser &Input, scene::scratch_data &Scratchpad, scene::node_data const &Nodedata );