fix(editor): the templates looked for in the background from the start, the panels don't wait for them

The include bank read every .inc of the scenery directory on the first use, on the main thread: opening the
switch panel or selecting a switch stopped the editor until all of them were read, which on a large scenery
directory under Windows takes long. The bank now starts looking as the editor starts, in the background,
and the main thread only reads the descriptions once the files are read.

- the switch drives, turntables and signals are read again from the bank when it changes; the panels show
  "Looking for the templates..." meanwhile, only the edits which need the templates wait for them
- the sweep tool takes the templates from the bank instead of looking through the scenery directory again
- a folder linked into one of its own subfolders no longer leads the walk in circles
- the log tells how many templates were found and how long it took

application/editormode.cpp

@@ -1286,6 +1286,12 @@ bool editor_mode::update()
     // bindings are matched with the objects by their locations, before anything gets moved
     if (false == m_bindings_loaded && simulation::is_ready)
         infra_load();
+    // the templates are looked for in the background from the start, for the tools to have them at hand
+    if (simulation::is_ready)
+    {
+        EditorIncludes.start();
+        EditorIncludes.poll();
+    }
 
     auto const deltarealtime = Timer::GetDeltaRenderTime();
 

application/editormode.h

@@ -1152,7 +1152,7 @@ class editor_mode : public application_mode, private editor_track::observer
 			bool preferred{false};
 		};
 		std::vector<drive_family> drives;
-		bool drives_scanned{false};
+		int drives_revision{-1}; // of the include bank the drives were read from
 		int drive{-1};
 		int drive_side{0};
 		struct slide_state
@@ -1231,7 +1231,8 @@ class editor_mode : public application_mode, private editor_track::observer
 	void switch_slide_update();
 	void switch_slide_finish();
 	void draw_switch_slide() const;
-	void scan_switch_drives();
+	// reads the drives from the include bank once it changes; Wait: for the templates being looked for. true if it read them
+	bool scan_switch_drives(bool Wait = true);
 	std::string switch_name_for_new() const;
 	bool place_switch_drive(TTrack &Switch);
 	bool flip_switch_drive(TTrack &Switch);
@@ -1264,7 +1265,7 @@ class editor_mode : public application_mode, private editor_track::observer
 	struct turntable_tool
 	{
 		std::vector<turntable_template> templates;
-		bool scanned{false};
+		int revision{-1}; // of the include bank the templates were read from
 		int chosen{-1};
 		char name[64]{};
 		bool placing{false};
@@ -1290,7 +1291,7 @@ class editor_mode : public application_mode, private editor_track::observer
 		std::string error;
 	};
 	turntable_tool m_turntable;
-	void scan_turntables();
+	bool scan_turntables(bool Wait = true);
 	std::string turntable_name_for_new() const;
 	bool turntable_read(TTrack *Table);
 	std::vector<scene::instance_handle> turntable_includes(TTrack const &Table);
@@ -1833,7 +1834,7 @@ class editor_mode : public application_mode, private editor_track::observer
 		int family{-1};
 		int index{0};
 	};
-	std::optional<switch_drive> find_switch_drive(TTrack const &Switch);
+	std::optional<switch_drive> find_switch_drive(TTrack const &Switch, bool Wait = true);
 	std::string missing_drive_variant(int const Family, bool const Rightswitch, bool const Driveright) const;
 	bool replace_switch_drive(TTrack &Switch, int Family, bool Flip);
 	void apply_switch_drive_choice(TTrack &Switch);
@@ -2058,7 +2059,7 @@ class editor_mode : public application_mode, private editor_track::observer
 		std::string status;
 	};
 	signal_tool m_signal;
-	void signal_scan();
+	bool signal_scan(bool Wait = true);
 	signal_template const *signal_armed() const;
 	signal_template const *signal_template_of(std::string const &File) const;
 	bool signal_spot_at(TTrack &Track, int const Path, glm::dvec3 const &Point, glm::dvec3 const &Cursor, int const Facing, int const Side, signal_template const &Template, signal_spot &Spot) const;
@@ -2091,6 +2092,7 @@ class editor_mode : public application_mode, private editor_track::observer
 		char search[64]{};
 		std::vector<std::string> models;
 		bool scanned{false};
+		int revision{-1}; // of the include bank the templates were taken from
 		// parameters of a template laid along the track, with their names
 		std::string parameters_for;
 		std::vector<std::array<char, 128>> parameters;

application/editormodeselftestsignal.cpp

@@ -61,6 +61,7 @@ editor_mode::selftest_result editor_mode::selftest_signal(std::string const &com
 				values[pair.substr(0, equals)] = pair.substr(equals + 1);
 		}
 		show_track_tab(track_tab::signals);
+		signal_scan();
 		signal_refresh(true);
 		auto &tool{m_signal};
 		auto const standing_named = [&](std::string const &Name) -> signal_standing const * {

application/editormodesignal.cpp

@@ -214,13 +214,15 @@ void draw_signal_icon(ImDrawList *Draw, ImVec2 const Origin, ImVec2 const Size,
 
 } // namespace
 
-void editor_mode::signal_scan()
+bool editor_mode::signal_scan(bool const Wait)
 {
 	auto &tool{m_signal};
-	if (false == EditorIncludes.scanned())
-		EditorIncludes.scan();
-	if (tool.revision == EditorIncludes.revision())
-		return;
+	if (Wait)
+		EditorIncludes.wait();
+	else
+		EditorIncludes.start();
+	if (false == EditorIncludes.scanned() || tool.revision == EditorIncludes.revision())
+		return false;
 	tool.revision = EditorIncludes.revision();
 	std::string chosen[kKindCount];
 	for (int i = 0; i < kKindCount; ++i)
@@ -280,6 +282,7 @@ void editor_mode::signal_scan()
 		}
 		tool.chosen[i] = best;
 	}
+	return true;
 }
 
 editor_mode::signal_template const *editor_mode::signal_armed() const
@@ -346,9 +349,9 @@ std::string editor_mode::signal_intent_label(signal_template const &Template, si
 void editor_mode::signal_refresh(bool const Force)
 {
 	auto &tool{m_signal};
-	signal_scan();
+	auto const scanned{signal_scan(false)};
 	auto const now{ImGui::GetTime()};
-	if (false == Force && tool.standing_history == m_history.size() && tool.standing_redo == g_redo.size() && now - tool.standing_time < kRefreshPeriod)
+	if (false == Force && false == scanned && tool.standing_history == m_history.size() && tool.standing_redo == g_redo.size() && now - tool.standing_time < kRefreshPeriod)
 		return;
 	tool.standing_history = m_history.size();
 	tool.standing_redo = g_redo.size();
@@ -893,7 +896,12 @@ void editor_mode::render_signal_selected()
 void editor_mode::render_signal_ui()
 {
 	auto &tool{m_signal};
-	signal_scan();
+	signal_scan(false);
+	if (tool.templates.empty() && EditorIncludes.searching())
+	{
+		ImGui::TextDisabled("%s", STR_C("Looking for the templates..."));
+		return;
+	}
 	if (tool.templates.empty())
 	{
 		ImGui::TextWrapped("%s", STR_C("No signal templates: no template of the scenery directory is described with category: signal. tools/editor/signal_headers.py writes the descriptions into the signal templates."));

application/editormodesweep.cpp

@@ -137,31 +137,40 @@ std::string lower(std::string Text)
 void editor_mode::sweep_scan_models()
 {
 	auto &tool{m_sweep};
-	if (tool.scanned)
+	EditorIncludes.start();
+	auto changed{false};
+	if (false == tool.scanned)
+	{
+		tool.scanned = true;
+		changed = true;
+		std::error_code error;
+		for (auto const &entry : std::filesystem::recursive_directory_iterator("models", std::filesystem::directory_options::skip_permission_denied, error))
+		{
+			if (false == entry.is_regular_file(error))
+				continue;
+			auto const extension{lower(entry.path().extension().string())};
+			if (extension != ".e3d" && extension != ".t3d")
+				continue;
+			auto name{std::filesystem::relative(entry.path(), "models", error).generic_string()};
+			name.erase(name.size() - extension.size());
+			tool.models.push_back(name);
+		}
+	}
+	// the templates come from the include bank, which looks through the scenery directory in the background
+	if (EditorIncludes.scanned() && tool.revision != EditorIncludes.revision())
+	{
+		tool.revision = EditorIncludes.revision();
+		std::erase_if(tool.models, [](std::string const &Model) { return Model.ends_with(".inc"); });
+		for (auto const &entry : EditorIncludes.entries())
+			tool.models.push_back(entry.file);
+		changed = true;
+	}
+	if (false == changed)
 		return;
-	tool.scanned = true;
-	std::error_code error;
-	std::set<std::string> found;
-	for (auto const &entry : std::filesystem::recursive_directory_iterator("models", std::filesystem::directory_options::skip_permission_denied, error))
-	{
-		if (false == entry.is_regular_file(error))
-			continue;
-		auto const extension{lower(entry.path().extension().string())};
-		if (extension != ".e3d" && extension != ".t3d")
-			continue;
-		auto name{std::filesystem::relative(entry.path(), "models", error).generic_string()};
-		name.erase(name.size() - extension.size());
-		found.insert(name);
-	}
-	for (auto const &entry : std::filesystem::recursive_directory_iterator("scenery", std::filesystem::directory_options::skip_permission_denied, error))
-	{
-		if (entry.is_regular_file(error) && lower(entry.path().extension().string()) == ".inc")
-			found.insert(std::filesystem::relative(entry.path(), "scenery", error).generic_string());
-	}
-	tool.models.assign(found.begin(), found.end());
+	std::sort(tool.models.begin(), tool.models.end());
+	tool.models.erase(std::unique(tool.models.begin(), tool.models.end()), tool.models.end());
 }
 
-
 void editor_mode::sweep_curve()
 {
 	auto &tool{m_sweep};

application/editormodeswitchdrive.cpp

@@ -77,15 +77,20 @@ double angle_value(std::vector<std::string> const &Values, std::size_t const Ind
 
 } // namespace
 
-void editor_mode::scan_switch_drives()
+bool editor_mode::scan_switch_drives(bool const Wait)
 {
 	auto &tool{m_switch};
-	if (tool.drives_scanned)
-		return;
-	tool.drives_scanned = true;
+	if (Wait)
+		EditorIncludes.wait();
+	else
+		EditorIncludes.start();
+	if (false == EditorIncludes.scanned() || tool.drives_revision == EditorIncludes.revision())
+		return false;
+	tool.drives_revision = EditorIncludes.revision();
+	// the drive chosen stays chosen, also none
+	auto const chosen{tool.drive >= 0 && tool.drive < static_cast<int>(tool.drives.size()) ? std::optional<std::string>{tool.drives[tool.drive].name} : std::nullopt};
+	auto const none{false == tool.drives.empty() && tool.drive < 0};
 	tool.drives.clear();
-	if (false == EditorIncludes.scanned())
-		EditorIncludes.scan();
 	for (auto const &entry : EditorIncludes.entries())
 	{
 		if (entry.category != "switch drive")
@@ -111,10 +116,14 @@ void editor_mode::scan_switch_drives()
 	}
 	std::sort(tool.drives.begin(), tool.drives.end(), [](switch_tool::drive_family const &A, switch_tool::drive_family const &B) { return A.name < B.name; });
 	auto const preferred{std::find_if(tool.drives.begin(), tool.drives.end(), [](switch_tool::drive_family const &Family) { return Family.preferred; })};
-	if (tool.drives.empty())
+	auto const kept{std::find_if(tool.drives.begin(), tool.drives.end(), [&](switch_tool::drive_family const &Family) { return chosen.has_value() && Family.name == *chosen; })};
+	if (tool.drives.empty() || none)
 		tool.drive = -1;
+	else if (kept != tool.drives.end())
+		tool.drive = static_cast<int>(kept - tool.drives.begin());
 	else
 		tool.drive = preferred != tool.drives.end() ? static_cast<int>(preferred - tool.drives.begin()) : 0;
+	return true;
 }
 
 void editor_mode::switch_drive_captured(TTrack const &Track)
@@ -204,10 +213,10 @@ bool editor_mode::place_switch_drive(TTrack &Switch)
 	return true;
 }
 
-std::optional<editor_mode::switch_drive> editor_mode::find_switch_drive(TTrack const &Switch)
+std::optional<editor_mode::switch_drive> editor_mode::find_switch_drive(TTrack const &Switch, bool const Wait)
 {
 	auto const &tool{m_switch};
-	scan_switch_drives();
+	scan_switch_drives(Wait);
 	std::map<std::string, std::pair<int, int>, std::less<>> variants;
 	for (int family = 0; family < static_cast<int>(tool.drives.size()); ++family)
 		for (int index = 0; index < 4; ++index)
@@ -286,10 +295,10 @@ bool editor_mode::replace_switch_drive(TTrack &Switch, int const Family, bool co
 bool editor_mode::render_switch_drive_choice()
 {
 	auto &tool{m_switch};
-	scan_switch_drives();
+	scan_switch_drives(false);
 	if (tool.drives.empty())
 	{
-		ImGui::TextDisabled("%s", STR_C("No switch drive models"));
+		ImGui::TextDisabled("%s", EditorIncludes.searching() ? STR_C("Looking for the templates...") : STR_C("No switch drive models"));
 		return false;
 	}
 	auto const label = [&](int const Index) { return Index < 0 ? std::string{STR_C("no drive")} : tool.drives[Index].name; };

application/editormodetrack.cpp

@@ -410,11 +410,10 @@ void editor_mode::show_track_tab(track_tab const Wanted)
 			profile_open_run(*track);
 		break;
 	case track_tab::turntable:
-		scan_turntables();
+		scan_turntables(false);
 		turntable_read(track);
 		break;
 	case track_tab::signals:
-		signal_scan();
 		signal_refresh(true);
 		break;
 	default:
@@ -982,7 +981,9 @@ void editor_mode::render_turnout_ui()
 		ImGui::PushID(&Switch);
 		if (set.has_value())
 			ImGui::TextUnformatted(Switch.name().c_str());
-		if (auto const drive{find_switch_drive(Switch)})
+		if (false == EditorIncludes.scanned())
+			ImGui::TextDisabled("%s", STR_C("Looking for the templates..."));
+		else if (auto const drive{find_switch_drive(Switch, false)})
 		{
 			auto const family{drive->family};
 			auto const variant{drive->index};

application/editormodeturntable.cpp

@@ -189,15 +189,18 @@ void label(screen_projection const &Projection, glm::dvec3 const &Point, std::st
 
 } // namespace
 
-void editor_mode::scan_turntables()
+bool editor_mode::scan_turntables(bool const Wait)
 {
 	auto &tool{m_turntable};
-	if (tool.scanned)
-		return;
-	tool.scanned = true;
+	if (Wait)
+		EditorIncludes.wait();
+	else
+		EditorIncludes.start();
+	if (false == EditorIncludes.scanned() || tool.revision == EditorIncludes.revision())
+		return false;
+	tool.revision = EditorIncludes.revision();
+	auto const chosen{tool.chosen >= 0 && tool.chosen < static_cast<int>(tool.templates.size()) ? tool.templates[tool.chosen].file : std::string{}};
 	tool.templates.clear();
-	if (false == EditorIncludes.scanned())
-		EditorIncludes.scan();
 	for (auto const &entry : EditorIncludes.entries())
 	{
 		if (entry.category != "turntable")
@@ -224,7 +227,11 @@ void editor_mode::scan_turntables()
 		tool.templates.push_back(std::move(item));
 	}
 	std::stable_sort(tool.templates.begin(), tool.templates.end(), [](turntable_template const &A, turntable_template const &B) { return (A.angles > 0) != (B.angles > 0) ? A.angles > 0 : A.name < B.name; });
-	tool.chosen = tool.templates.empty() ? -1 : 0;
+	auto const kept{std::find_if(tool.templates.begin(), tool.templates.end(), [&](turntable_template const &Template) { return Template.file == chosen; })};
+	tool.chosen = tool.templates.empty() ? -1 : kept != tool.templates.end() ? static_cast<int>(kept - tool.templates.begin()) : 0;
+	// the turntable read before points at the templates of before
+	tool.table = nullptr;
+	return true;
 }
 
 std::string editor_mode::turntable_name_for_new() const
@@ -258,7 +265,7 @@ bool editor_mode::turntable_read(TTrack *Table)
 		tool.status.clear();
 		tool.error.clear();
 	}
-	scan_turntables();
+	scan_turntables(false);
 	tool.table = Table;
 	tool.length = length_of(*Table);
 	auto const &path{Table->m_paths.front()};
@@ -794,7 +801,7 @@ editor_mode::track_intent editor_mode::turntable_intent(glm::dvec3 const &Ground
 			return make(kind::select, track == tool.table ? std::string{} : STR("Click: this turntable"));
 		}
 	}
-	scan_turntables();
+	scan_turntables(false);
 	if (tool.chosen < 0)
 		return intent;
 	if (false == Shift)
@@ -813,7 +820,7 @@ editor_mode::track_intent editor_mode::turntable_intent(glm::dvec3 const &Ground
 void editor_mode::render_turntable_ui()
 {
 	auto &tool{m_turntable};
-	scan_turntables();
+	scan_turntables(false);
 	auto *selected{selected_track()};
 	if (tool.table != selected)
 		turntable_read(selected);
@@ -979,7 +986,9 @@ void editor_mode::render_turntable_ui()
 		if (selected != nullptr && selected->eType == tt_Table)
 			ImGui::TextColored(warncolour, "%s", STR_C("This turntable can't be read"));
 		ImGui::TextUnformatted(STR_C("New turntable"));
-		if (tool.templates.empty())
+		if (tool.templates.empty() && EditorIncludes.searching())
+			ImGui::TextDisabled("%s", STR_C("Looking for the templates..."));
+		else if (tool.templates.empty())
 			ImGui::TextColored(warncolour, "%s", STR_C("No turntables: no template is described with category: turntable and its turntable: {length}"));
 		for (int i = 0; i < static_cast<int>(tool.templates.size()); ++i)
 		{

application/editoruipanels.cpp

@@ -867,9 +867,11 @@ void nodebank_panel::render_list(std::vector<std::pair<std::string, std::shared_
 
 void nodebank_panel::render_templates_list(std::string const &Filter)
 {
-	if (false == EditorIncludes.scanned())
+	EditorIncludes.start();
+	if (EditorIncludes.searching() && EditorIncludes.ready().empty())
 	{
-		EditorIncludes.scan();
+		ImGui::TextDisabled("%s", STR_C("Looking for the templates..."));
+		return;
 	}
 	auto const &templates{EditorIncludes.entries()};
 	std::string const *category{nullptr};
@@ -2203,10 +2205,7 @@ void includes_panel::open(std::string const &File)
 
 void includes_panel::render_list()
 {
-	if (false == EditorIncludes.scanned())
-	{
-		EditorIncludes.scan();
-	}
+	EditorIncludes.start();
 	auto const &entries{EditorIncludes.entries()};
 
 	ImGui::PushItemWidth(ImGui::GetFontSize() * 12.0f);
@@ -2221,9 +2220,13 @@ void includes_panel::render_list()
 		changed |= ImGui::Checkbox(STR_C("Used by the scenery"), &m_usedonly);
 	}
 	ImGui::SameLine();
-	if (ImGui::Button(STR_C("Rescan")))
+	if (EditorIncludes.searching())
 	{
-		EditorIncludes.scan();
+		ImGui::TextDisabled("%s", STR_C("Looking for the templates..."));
+	}
+	else if (ImGui::Button(STR_C("Rescan")))
+	{
+		EditorIncludes.rescan();
 	}
 	if (ImGui::IsItemHovered())
 	{