18
0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-11-19 22:44:59 +01:00
Files
maszyna/scripting/PyInt.cpp
maj00r afbc84c6bc fix(files): the banks keep the names asked for, only the opening takes the case of the disk
The textures, materials and sounds are kept by the names they're asked for.
Given back as named on the disk, a file in another case missed the bank and
was loaded again on every request.

- FileExists with names and extensions finds the file in any case but gives
  back the name asked for; FileOnDisk gives the file to open, used where
  textures, sounds and scripts are opened
- FileExists of a single name is a plain check again, the places checking
  it and opening the same name stay as they were
- the texture format is told by the extension asked for again
- a folder changed shortly before it was read is read again for a name
  missing from it, the time of a change being as coarse as the clock of the
  file system; a listing cut short isn't kept
2026-11-13 15:47:52 +01:00

716 lines
20 KiB
C++

/*
This Source Code Form is subject to the
terms of the Mozilla Public License, v.
2.0. If a copy of the MPL was not
distributed with this file, You can
obtain one at
http://mozilla.org/MPL/2.0/.
*/
#include "scripting/PyInt.h"
#include "stdafx.h"
#include "application/application.h"
#include "utilities/Globals.h"
#include "utilities/Logs.h"
#include "utilities/dictionary.h"
#ifdef __GNUC__
#pragma GCC diagnostic ignored "-Wwrite-strings"
#endif
#include <simulation/simulation.h>
void render_task::run()
{
// convert provided input to a python dictionary
auto *input = PyDict_New();
if (input == nullptr)
{
cancel();
return;
}
for (auto const &[dataname, datavalue] : m_input->floats)
{
auto *value{PyGetFloat(datavalue)};
if (value == nullptr)
{
PyErr_Clear();
continue;
}
PyDict_SetItemString(input, dataname.c_str(), value);
Py_DECREF(value);
}
for (auto const &[dataname, datavalue] : m_input->integers)
{
auto *value{PyLong_FromLong(datavalue)};
if (value == nullptr)
{
PyErr_Clear();
continue;
}
PyDict_SetItemString(input, dataname.c_str(), value);
Py_DECREF(value);
}
for (auto const &[dataname, datavalue] : m_input->bools)
{
// Py_True / Py_False sa niesmiertelne, ale PyDict_SetItemString i tak
// pobiera wlasna referencje - nie zwalniamy
auto *value{PyGetBool(datavalue)};
PyDict_SetItemString(input, dataname.c_str(), value);
}
for (auto const &[dataname, datavalue] : m_input->strings)
{
// Nazwy scenerii/wagonow (asName, SceneryFile, asCarName, cCode...) moga
// byc albo w UTF-8, albo w starym kodowaniu Windows-1250. PyUnicode_FromString
// wymaga POPRAWNEGO UTF-8 - dla nie-UTF-8 zwracalo NULL, a potem
// PyDict_SetItemString(dict, key, NULL) robilo Py_INCREF na NULL -> crash.
//
// Strategia: najpierw probujemy UTF-8 (strict). Jesli sie nie uda - probujemy
// cp1250. Jesli oba zawioda - pomijamy klucz, ale nie wywracamy symulatora.
char const *const str{datavalue.c_str()};
Py_ssize_t const len{static_cast<Py_ssize_t>(datavalue.size())};
auto *value{PyUnicode_DecodeUTF8(str, len, "strict")};
if (value == nullptr)
{
PyErr_Clear();
value = PyUnicode_Decode(str, len, "cp1250", "strict");
}
if (value == nullptr)
{
PyErr_Clear();
continue;
}
PyDict_SetItemString(input, dataname.c_str(), value);
Py_DECREF(value);
}
for (auto const &[dataname, datavalue] : m_input->vec2_lists)
{
PyObject *list = PyList_New(datavalue.size());
for (size_t i = 0; i < datavalue.size(); i++)
{
auto const &vec = datavalue[i];
WriteLog("passing " + glm::to_string(vec));
PyObject *tuple = PyTuple_New(2);
PyTuple_SetItem(tuple, 0, PyGetFloat(vec.x)); // steals ref
PyTuple_SetItem(tuple, 1, PyGetFloat(vec.y)); // steals ref
PyList_SetItem(list, i, tuple); // steals ref
}
PyDict_SetItemString(input, dataname.c_str(), list);
Py_DECREF(list);
}
m_input = nullptr;
// call the renderer
auto *output{PyObject_CallMethod(m_renderer, "render", "O", input)};
Py_DECREF(input);
if (output != nullptr)
{
auto *outputWidth = PyObject_CallMethod(m_renderer, "get_width", nullptr);
auto *outputHeight = PyObject_CallMethod(m_renderer, "get_height", nullptr);
if (outputWidth != nullptr && outputHeight != nullptr && m_target != nullptr)
{
const auto screenWidth = static_cast<int>(PyLong_AsLong(outputWidth));
const auto screenHeight = static_cast<int>(PyLong_AsLong(outputHeight));
const bool useRgb = false && !Global.gfx_usegles;
const int glFormat = useRgb ? GL_SRGB8 : GL_SRGB8_ALPHA8;
const int glComponents = useRgb ? GL_RGB : GL_RGBA;
const size_t bytesPerPixel = useRgb ? 3u : 4u;
const size_t expectedBytes = static_cast<size_t>(screenWidth) * static_cast<size_t>(screenHeight) * bytesPerPixel;
Py_ssize_t pythonBufferBytes = 0;
char *pythonBufferPtr = nullptr;
const bool bufferExtracted = PyBytes_AsStringAndSize(output, &pythonBufferPtr, &pythonBufferBytes) == 0 && pythonBufferPtr != nullptr;
if (!bufferExtracted)
{
ErrorLog("Python screen renderer: output is not a valid byte buffer");
}
else if (pythonBufferBytes < static_cast<Py_ssize_t>(expectedBytes))
{
ErrorLog(std::format("Python screen renderer: output buffer too small ({} bytes, expected {})", pythonBufferBytes, expectedBytes));
}
else
{
std::lock_guard guard(m_target->mutex);
if (m_target->image.size() != expectedBytes)
m_target->image.resize(expectedBytes);
std::memcpy(m_target->image.data(), pythonBufferPtr, expectedBytes);
m_target->width = screenWidth;
m_target->height = screenHeight;
m_target->components = glComponents;
m_target->format = glFormat;
m_target->timestamp = std::chrono::high_resolution_clock::now();
}
}
if (outputHeight != nullptr)
Py_DECREF(outputHeight);
if (outputWidth != nullptr)
Py_DECREF(outputWidth);
Py_DECREF(output);
}
// get commands from renderer
auto *commandsPO = PyObject_CallMethod(m_renderer, "getCommands", nullptr);
if (commandsPO != nullptr)
{
std::vector<std::string> commands = python_external_utils::PyObjectToStringArray(commandsPO);
Py_DECREF(commandsPO);
// we perform any actions ONLY when there are any commands in buffer
if (!commands.empty())
{
for (const auto &cmd : commands)
{
std::string baseCmd;
int p1 = 0;
int p2 = 0;
if (size_t pos1 = cmd.find(';'); pos1 == std::string::npos)
{
baseCmd = cmd;
}
else
{
baseCmd = cmd.substr(0, pos1);
size_t pos2 = cmd.find(';', pos1 + 1);
if (pos2 == std::string::npos)
{
p1 = std::stoi(cmd.substr(pos1 + 1));
}
else
{
p1 = std::stoi(cmd.substr(pos1 + 1, pos2 - pos1 - 1));
p2 = std::stoi(cmd.substr(pos2 + 1));
}
}
auto it = simulation::commandMap.find(baseCmd);
if (it != simulation::commandMap.end())
{
command_data cd;
cd.command = it->second;
cd.action = GLFW_PRESS;
cd.param1 = p1;
cd.param2 = p2;
WriteLog("Python: Executing command [" + baseCmd + "] with params: P1=" + std::to_string(p1) + " P2=" + std::to_string(p2) +
" Target ID=" + std::to_string(simulation::Train->id()));
simulation::Commands.push(cd, static_cast<size_t>(command_target::vehicle) | simulation::Train->id());
}
else
{
ErrorLog("Python: Command [" + baseCmd + "] not found!");
}
}
}
}
}
void render_task::upload() const
{
if (Global.python_uploadmain && m_target && m_target->shared_tex)
{
m_target->shared_tex->update_from_memory(m_target->width, m_target->height, reinterpret_cast<const uint8_t *>(m_target->image.data()));
}
}
void render_task::cancel() const { /* nothing to do */ }
// initializes the module. returns true on success
auto python_taskqueue::init() -> bool
{
crashreport_add_info("python.threadedupload", Global.python_threadedupload ? "yes" : "no");
crashreport_add_info("python.uploadmain", Global.python_uploadmain ? "yes" : "no");
#ifdef _WIN32
const wchar_t *pythonhome = sizeof(void *) == 8 ? L"python64" : L"python";
#elif __linux__
const wchar_t *pythonhome = (sizeof(void *) == 8) ? L"linuxpython64" : L"linuxpython";
#elif __APPLE__
const wchar_t *pythonhome = (sizeof(void *) == 8) ? L"macpython64" : L"macpython";
#endif
{
// Py_SetPythonHome / Py_InitializeEx are deprecated since Python 3.11.
// Use the PyConfig init API (PEP 587) instead.
PyConfig config;
PyConfig_InitPythonConfig(&config);
config.install_signal_handlers = 0; // matches former Py_InitializeEx(0)
PyStatus status = PyConfig_SetString(&config, &config.home, pythonhome);
if (PyStatus_Exception(status))
{
PyConfig_Clear(&config);
ErrorLog("Python Interpreter: failed to set PYTHONHOME");
return false;
}
status = Py_InitializeFromConfig(&config);
if (PyStatus_Exception(status))
{
std::string msg = "Python Interpreter: Py_InitializeFromConfig failed";
if (status.err_msg != nullptr)
msg += std::string(": ") + status.err_msg;
if (status.func != nullptr)
msg += std::string(" (in ") + status.func + ")";
PyConfig_Clear(&config);
ErrorLog(msg);
return false;
}
PyConfig_Clear(&config);
}
// make CWD-local packages (e.g. the asset-side "scripts" package used by
// texture/screen renderers) importable. Embedded CPython builds sys.path
// from PYTHONHOME only; unlike the python CLI it does not prepend the
// working directory, and unlike Windows' getpathp.c it does not add the
// executable's directory either, so on POSIX "from scripts import ..."
// would otherwise fail. "." matches how the engine resolves all other
// assets (relative to CWD). Idempotent/harmless on Windows.
if (PyObject *syspath = PySys_GetObject("path")) // borrowed ref
{
PyObject *cwd = PyUnicode_FromString(".");
if (cwd != nullptr)
{
PyList_Insert(syspath, 0, cwd);
Py_DECREF(cwd);
}
}
PyObject *stringiomodule{nullptr};
PyObject *stringioclassname{nullptr};
PyObject *stringioobject{nullptr};
// do the setup work while we hold the lock
m_main = PyImport_ImportModule("__main__");
if (m_main == nullptr)
{
ErrorLog("Python Interpreter: __main__ module is missing");
goto release_and_exit;
}
stringiomodule = PyImport_ImportModule("io");
stringioclassname = stringiomodule != nullptr ? PyObject_GetAttrString(stringiomodule, "StringIO") : nullptr;
stringioobject = stringioclassname != nullptr ? PyObject_CallObject(stringioclassname, nullptr) : nullptr;
m_stderr = {(stringioobject == nullptr ? nullptr : PySys_SetObject("stderr", stringioobject) != 0 ? nullptr : stringioobject)};
if (false == run_file("abstractscreenrenderer"))
{
goto release_and_exit;
}
// release the lock, save the state for future use
m_mainthread = PyEval_SaveThread();
WriteLog("Python Interpreter: setup complete");
// init workers
for (auto &worker : m_workers)
{
GLFWwindow *openglcontextwindow = nullptr;
if (Global.python_threadedupload)
openglcontextwindow = Application.window(-1);
worker = std::jthread(&python_taskqueue::run, this, openglcontextwindow, std::ref(m_tasks), std::ref(m_uploadtasks), std::ref(m_condition), std::ref(m_exit));
if (false == worker.joinable())
{
return false;
}
}
m_initialized = true;
return true;
release_and_exit:
PyEval_SaveThread();
return false;
}
// shuts down the module
void python_taskqueue::exit()
{
if (!m_initialized)
return;
// let the workers know we're done with them
m_exit = true;
m_condition.notify_all();
// let them free up their shit before we proceed
m_workers = {};
// drop any pending render/upload tasks. the workers are joined at this
// point so the locks are strictly defensive, but they cost nothing and
// document intent. clearing the deque also actually releases the tasks,
// which the previous code did not do (cancel() is a no-op stub).
{
std::scoped_lock lock(m_tasks.mutex);
for (auto const &task : m_tasks.data)
{
task->cancel();
}
m_tasks.data.clear();
}
{
std::scoped_lock lock(m_uploadtasks.mutex);
m_uploadtasks.data.clear();
}
// reclaim cached python objects while the interpreter is still alive,
// so no Py_DECREF lands on a finalized interpreter during later teardown
acquire_lock();
for (auto const &[renderername, rendererobject] : m_renderers)
{
Py_XDECREF(rendererobject);
}
m_renderers.clear();
Py_XDECREF(m_stderr);
m_stderr = nullptr;
Py_XDECREF(m_main);
m_main = nullptr;
// take a bow
Py_Finalize();
m_initialized = false;
}
// adds specified task along with provided collection of data to the work queue. returns true on success
auto python_taskqueue::insert(task_request const &Task) -> bool
{
if (!m_initialized || false == Global.python_enabled || Task.renderer.empty() || Task.input == nullptr || Task.target == nullptr)
{
return false;
}
auto *renderer{fetch_renderer(Task.renderer)};
if (renderer == nullptr)
{
return false;
}
auto newtask = std::make_shared<render_task>(renderer, Task.input, Task.target);
bool newtaskinserted{false};
// acquire a lock on the task queue and add the new task
{
std::scoped_lock lock(m_tasks.mutex);
// check the task list for a pending request with the same target
for (auto &task : m_tasks.data)
{
if (task->target() == Task.target)
{
// replace pending task in the slot with the more recent one
task->cancel();
task = newtask;
newtaskinserted = true;
break;
}
}
if (false == newtaskinserted)
{
m_tasks.data.emplace_back(newtask);
}
}
// potentially wake a worker to handle the new task
m_condition.notify_one();
// all done
return true;
}
// executes python script stored in specified file. returns true on success
auto python_taskqueue::run_file(std::string const &File, std::string const &Path) -> bool
{
auto const [filepath, fileextension]{FileExists({Path + File, "python/local/" + File}, {".py"})};
if (filepath.empty())
{
return false;
}
std::ifstream inputfile{FileOnDisk(filepath + fileextension)};
std::string input;
input.assign(std::istreambuf_iterator<char>(inputfile), std::istreambuf_iterator<char>());
if (PyRun_SimpleString(input.c_str()) != 0)
{
error();
return false;
}
return true;
}
// acquires the python gil and sets the main thread as current
void python_taskqueue::acquire_lock()
{
PyEval_RestoreThread(m_mainthread);
}
// releases the python gil and swaps the main thread out
void python_taskqueue::release_lock() const
{
PyEval_SaveThread();
}
auto python_taskqueue::fetch_renderer(std::string const &Renderer) -> PyObject *
{
if (auto const lookup{m_renderers.find(Renderer)}; lookup != std::end(m_renderers))
{
return lookup->second;
}
// try to load specified renderer class
std::string const path{substr_path(Renderer)};
auto const file{Renderer.substr(path.size())};
PyObject *renderer{nullptr};
PyObject *rendererarguments{nullptr};
PyObject *renderername{nullptr};
acquire_lock();
{
if (m_main == nullptr)
{
ErrorLog("Python Renderer: __main__ module is missing");
goto cache_and_return;
}
if (false == run_file(file, path))
{
goto cache_and_return;
}
renderername = PyObject_GetAttrString(m_main, file.c_str());
if (renderername == nullptr)
{
ErrorLog("Python Renderer: class \"" + file + "\" not defined");
goto cache_and_return;
}
rendererarguments = Py_BuildValue("(s)", path.c_str());
if (rendererarguments == nullptr)
{
ErrorLog("Python Renderer: failed to create initialization arguments");
goto cache_and_return;
}
renderer = PyObject_CallObject(renderername, rendererarguments);
PyObject_CallMethod(renderer, "manul_set_format", "(s)", "RGBA");
if (PyErr_Occurred() != nullptr)
{
error();
renderer = nullptr;
}
cache_and_return:
// clean up after yourself
if (rendererarguments != nullptr)
{
Py_DECREF(rendererarguments);
}
}
release_lock();
// cache the failures as well so we don't try again on subsequent requests
m_renderers.try_emplace(Renderer, renderer);
return renderer;
}
void python_taskqueue::run(GLFWwindow *Context, rendertask_sequence &Tasks, uploadtask_sequence &Upload_Tasks, threading::condition_variable &Condition, std::atomic<bool> const &Exit)
{
if (Context)
glfwMakeContextCurrent(Context);
// create a state object for this thread
PyEval_AcquireThread(m_mainthread);
auto *threadstate{PyThreadState_New(m_mainthread->interp)};
PyEval_ReleaseThread(m_mainthread);
std::shared_ptr<render_task> task{nullptr};
while (false == Exit.load())
{
// regardless of the reason we woke up prime the spurious wakeup flag for the next time
Condition.spurious(true);
// keep working as long as there's any scheduled tasks
do
{
task = nullptr;
// acquire a lock on the task queue and potentially grab a task from it
{
std::scoped_lock lock(Tasks.mutex);
if (false == Tasks.data.empty())
{
// fifo
task = Tasks.data.front();
Tasks.data.pop_front();
}
}
if (task != nullptr)
{
// swap in my thread state
PyEval_RestoreThread(threadstate);
{
// execute python code
task->run();
if (Context)
task->upload();
else
{
std::scoped_lock lock(Upload_Tasks.mutex);
Upload_Tasks.data.push_back(task);
}
if (PyErr_Occurred() != nullptr)
error();
}
// clear the thread state
PyEval_SaveThread();
}
// TBD, TODO: add some idle time between tasks in case we're on a single thread cpu?
} while (task != nullptr);
// if there's nothing left to do wait until there is
// short timeout: notify_all() in exit() can race with the worker's
// transition into the wait, and the run loop also drives prompt
// shutdown checks
Condition.wait_for(std::chrono::milliseconds(250));
}
// clean up thread state data
// Re-acquire the GIL with this worker's thread state, then clear and delete it.
// PyThreadState_DeleteCurrent() frees the current thread state AND releases the GIL,
// so we must NOT call PyEval_SaveThread() afterwards: at that point the current
// thread state is already NULL and PyEval_SaveThread() fatals on a NULL tstate.
PyEval_RestoreThread(threadstate);
PyThreadState_Clear(threadstate);
PyThreadState_DeleteCurrent();
// detach the GL context before the worker terminates; some drivers
// (NVIDIA on X11, certain Mesa/Wayland configs) hang in process teardown
// if a context is still current on a dying thread
if (Context)
glfwMakeContextCurrent(nullptr);
}
void python_taskqueue::update()
{
std::scoped_lock lock(m_uploadtasks.mutex);
for (auto const &task : m_uploadtasks.data)
task->upload();
m_uploadtasks.data.clear();
}
void python_taskqueue::error()
{
if (m_stderr != nullptr)
{
// std err pythona jest buforowane
PyErr_Print();
if (auto *errortext{PyObject_CallMethod(m_stderr, "getvalue", nullptr)}; errortext != nullptr)
{
if (const char *errstr = PyUnicode_AsUTF8(errortext); errstr != nullptr)
ErrorLog(errstr);
Py_DECREF(errortext);
}
PyObject_CallMethod(m_stderr, "truncate", "L", (long long)0);
PyObject_CallMethod(m_stderr, "seek", "L", (long long)0);
}
else
{
// nie dziala buffor pythona
PyObject *type;
PyObject *value;
PyObject *traceback;
PyErr_Fetch(&type, &value, &traceback);
if (type == nullptr)
{
ErrorLog("Python Interpreter: don't know how to handle null exception");
}
PyErr_NormalizeException(&type, &value, &traceback);
if (type == nullptr)
{
ErrorLog("Python Interpreter: don't know how to handle null exception");
}
if (auto *typetext{PyObject_Str(type)}; typetext != nullptr)
{
if (const char *s = PyUnicode_AsUTF8(typetext); s)
ErrorLog(s);
Py_DECREF(typetext);
}
if (value != nullptr)
{
auto *valuetext{PyObject_Str(value)};
if (valuetext != nullptr)
{
if (const char *s = PyUnicode_AsUTF8(valuetext); s)
ErrorLog(s);
Py_DECREF(valuetext);
}
}
auto *tracebacktext{PyObject_Str(traceback)};
if (tracebacktext != nullptr)
{
if (const char *s = PyUnicode_AsUTF8(tracebacktext); s)
ErrorLog(s);
Py_DECREF(tracebacktext);
}
else
{
WriteLog("Python Interpreter: failed to retrieve the stack traceback");
}
}
}
std::vector<std::string> python_external_utils::PyObjectToStringArray(PyObject *pyList)
{
std::vector<std::string> result;
std::vector<std::string> emptyIfError = {};
if (!PySequence_Check(pyList))
{
ErrorLog("Python: Failed to convert PyObject -> vector<string>");
return emptyIfError;
}
Py_ssize_t size = PySequence_Size(pyList);
for (Py_ssize_t i = 0; i < size; ++i)
{
PyObject *item = PySequence_GetItem(pyList, i); // Increments reference count
if (item == nullptr)
{
ErrorLog("Python: Failed to get item from sequence.");
return emptyIfError;
}
const char *str = PyUnicode_AsUTF8(item);
if (str == nullptr)
{
Py_DECREF(item);
ErrorLog("Python: Failed to convert item to string.");
return emptyIfError;
}
result.emplace_back(str);
Py_DECREF(item); // Decrease reference count for the item
}
return result;
}
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif