/* This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "scripting/PyInt.h" #include "stdafx.h" #include "application/application.h" #include "utilities/Globals.h" #include "utilities/Logs.h" #include "utilities/dictionary.h" #ifdef __GNUC__ #pragma GCC diagnostic ignored "-Wwrite-strings" #endif #include void render_task::run() { // convert provided input to a python dictionary auto *input = PyDict_New(); if (input == nullptr) { cancel(); return; } for (auto const &datapair : m_input->floats) { auto *value{PyGetFloat(datapair.second)}; PyDict_SetItemString(input, datapair.first.c_str(), value); Py_DECREF(value); } for (auto const &datapair : m_input->integers) { auto *value{PyGetInt(datapair.second)}; PyDict_SetItemString(input, datapair.first.c_str(), value); Py_DECREF(value); } for (auto const &datapair : m_input->bools) { auto *value{PyGetBool(datapair.second)}; PyDict_SetItemString(input, datapair.first.c_str(), value); } for (auto const &datapair : m_input->strings) { auto *value{PyGetString(datapair.second.c_str())}; PyDict_SetItemString(input, datapair.first.c_str(), value); Py_DECREF(value); } for (auto const &datapair : m_input->vec2_lists) { PyObject *list = PyList_New(datapair.second.size()); for (size_t i = 0; i < datapair.second.size(); i++) { auto const &vec = datapair.second[i]; WriteLog("passing " + glm::to_string(vec)); PyObject *tuple = PyTuple_New(2); PyTuple_SetItem(tuple, 0, PyGetFloat(vec.x)); // steals ref PyTuple_SetItem(tuple, 1, PyGetFloat(vec.y)); // steals ref PyList_SetItem(list, i, tuple); // steals ref } PyDict_SetItemString(input, datapair.first.c_str(), list); Py_DECREF(list); } m_input = nullptr; // call the renderer auto *output{PyObject_CallMethod(m_renderer, const_cast("render"), const_cast("O"), input)}; Py_DECREF(input); if (output != nullptr) { auto *outputWidth = PyObject_CallMethod(m_renderer, const_cast("get_width"), nullptr); auto *outputHeight = PyObject_CallMethod(m_renderer, const_cast("get_height"), nullptr); if (outputWidth != nullptr && outputHeight != nullptr && m_target != nullptr) { const int screenWidth = static_cast(PyInt_AsLong(outputWidth)); const int screenHeight = static_cast(PyInt_AsLong(outputHeight)); const bool useRgb = (false && !Global.gfx_usegles); const int glFormat = useRgb ? GL_SRGB8 : GL_SRGB8_ALPHA8; const int glComponents = useRgb ? GL_RGB : GL_RGBA; const size_t bytesPerPixel = useRgb ? 3u : 4u; const size_t expectedBytes = static_cast(screenWidth) * static_cast(screenHeight) * bytesPerPixel; Py_ssize_t pythonBufferBytes = 0; char *pythonBufferPtr = nullptr; const bool bufferExtracted = (PyString_AsStringAndSize(output, &pythonBufferPtr, &pythonBufferBytes) == 0) && (pythonBufferPtr != nullptr); if (!bufferExtracted) { ErrorLog("Python screen renderer: output is not a valid byte buffer"); } else if (pythonBufferBytes < static_cast(expectedBytes)) { ErrorLog(std::format("Python screen renderer: output buffer too small ({} bytes, expected {})", pythonBufferBytes, expectedBytes)); } else { std::lock_guard guard(m_target->mutex); if (m_target->image.size() != expectedBytes) m_target->image.resize(expectedBytes); std::memcpy(m_target->image.data(), pythonBufferPtr, expectedBytes); m_target->width = screenWidth; m_target->height = screenHeight; m_target->components = glComponents; m_target->format = glFormat; m_target->timestamp = std::chrono::high_resolution_clock::now(); } } if (outputHeight != nullptr) Py_DECREF(outputHeight); if (outputWidth != nullptr) Py_DECREF(outputWidth); Py_DECREF(output); } // get commands from renderer auto *commandsPO = PyObject_CallMethod(m_renderer, const_cast("getCommands"), nullptr); if (commandsPO != nullptr) { std::vector commands = python_external_utils::PyObjectToStringArray(commandsPO); Py_DECREF(commandsPO); // we perform any actions ONLY when there are any commands in buffer if (!commands.empty()) { for (const auto &cmd : commands) { std::string baseCmd; int p1 = 0, p2 = 0; size_t pos1 = cmd.find(';'); if (pos1 == std::string::npos) { baseCmd = cmd; } else { baseCmd = cmd.substr(0, pos1); size_t pos2 = cmd.find(';', pos1 + 1); if (pos2 == std::string::npos) { p1 = std::stoi(cmd.substr(pos1 + 1)); } else { p1 = std::stoi(cmd.substr(pos1 + 1, pos2 - pos1 - 1)); p2 = std::stoi(cmd.substr(pos2 + 1)); } } auto it = simulation::commandMap.find(baseCmd); if (it != simulation::commandMap.end()) { command_data cd; cd.command = it->second; cd.action = GLFW_PRESS; cd.param1 = p1; cd.param2 = p2; WriteLog("Python: Executing command [" + baseCmd + "] with params: P1=" + std::to_string(p1) + " P2=" + std::to_string(p2) + " Target ID=" + std::to_string(simulation::Train->id())); simulation::Commands.push(cd, static_cast(command_target::vehicle) | simulation::Train->id()); } else { ErrorLog("Python: Command [" + baseCmd + "] not found!"); } } } } } void render_task::upload() { if (Global.python_uploadmain && m_target && m_target->shared_tex) { m_target->shared_tex->update_from_memory(m_target->width, m_target->height, reinterpret_cast(m_target->image.data())); // glBindTexture(GL_TEXTURE_2D, m_target->shared_tex->get_id()); // glTexImage2D( // GL_TEXTURE_2D, 0, // m_target->format, // m_target->width, m_target->height, 0, // m_target->components, GL_UNSIGNED_BYTE, m_target->image); // // if (Global.python_mipmaps) //{ // glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR); // glGenerateMipmap(GL_TEXTURE_2D); //} // else //{ // glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); //} // // if (Global.python_threadedupload) // glFlush(); } } void render_task::cancel() {} // initializes the module. returns true on success auto python_taskqueue::init() -> bool { crashreport_add_info("python.threadedupload", Global.python_threadedupload ? "yes" : "no"); crashreport_add_info("python.uploadmain", Global.python_uploadmain ? "yes" : "no"); #ifdef _WIN32 if (sizeof(void *) == 8) Py_SetPythonHome(const_cast("python64")); else Py_SetPythonHome(const_cast("python")); #elif __linux__ if (sizeof(void *) == 8) Py_SetPythonHome(const_cast("linuxpython64")); else Py_SetPythonHome(const_cast("linuxpython")); #elif __APPLE__ if (sizeof(void *) == 8) Py_SetPythonHome(const_cast("macpython64")); else Py_SetPythonHome(const_cast("macpython")); #endif Py_InitializeEx(0); PyEval_InitThreads(); PyObject *stringiomodule{nullptr}; PyObject *stringioclassname{nullptr}; PyObject *stringioobject{nullptr}; // do the setup work while we hold the lock m_main = PyImport_ImportModule("__main__"); if (m_main == nullptr) { ErrorLog("Python Interpreter: __main__ module is missing"); goto release_and_exit; } stringiomodule = PyImport_ImportModule("cStringIO"); stringioclassname = (stringiomodule != nullptr ? PyObject_GetAttrString(stringiomodule, "StringIO") : nullptr); stringioobject = (stringioclassname != nullptr ? PyObject_CallObject(stringioclassname, nullptr) : nullptr); m_stderr = {(stringioobject == nullptr ? nullptr : PySys_SetObject(const_cast("stderr"), stringioobject) != 0 ? nullptr : stringioobject)}; if (false == run_file("abstractscreenrenderer")) { goto release_and_exit; } // release the lock, save the state for future use m_mainthread = PyEval_SaveThread(); WriteLog("Python Interpreter: setup complete"); // init workers for (auto &worker : m_workers) { GLFWwindow *openglcontextwindow = nullptr; if (Global.python_threadedupload) openglcontextwindow = Application.window(-1); worker = std::jthread(&python_taskqueue::run, this, openglcontextwindow, std::ref(m_tasks), std::ref(m_uploadtasks), std::ref(m_condition), std::ref(m_exit)); if (false == worker.joinable()) { return false; } } m_initialized = true; return true; release_and_exit: PyEval_ReleaseLock(); return false; } // shuts down the module void python_taskqueue::exit() { if (!m_initialized) return; // let the workers know we're done with them m_exit = true; m_condition.notify_all(); // let them free up their shit before we proceed m_workers = {}; // get rid of the leftover tasks // with the workers dead we don't have to worry about concurrent access anymore for (auto task : m_tasks.data) { task->cancel(); } // take a bow acquire_lock(); Py_Finalize(); } // adds specified task along with provided collection of data to the work queue. returns true on success auto python_taskqueue::insert(task_request const &Task) -> bool { if (!m_initialized || (false == Global.python_enabled) || (Task.renderer.empty()) || (Task.input == nullptr) || (Task.target == 0)) { return false; } auto *renderer{fetch_renderer(Task.renderer)}; if (renderer == nullptr) { return false; } auto newtask = std::make_shared(renderer, Task.input, Task.target); bool newtaskinserted{false}; // acquire a lock on the task queue and add the new task { std::lock_guard lock(m_tasks.mutex); // check the task list for a pending request with the same target for (auto &task : m_tasks.data) { if (task->target() == Task.target) { // replace pending task in the slot with the more recent one task->cancel(); task = newtask; newtaskinserted = true; break; } } if (false == newtaskinserted) { m_tasks.data.emplace_back(newtask); } } // potentially wake a worker to handle the new task m_condition.notify_one(); // all done return true; } // executes python script stored in specified file. returns true on success auto python_taskqueue::run_file(std::string const &File, std::string const &Path) -> bool { auto const lookup{FileExists({Path + File, "python/local/" + File}, {".py"})}; if (lookup.first.empty()) { return false; } std::ifstream inputfile{lookup.first + lookup.second}; std::string input; input.assign(std::istreambuf_iterator(inputfile), std::istreambuf_iterator()); if (PyRun_SimpleString(input.c_str()) != 0) { error(); return false; } return true; } // acquires the python gil and sets the main thread as current void python_taskqueue::acquire_lock() { PyEval_RestoreThread(m_mainthread); } // releases the python gil and swaps the main thread out void python_taskqueue::release_lock() { PyEval_SaveThread(); } auto python_taskqueue::fetch_renderer(std::string const Renderer) -> PyObject * { auto const lookup{m_renderers.find(Renderer)}; if (lookup != std::end(m_renderers)) { return lookup->second; } // try to load specified renderer class auto const path{substr_path(Renderer)}; auto const file{Renderer.substr(path.size())}; PyObject *renderer{nullptr}; PyObject *rendererarguments{nullptr}; PyObject *renderername{nullptr}; acquire_lock(); { if (m_main == nullptr) { ErrorLog("Python Renderer: __main__ module is missing"); goto cache_and_return; } if (false == run_file(file, path)) { goto cache_and_return; } renderername = PyObject_GetAttrString(m_main, file.c_str()); if (renderername == nullptr) { ErrorLog("Python Renderer: class \"" + file + "\" not defined"); goto cache_and_return; } rendererarguments = Py_BuildValue("(s)", path.c_str()); if (rendererarguments == nullptr) { ErrorLog("Python Renderer: failed to create initialization arguments"); goto cache_and_return; } renderer = PyObject_CallObject(renderername, rendererarguments); PyObject_CallMethod(renderer, const_cast("manul_set_format"), const_cast("(s)"), "RGBA"); if (PyErr_Occurred() != nullptr) { error(); renderer = nullptr; } cache_and_return: // clean up after yourself if (rendererarguments != nullptr) { Py_DECREF(rendererarguments); } } release_lock(); // cache the failures as well so we don't try again on subsequent requests m_renderers.emplace(Renderer, renderer); return renderer; } void python_taskqueue::run(GLFWwindow *Context, rendertask_sequence &Tasks, uploadtask_sequence &Upload_Tasks, threading::condition_variable &Condition, std::atomic &Exit) { if (Context) glfwMakeContextCurrent(Context); // create a state object for this thread PyEval_AcquireLock(); auto *threadstate{PyThreadState_New(m_mainthread->interp)}; PyEval_ReleaseLock(); std::shared_ptr task{nullptr}; while (false == Exit.load()) { // regardless of the reason we woke up prime the spurious wakeup flag for the next time Condition.spurious(true); // keep working as long as there's any scheduled tasks do { task = nullptr; // acquire a lock on the task queue and potentially grab a task from it { std::lock_guard lock(Tasks.mutex); if (false == Tasks.data.empty()) { // fifo task = Tasks.data.front(); Tasks.data.pop_front(); } } if (task != nullptr) { // swap in my thread state PyEval_RestoreThread(threadstate); { // execute python code task->run(); if (Context) task->upload(); else { std::lock_guard lock(Upload_Tasks.mutex); Upload_Tasks.data.push_back(task); } if (PyErr_Occurred() != nullptr) error(); } // clear the thread state PyEval_SaveThread(); } // TBD, TODO: add some idle time between tasks in case we're on a single thread cpu? } while (task != nullptr); // if there's nothing left to do wait until there is // but check every now and then on your own to minimize potential deadlock situations Condition.wait_for(std::chrono::seconds(5)); } // clean up thread state data PyEval_AcquireLock(); PyThreadState_Swap(nullptr); PyThreadState_Clear(threadstate); PyThreadState_Delete(threadstate); PyEval_ReleaseLock(); } void python_taskqueue::update() { std::lock_guard lock(m_uploadtasks.mutex); for (auto &task : m_uploadtasks.data) task->upload(); m_uploadtasks.data.clear(); } void python_taskqueue::error() { if (m_stderr != nullptr) { // std err pythona jest buforowane PyErr_Print(); auto *errortext{PyObject_CallMethod(m_stderr, const_cast("getvalue"), nullptr)}; ErrorLog(PyString_AsString(errortext)); // czyscimy bufor na kolejne bledy PyObject_CallMethod(m_stderr, const_cast("truncate"), const_cast("i"), 0); } else { // nie dziala buffor pythona PyObject *type, *value, *traceback; PyErr_Fetch(&type, &value, &traceback); if (type == nullptr) { ErrorLog("Python Interpreter: don't know how to handle null exception"); } PyErr_NormalizeException(&type, &value, &traceback); if (type == nullptr) { ErrorLog("Python Interpreter: don't know how to handle null exception"); } auto *typetext{PyObject_Str(type)}; if (typetext != nullptr) { ErrorLog(PyString_AsString(typetext)); } if (value != nullptr) { ErrorLog(PyString_AsString(value)); } auto *tracebacktext{PyObject_Str(traceback)}; if (tracebacktext != nullptr) { ErrorLog(PyString_AsString(tracebacktext)); } else { WriteLog("Python Interpreter: failed to retrieve the stack traceback"); } } } std::vector python_external_utils::PyObjectToStringArray(PyObject *pyList) { std::vector result; std::vector emptyIfError = {}; if (!PySequence_Check(pyList)) { ErrorLog("Python: Failed to convert PyObject -> vector"); return emptyIfError; } Py_ssize_t size = PySequence_Size(pyList); for (Py_ssize_t i = 0; i < size; ++i) { PyObject *item = PySequence_GetItem(pyList, i); // Increments reference count if (item == nullptr) { ErrorLog("Python: Failed to get item from sequence."); return emptyIfError; } const char *str = PyString_AsString(item); if (str == nullptr) { Py_DECREF(item); ErrorLog("Python: Failed to convert item to string."); return emptyIfError; } result.push_back(std::string(str)); Py_DECREF(item); // Decrease reference count for the item } return result; } #ifdef __GNUC__ #pragma GCC diagnostic pop #endif