eu07/maszyna
1
0
Fork 0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-03-22 15:05:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

milek7/sim branch python taskqueue code import

Browse Source
This commit is contained in:
tmj-fstate
2020-02-22 01:41:34 +01:00
parent 6b1ab6c125
commit fcc8460d88
9 changed files with 304 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
Globals.cpp
View File

@@ -859,6 +859,16 @@ global_settings::ConfigParse(cParser &Parser) {
Parser >> gfx_shadergamma;
}
*/
else if (token == "python.threadedupload")
{
Parser.getTokens(1);
Parser >> python_threadedupload;
}
else if (token == "python.uploadmain")
{
Parser.getTokens(1);
Parser >> python_uploadmain;
}
else if (token == "python.mipmaps")
{
Parser.getTokens(1);

4
Globals.h
View File

@@ -195,7 +195,9 @@ struct global_settings {
std::string fullscreen_monitor;
bool python_mipmaps = true;
bool python_mipmaps = true;
bool python_threadedupload = true;
bool python_uploadmain = true;
int gfx_framebuffer_width = -1;
int gfx_framebuffer_height = -1;

206
PyInt.cpp
View File

@@ -13,16 +13,44 @@ http://mozilla.org/MPL/2.0/.
#include "dictionary.h"
#include "application.h"
#include "Logs.h"
#include "Globals.h"
#ifdef __GNUC__
#pragma GCC diagnostic ignored "-Wwrite-strings"
#endif
void render_task::run() {
// convert provided input to a python dictionary
auto *input = PyDict_New();
if( input == nullptr ) { goto exit; }
if (input == nullptr) {
cancel();
return;
}
for( auto const &datapair : m_input->floats ) { auto *value{ PyGetFloat( datapair.second ) }; PyDict_SetItemString( input, datapair.first.c_str(), value ); Py_DECREF( value ); }
for( auto const &datapair : m_input->integers ) { auto *value{ PyGetInt( datapair.second ) }; PyDict_SetItemString( input, datapair.first.c_str(), value ); Py_DECREF( value ); }
for( auto const &datapair : m_input->bools ) { auto *value{ PyGetBool( datapair.second ) }; PyDict_SetItemString( input, datapair.first.c_str(), value ); }
for( auto const &datapair : m_input->strings ) { auto *value{ PyGetString( datapair.second.c_str() ) }; PyDict_SetItemString( input, datapair.first.c_str(), value ); Py_DECREF( value ); }
/*
for (auto const &datapair : m_input->vec2_lists) {
PyObject *list = PyList_New(datapair.second.size());
for (size_t i = 0; i < datapair.second.size(); i++) {
auto const &vec = datapair.second[i];
PyObject *tuple = PyTuple_New(2);
PyTuple_SetItem(tuple, 0, PyGetFloat(vec.x)); // steals ref
PyTuple_SetItem(tuple, 1, PyGetFloat(vec.y)); // steals ref
PyList_SetItem(list, i, tuple); // steals ref
}
PyDict_SetItemString(input, datapair.first.c_str(), list);
Py_DECREF(list);
}
*/
delete m_input;
m_input = nullptr;
// call the renderer
auto *output { PyObject_CallMethod( m_renderer, "render", "O", input ) };
@@ -33,40 +61,82 @@ void render_task::run() {
auto *outputheight { PyObject_CallMethod( m_renderer, "get_height", nullptr ) };
// upload texture data
if( ( outputwidth != nullptr )
&& ( outputheight != nullptr ) ) {
&& ( outputheight != nullptr )
&& m_target) {
int width = PyInt_AsLong( outputwidth );
int height = PyInt_AsLong( outputheight );
int components, format;
::glBindTexture( GL_TEXTURE_2D, m_target );
// build texture
::glTexImage2D(
GL_TEXTURE_2D, 0,
( Global.GfxFramebufferSRGB ? GL_SRGB8 : GL_RGBA8 ),
PyInt_AsLong( outputwidth ), PyInt_AsLong( outputheight ), 0,
GL_RGB, GL_UNSIGNED_BYTE, reinterpret_cast<GLubyte const *>( PyString_AsString( output ) ) );
// setup texture parameters
if( ( Global.AnisotropicFiltering >= 0 )
&& ( GL_EXT_texture_filter_anisotropic != 0 ) ) {
// anisotropic filtering
::glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY_EXT, Global.AnisotropicFiltering );
}
if( Global.python_mipmaps ) {
::glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR );
::glGenerateMipmap( GL_TEXTURE_2D );
}
else {
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
}
// all done
::glFlush();
const unsigned char *image = reinterpret_cast<const unsigned char *>( PyString_AsString( output ) );
std::lock_guard<std::mutex> guard(m_target->mutex);
if (m_target->image)
delete[] m_target->image;
if (!Global.gfx_usegles)
{
int size = width * height * 3;
format = GL_SRGB8;
components = GL_RGB;
m_target->image = new unsigned char[size];
memcpy(m_target->image, image, size);
}
else
{
format = GL_SRGB8_ALPHA8;
components = GL_RGBA;
m_target->image = new unsigned char[width * height * 4];
int w = width;
int h = height;
for (int y = 0; y < h; y++)
for (int x = 0; x < w; x++)
{
m_target->image[(y * w + x) * 4 + 0] = image[(y * w + x) * 3 + 0];
m_target->image[(y * w + x) * 4 + 1] = image[(y * w + x) * 3 + 1];
m_target->image[(y * w + x) * 4 + 2] = image[(y * w + x) * 3 + 2];
m_target->image[(y * w + x) * 4 + 3] = 0xFF;
}
}
m_target->width = width;
m_target->height = height;
m_target->components = components;
m_target->format = format;
m_target->timestamp = std::chrono::high_resolution_clock::now();
}
if( outputheight != nullptr ) { Py_DECREF( outputheight ); }
if( outputwidth != nullptr ) { Py_DECREF( outputwidth ); }
Py_DECREF( output );
}
}
exit:
// clean up after yourself
delete m_input;
delete this;
void render_task::upload()
{
if (Global.python_uploadmain && m_target->image)
{
glBindTexture(GL_TEXTURE_2D, m_target->shared_tex);
glTexImage2D(
GL_TEXTURE_2D, 0,
m_target->format,
m_target->width, m_target->height, 0,
m_target->components, GL_UNSIGNED_BYTE, m_target->image);
if (Global.python_mipmaps)
{
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
glGenerateMipmap(GL_TEXTURE_2D);
}
else
{
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
}
if (Global.python_threadedupload)
glFlush();
}
delete this;
}
void render_task::cancel() {
@@ -88,9 +158,21 @@ auto python_taskqueue::init() -> bool {
Py_SetPythonHome("linuxpython64");
else
Py_SetPythonHome("linuxpython");
#elif __APPLE__
if (sizeof(void*) == 8)
Py_SetPythonHome("macpython64");
else
Py_SetPythonHome("macpython");
#endif
Py_Initialize();
PyEval_InitThreads();
m_initialized = true;
PyObject *stringiomodule { nullptr };
PyObject *stringioclassname { nullptr };
PyObject *stringioobject { nullptr };
// do the setup work while we hold the lock
m_main = PyImport_ImportModule("__main__");
if (m_main == nullptr) {
@@ -98,15 +180,15 @@ auto python_taskqueue::init() -> bool {
goto release_and_exit;
}
auto *stringiomodule { PyImport_ImportModule( "cStringIO" ) };
auto *stringioclassname { (
stringiomodule = PyImport_ImportModule( "cStringIO" );
stringioclassname = (
stringiomodule != nullptr ?
PyObject_GetAttrString( stringiomodule, "StringIO" ) :
nullptr ) };
auto *stringioobject { (
nullptr );
stringioobject = (
stringioclassname != nullptr ?
PyObject_CallObject( stringioclassname, nullptr ) :
nullptr ) };
nullptr );
m_stderr = { (
stringioobject == nullptr ? nullptr :
PySys_SetObject( "stderr", stringioobject ) != 0 ? nullptr :
@@ -117,15 +199,17 @@ auto python_taskqueue::init() -> bool {
// release the lock, save the state for future use
m_mainthread = PyEval_SaveThread();
WriteLog( "Python Interpreter: setup complete" );
WriteLog( "Python Interpreter setup complete" );
// init workers
for( auto &worker : m_workers ) {
auto *openglcontextwindow { Application.window( -1 ) };
GLFWwindow *openglcontextwindow = nullptr;
if (Global.python_threadedupload)
openglcontextwindow = Application.window( -1 );
worker = std::thread(
&python_taskqueue::run, this,
openglcontextwindow, std::ref( m_tasks ), std::ref( m_condition ), std::ref( m_exit ) );
openglcontextwindow, std::ref( m_tasks ), std::ref(m_uploadtasks), std::ref( m_condition ), std::ref( m_exit ) );
if( false == worker.joinable() ) { return false; }
}
@@ -139,12 +223,16 @@ release_and_exit:
// shuts down the module
void python_taskqueue::exit() {
if (!m_initialized)
return;
// let the workers know we're done with them
m_exit = true;
m_condition.notify_all();
// let them free up their shit before we proceed
for( auto &worker : m_workers ) {
worker.join();
if (worker.joinable())
worker.join();
}
// get rid of the leftover tasks
// with the workers dead we don't have to worry about concurrent access anymore
@@ -161,8 +249,7 @@ auto python_taskqueue::insert( task_request const &Task ) -> bool {
if( ( Task.renderer.empty() )
|| ( Task.input == nullptr )
|| ( Task.target == 0 )
|| ( Task.target == (GLuint)-1 ) ) { return false; }
|| ( Task.target == 0 ) ) { return false; }
auto *renderer { fetch_renderer( Task.renderer ) };
if( renderer == nullptr ) { return false; }
@@ -199,7 +286,8 @@ auto python_taskqueue::run_file( std::string const &File, std::string const &Pat
if( lookup.first.empty() ) { return false; }
std::ifstream inputfile { lookup.first + lookup.second };
std::string const input { std::istreambuf_iterator<char>( inputfile ), std::istreambuf_iterator<char>() };
std::string input;
input.assign( std::istreambuf_iterator<char>( inputfile ), std::istreambuf_iterator<char>() );
if( PyRun_SimpleString( input.c_str() ) != 0 ) {
error();
@@ -221,7 +309,7 @@ void python_taskqueue::release_lock() {
PyEval_SaveThread();
}
auto python_taskqueue::fetch_renderer( std::string const Renderer ) -> PyObject * {
auto python_taskqueue::fetch_renderer( std::string const Renderer ) ->PyObject * {
auto const lookup { m_renderers.find( Renderer ) };
if( lookup != std::end( m_renderers ) ) {
@@ -232,6 +320,7 @@ auto python_taskqueue::fetch_renderer( std::string const Renderer ) -> PyObject
auto const file { Renderer.substr( path.size() ) };
PyObject *renderer { nullptr };
PyObject *rendererarguments { nullptr };
PyObject *renderername { nullptr };
acquire_lock();
{
if( m_main == nullptr ) {
@@ -242,7 +331,7 @@ auto python_taskqueue::fetch_renderer( std::string const Renderer ) -> PyObject
if( false == run_file( file, path ) ) {
goto cache_and_return;
}
auto *renderername{ PyObject_GetAttrString( m_main, file.c_str() ) };
renderername = PyObject_GetAttrString( m_main, file.c_str() );
if( renderername == nullptr ) {
ErrorLog( "Python Renderer: class \"" + file + "\" not defined" );
goto cache_and_return;
@@ -271,9 +360,11 @@ cache_and_return:
return renderer;
}
void python_taskqueue::run( GLFWwindow *Context, rendertask_sequence &Tasks, threading::condition_variable &Condition, std::atomic<bool> &Exit ) {
void python_taskqueue::run( GLFWwindow *Context, rendertask_sequence &Tasks, uploadtask_sequence &Upload_Tasks, threading::condition_variable &Condition, std::atomic<bool> &Exit ) {
if (Context)
glfwMakeContextCurrent( Context );
glfwMakeContextCurrent( Context );
// create a state object for this thread
PyEval_AcquireLock();
auto *threadstate { PyThreadState_New( m_mainthread->interp ) };
@@ -302,9 +393,15 @@ void python_taskqueue::run( GLFWwindow *Context, rendertask_sequence &Tasks, thr
{
// execute python code
task->run();
if( PyErr_Occurred() != nullptr ) {
error();
}
if (Context)
task->upload();
else
{
std::lock_guard<std::mutex> lock(Upload_Tasks.mutex);
Upload_Tasks.data.push_back(task);
}
if( PyErr_Occurred() != nullptr )
error();
}
// clear the thread state
PyEval_SaveThread();
@@ -323,10 +420,19 @@ void python_taskqueue::run( GLFWwindow *Context, rendertask_sequence &Tasks, thr
PyEval_ReleaseLock();
}
void python_taskqueue::update()
{
std::lock_guard<std::mutex> lock(m_uploadtasks.mutex);
for (auto &task : m_uploadtasks.data)
task->upload();
m_uploadtasks.data.clear();
}
void
python_taskqueue::error() {
ErrorLog( "Python Interpreter: encountered error" );
if( m_stderr != nullptr ) {
// std err pythona jest buforowane
PyErr_Print();
@@ -362,3 +468,7 @@ python_taskqueue::error() {
}
}
}
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif

71
PyInt.h
View File

@@ -7,7 +7,25 @@ obtain one at
http://mozilla.org/MPL/2.0/.
*/
#pragma once
#ifndef PYINT_H
#define PYINT_H
#ifdef _POSIX_C_SOURCE
#undef _POSIX_C_SOURCE
#endif
#ifdef _XOPEN_SOURCE
#undef _XOPEN_SOURCE
#endif
#ifdef _MSC_VER
#pragma warning( push )
#pragma warning( disable : 5033 )
#endif
#ifdef __GNUC__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wregister"
#endif
#ifdef _DEBUG
#undef _DEBUG // bez tego macra Py_DECREF powoduja problemy przy linkowaniu
@@ -16,35 +34,63 @@ http://mozilla.org/MPL/2.0/.
#else
#include "Python.h"
#endif
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif
#ifdef _MSC_VER
#pragma warning( pop )
#endif
#include "Classes.h"
#include "utilities.h"
#define PyGetFloat(param) PyFloat_FromDouble(param)
#define PyGetInt(param) PyInt_FromLong(param)
#define PyGetBool(param) param ? Py_True : Py_False
#define PyGetString(param) PyString_FromString(param)
// python rendertarget
struct python_rt {
std::mutex mutex;
GLuint shared_tex;
int format;
int components;
int width;
int height;
unsigned char *image = nullptr;
std::chrono::high_resolution_clock::time_point timestamp;
~python_rt() {
if (image)
delete[] image;
}
};
// TODO: extract common base and inherit specialization from it
class render_task {
public:
// constructors
render_task( PyObject *Renderer, dictionary_source *Input, GLuint Target ) :
render_task( PyObject *Renderer, dictionary_source *Input, std::shared_ptr<python_rt> Target ) :
m_renderer( Renderer ), m_input( Input ), m_target( Target )
{}
// methods
void run();
void run();
void upload();
void cancel();
auto target() const -> texture_handle { return m_target; }
auto target() const -> std::shared_ptr<python_rt> { return m_target; }
private:
// members
PyObject *m_renderer {nullptr};
dictionary_source *m_input { nullptr };
GLuint m_target { 0 };
std::shared_ptr<python_rt> m_target { nullptr };
};
class python_taskqueue {
public:
@@ -53,7 +99,7 @@ public:
std::string const &renderer;
dictionary_source *input;
GLuint target;
std::shared_ptr<python_rt> target;
};
// constructors
python_taskqueue() = default;
@@ -71,14 +117,17 @@ public:
// releases the python gil and swaps the main thread out
void release_lock();
void update();
private:
// types
static int const WORKERCOUNT { 1 };
using worker_array = std::array<std::thread, WORKERCOUNT >;
using rendertask_sequence = threading::lockable< std::deque<render_task *> >;
using uploadtask_sequence = threading::lockable< std::deque<render_task *> >;
// methods
auto fetch_renderer( std::string const Renderer ) -> PyObject *;
void run( GLFWwindow *Context, rendertask_sequence &Tasks, threading::condition_variable &Condition, std::atomic<bool> &Exit );
void run(GLFWwindow *Context, rendertask_sequence &Tasks, uploadtask_sequence &Upload_Tasks, threading::condition_variable &Condition, std::atomic<bool> &Exit );
void error();
// members
PyObject *m_main { nullptr };
@@ -89,4 +138,8 @@ private:
std::atomic<bool> m_exit { false }; // signals the workers to quit
std::unordered_map<std::string, PyObject *> m_renderers; // cache of python classes
rendertask_sequence m_tasks;
uploadtask_sequence m_uploadtasks;
bool m_initialized { false };
};
#endif

5
Texture.cpp
View File

@@ -293,7 +293,10 @@ opengl_texture::make_request() {
auto *dictionary { new dictionary_source( components.back() ) };
if( dictionary == nullptr ) { return; }
Application.request( { ToLower( components.front() ), dictionary, id } );
auto rt = std::make_shared<python_rt>();
rt->shared_tex = id;
Application.request( { ToLower( components.front() ), dictionary, rt } );
}
void

3
Texture.h
View File

@@ -35,6 +35,8 @@ struct opengl_texture {
// releases resources allocated on the opengl end, storing local copy if requested
void
release();
void
make_stub();
void
alloc_rendertarget( GLint format, GLint components, int width, int height, int layers = 1, int samples = 1, GLint wrap = GL_CLAMP_TO_EDGE );
void
@@ -65,7 +67,6 @@ struct opengl_texture {
private:
// methods
void make_stub();
void make_request();
void load_BMP();
void load_DDS();

69
Train.cpp
View File

@@ -6913,7 +6913,12 @@ bool TTrain::Update( double const Deltatime )
&& ( false == FreeFlyModeFlag ) ) { // don't bother if we're outside
fScreenTimer = 0.f;
for( auto const &screen : m_screens ) {
Application.request( { screen.first, GetTrainState(), GfxRenderer->Texture( screen.second ).id } );
auto state_dict = GetTrainState();
/*
state_dict->insert("touches", *screen.touch_list);
screen.touch_list->clear();
*/
Application.request({ screen.rendererpath, state_dict, screen.rt } );
}
}
// sounds
@@ -7707,23 +7712,53 @@ bool TTrain::InitializeCab(int NewCabNo, std::string const &asFileName)
>> submodelname
>> renderername;
auto const *submodel { ( DynamicObject->mdKabina ? DynamicObject->mdKabina->GetFromName( submodelname ) : nullptr ) };
if( submodel == nullptr ) {
WriteLog( "Python Screen: submodel " + submodelname + " not found - Ignoring screen" );
continue;
}
auto const material { submodel->GetMaterial() };
if( material <= 0 ) {
// sub model nie posiada tekstury lub tekstura wymienna - nie obslugiwana
WriteLog( "Python Screen: invalid texture id " + std::to_string( material ) + " - Ignoring screen" );
continue;
}
const std::string rendererpath {
substr_path(renderername).empty() ? // supply vehicle folder as path if none is provided
DynamicObject->asBaseDir + renderername :
renderername };
opengl_texture *tex = nullptr;
TSubModel *submodel = nullptr;
if (submodelname != "none") {
submodel = ( DynamicObject->mdKabina ? DynamicObject->mdKabina->GetFromName( submodelname ) : nullptr );
if( submodel == nullptr ) {
WriteLog( "Python Screen: submodel " + submodelname + " not found - Ignoring screen" );
continue;
}
auto const material { submodel->GetMaterial() };
if( material <= 0 ) {
// sub model nie posiada tekstury lub tekstura wymienna - nie obslugiwana
WriteLog( "Python Screen: invalid texture id " + std::to_string( material ) + " - Ignoring screen" );
continue;
}
tex = &GfxRenderer->Texture(GfxRenderer->Material(material).textures[0]);
}
else {
// TODO: fix leak
tex = new opengl_texture();
tex->make_stub();
}
tex->create();
auto touch_list = std::make_shared<std::vector<glm::vec2>>();
auto rt = std::make_shared<python_rt>();
rt->shared_tex = tex->id;
/*
if (submodel)
submodel->screen_touch_list = touch_list;
*/
// record renderer and material binding for future update requests
m_screens.emplace_back(
( substr_path(renderername).empty() ? // supply vehicle folder as path if none is provided
DynamicObject->asBaseDir + renderername :
renderername ),
GfxRenderer->Material( material ).textures[0] );
m_screens.emplace_back();
m_screens.back().rendererpath = rendererpath;
m_screens.back().rt = rt;
/*
m_screens.back().touch_list = touch_list;
if (Global.python_displaywindows)
m_screens.back().viewer = std::make_unique<python_screen_viewer>(rt, touch_list, rendererpath);
*/
}
// btLampkaUnknown.Init("unknown",mdKabina,false);
} while (token != "");

13
Train.h
View File

@@ -110,6 +110,17 @@ class TTrain {
bool springbrake_active;
};
struct screen_entry {
std::string rendererpath;
std::shared_ptr<python_rt> rt;
/*
std::unique_ptr<python_screen_viewer> viewer;
std::shared_ptr<std::vector<glm::vec2>> touch_list;
*/
};
typedef std::vector<screen_entry> screen_map;
// constructors
TTrain();
// methods
@@ -760,7 +771,7 @@ private:
float fPPress, fNPress;
bool m_mastercontrollerinuse { false };
float m_mastercontrollerreturndelay { 0.f };
std::vector<std::pair<std::string, texture_handle>> m_screens;
screen_map m_screens;
uint16_t vid { 0 }; // train network recipient id
float m_distancecounter { -1.f }; // distance traveled since meter was activated or -1 if inactive
double m_brakehandlecp{ 0.0 };

2
application.cpp
View File

@@ -306,7 +306,7 @@ eu07_application::run() {
// -------------------------------------------------------------------
// TODO: re-enable after python changes merge
// m_taskqueue.update();
m_taskqueue.update();
if (!GfxRenderer->Render())
return 0;