mirror of
https://github.com/MaSzyna-EU07/maszyna.git
synced 2026-11-18 22:16:02 +01:00
Script screens: led_post and resize of the Lua screens on the gpu
This commit is contained in:
@@ -145,9 +145,63 @@ template <typename Type_> Type_ &check_object(lua_State *L, int const Index, cha
|
||||
return *object;
|
||||
}
|
||||
|
||||
// an image of a script: what was drawn on it, and the post-processing (image:led_post, image:resize) asked for and not
|
||||
// run yet. the steps of the image render returns run in the queue, on the gpu when the renderer can; the steps of an
|
||||
// image the script goes on to read or draw on (or with) run here, on the cpu, when it does
|
||||
struct lua_image
|
||||
{
|
||||
canvas drawn;
|
||||
std::vector<post_step> steps;
|
||||
|
||||
int width() const
|
||||
{
|
||||
auto result{drawn.width()};
|
||||
for (auto const &step : steps)
|
||||
{
|
||||
result = width_after(step, result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
int height() const
|
||||
{
|
||||
auto result{drawn.height()};
|
||||
for (auto const &step : steps)
|
||||
{
|
||||
result = height_after(step, result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
// the image with its post-processing done
|
||||
canvas &done()
|
||||
{
|
||||
if (false == steps.empty())
|
||||
{
|
||||
post_image image{drawn.pixels(), drawn.width(), drawn.height()};
|
||||
// the sizes were checked when the steps were asked for
|
||||
apply_post(image, steps);
|
||||
drawn = canvas{image.width, image.height, std::move(image.pixels)};
|
||||
steps.clear();
|
||||
}
|
||||
return drawn;
|
||||
}
|
||||
// the image, a step further
|
||||
lua_image then(post_step const &Step) const
|
||||
{
|
||||
lua_image result{drawn, steps};
|
||||
result.steps.push_back(Step);
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
lua_image &check_lua_image(lua_State *L, int const Index)
|
||||
{
|
||||
return check_object<lua_image>(L, Index, image_metatable);
|
||||
}
|
||||
|
||||
// the image, to read or draw on: its post-processing is done first
|
||||
canvas &check_image(lua_State *L, int const Index)
|
||||
{
|
||||
return check_object<canvas>(L, Index, image_metatable);
|
||||
return check_lua_image(L, Index).done();
|
||||
}
|
||||
|
||||
int check_int(lua_State *L, int const Index)
|
||||
@@ -226,7 +280,7 @@ int api_image(lua_State *L)
|
||||
luaL_argcheck(L, valid_image_side(width), 1, "between 1 and 4096");
|
||||
luaL_argcheck(L, valid_image_side(height), 2, "between 1 and 4096");
|
||||
auto const color{lua_isnoneornil(L, 3) ? rgb{0, 0, 0} : check_color(L, 3)};
|
||||
push_object(L, canvas{width, height, color}, image_metatable);
|
||||
push_object(L, lua_image{canvas{width, height, color}, {}}, image_metatable);
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -388,13 +442,13 @@ int api_include(lua_State *L)
|
||||
|
||||
int image_width(lua_State *L)
|
||||
{
|
||||
lua_pushinteger(L, check_image(L, 1).width());
|
||||
lua_pushinteger(L, check_lua_image(L, 1).width());
|
||||
return 1;
|
||||
}
|
||||
|
||||
int image_height(lua_State *L)
|
||||
{
|
||||
lua_pushinteger(L, check_image(L, 1).height());
|
||||
lua_pushinteger(L, check_lua_image(L, 1).height());
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -498,36 +552,39 @@ int image_draw_image(lua_State *L)
|
||||
return 0;
|
||||
}
|
||||
|
||||
// image:led_post([bloom]): led_util's render_post, a new image (2w + 2) x (2h + 2)
|
||||
// image:led_post([bloom]): led_util's render_post, a new image (2w + 2) x (2h + 2). done when the image is shown or used
|
||||
int image_led_post(lua_State *L)
|
||||
{
|
||||
auto const &image{check_image(L, 1)};
|
||||
auto const &image{check_lua_image(L, 1)};
|
||||
auto const bloom{lua_isnoneornil(L, 2) || lua_toboolean(L, 2) != 0};
|
||||
luaL_argcheck(L, valid_image_side(image.width() * 2 + 2) && valid_image_side(image.height() * 2 + 2), 1, "too large");
|
||||
std::vector<std::uint8_t> pixels(image_size(image.width() * 2 + 2, image.height() * 2 + 2));
|
||||
led_post(image.pixels(), image.width(), image.height(), pixels, bloom);
|
||||
push_object(L, canvas{image.width() * 2 + 2, image.height() * 2 + 2, std::move(pixels)}, image_metatable);
|
||||
post_step step;
|
||||
step.type = post_step::kind::led;
|
||||
step.bloom = bloom;
|
||||
push_object(L, image.then(step), image_metatable);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// image:resize(width, height): scaled as Pillow's resize(..., LANCZOS), a new image
|
||||
// image:resize(width, height): scaled as Pillow's resize(..., LANCZOS), a new image. done when the image is shown or used
|
||||
int image_resize(lua_State *L)
|
||||
{
|
||||
auto const &image{check_image(L, 1)};
|
||||
auto const &image{check_lua_image(L, 1)};
|
||||
auto const width{check_int(L, 2)};
|
||||
auto const height{check_int(L, 3)};
|
||||
luaL_argcheck(L, valid_image_side(width), 2, "between 1 and 4096");
|
||||
luaL_argcheck(L, valid_image_side(height), 3, "between 1 and 4096");
|
||||
std::vector<std::uint8_t> pixels(image_size(width, height));
|
||||
resize_lanczos(image.pixels(), image.width(), image.height(), pixels, width, height);
|
||||
push_object(L, canvas{width, height, std::move(pixels)}, image_metatable);
|
||||
post_step step;
|
||||
step.type = post_step::kind::resize;
|
||||
step.width = width;
|
||||
step.height = height;
|
||||
push_object(L, image.then(step), image_metatable);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int image_copy(lua_State *L)
|
||||
{
|
||||
auto const &image{check_image(L, 1)};
|
||||
push_object(L, canvas{image}, image_metatable);
|
||||
auto const &image{check_lua_image(L, 1)};
|
||||
push_object(L, lua_image{image}, image_metatable);
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -695,7 +752,7 @@ void lua_program::register_api()
|
||||
lua_pushlightuserdata(L, this);
|
||||
lua_rawset(L, LUA_REGISTRYINDEX);
|
||||
|
||||
make_metatable(L, image_metatable, image_methods, collect<canvas>);
|
||||
make_metatable(L, image_metatable, image_methods, collect<lua_image>);
|
||||
make_metatable(L, font_metatable, font_methods, collect<bitmap_font>);
|
||||
make_metatable(L, sprite_metatable, sprite_methods, collect<glyph>);
|
||||
|
||||
@@ -806,11 +863,12 @@ bool lua_program::render(render_call &Call)
|
||||
}
|
||||
// nil: the screen didn't change, keep the image there is
|
||||
auto succeeded{true};
|
||||
if (auto const *image{test_object<canvas>(m_state, -1, image_metatable)}; image != nullptr)
|
||||
if (auto const *image{test_object<lua_image>(m_state, -1, image_metatable)}; image != nullptr)
|
||||
{
|
||||
Call.pixels = image->pixels();
|
||||
Call.width = image->width();
|
||||
Call.height = image->height();
|
||||
Call.pixels = image->drawn.pixels();
|
||||
Call.width = image->drawn.width();
|
||||
Call.height = image->drawn.height();
|
||||
Call.post = image->steps;
|
||||
}
|
||||
else if (false == lua_isnil(m_state, -1))
|
||||
{
|
||||
|
||||
@@ -14,6 +14,7 @@ http://mozilla.org/MPL/2.0/.
|
||||
#include <array>
|
||||
#include <cmath>
|
||||
#include <numbers>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace script_screens
|
||||
@@ -239,4 +240,42 @@ bool resize_lanczos(std::span<std::uint8_t const> const Source, int const Width,
|
||||
return true;
|
||||
}
|
||||
|
||||
int width_after(post_step const &Step, int const Width)
|
||||
{
|
||||
return Step.type == post_step::kind::led ? Width * 2 + 2 : Step.width;
|
||||
}
|
||||
|
||||
int height_after(post_step const &Step, int const Height)
|
||||
{
|
||||
return Step.type == post_step::kind::led ? Height * 2 + 2 : Step.height;
|
||||
}
|
||||
|
||||
bool apply_post(post_image &Image, std::span<post_step const> const Steps)
|
||||
{
|
||||
if (Steps.empty())
|
||||
{
|
||||
return true;
|
||||
}
|
||||
auto result{Image};
|
||||
for (auto const &step : Steps)
|
||||
{
|
||||
auto const width{width_after(step, result.width)};
|
||||
auto const height{height_after(step, result.height)};
|
||||
if (width <= 0 || height <= 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
std::vector<std::uint8_t> pixels(image_size(width, height));
|
||||
auto const done{step.type == post_step::kind::led ? led_post(result.pixels, result.width, result.height, pixels, step.bloom)
|
||||
: resize_lanczos(result.pixels, result.width, result.height, pixels, width, height)};
|
||||
if (false == done)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
result = post_image{std::move(pixels), width, height};
|
||||
}
|
||||
Image = std::move(result);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace script_screens
|
||||
|
||||
@@ -9,8 +9,10 @@ http://mozilla.org/MPL/2.0/.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
// image filters the script screens call through the host (led_post, resize_lanczos), run natively rather than
|
||||
// in the interpreter. they give the very pixels of the Python screens: led_util.render_post() and Pillow's
|
||||
@@ -33,4 +35,32 @@ bool led_post(std::span<std::uint8_t const> Source, int Width, int Height, std::
|
||||
// for sizes that don't fit the spans
|
||||
bool resize_lanczos(std::span<std::uint8_t const> Source, int Width, int Height, std::span<std::uint8_t> Target, int TargetWidth, int TargetHeight);
|
||||
|
||||
// one step of the post-processing a script asks for on its image (image:led_post, image:resize). the steps of the image
|
||||
// render returns run on the gpu when the renderer can (script_screens::post_processor), otherwise on the cpu
|
||||
struct post_step
|
||||
{
|
||||
enum class kind
|
||||
{
|
||||
led, // led_post
|
||||
resize // resize_lanczos
|
||||
};
|
||||
kind type{kind::led};
|
||||
bool bloom{true}; // led: with the glow
|
||||
int width{0}; // resize: the size it gives
|
||||
int height{0};
|
||||
};
|
||||
|
||||
struct post_image
|
||||
{
|
||||
std::vector<std::uint8_t> pixels; // RGBA, from the top row down
|
||||
int width{0};
|
||||
int height{0};
|
||||
};
|
||||
|
||||
// the size of an image Width wide, or Height high, after the step
|
||||
int width_after(post_step const &Step, int Width);
|
||||
int height_after(post_step const &Step, int Height);
|
||||
// runs the steps on the cpu, one after another. false for sizes the kernels refuse; the image is then as it was
|
||||
bool apply_post(post_image &Image, std::span<post_step const> Steps);
|
||||
|
||||
} // namespace script_screens
|
||||
|
||||
@@ -47,6 +47,8 @@ struct render_call
|
||||
std::vector<std::uint8_t> pixels;
|
||||
int width{0};
|
||||
int height{0};
|
||||
// the post-processing still to run on the image (image:led_post, image:resize of the Lua screens)
|
||||
std::vector<post_step> post;
|
||||
};
|
||||
|
||||
// a running script: a Lua state, a WebAssembly instance. made, called and destroyed on the worker thread, or by the
|
||||
|
||||
@@ -31,6 +31,7 @@ http://mozilla.org/MPL/2.0/.
|
||||
#include <iterator>
|
||||
#include <limits>
|
||||
#include <mutex>
|
||||
#include <span>
|
||||
#include <stop_token>
|
||||
#include <string_view>
|
||||
#include <thread>
|
||||
@@ -129,6 +130,32 @@ std::string_view language_name(script_language const Language)
|
||||
return Language == script_language::lua ? "Lua" : "WASM";
|
||||
}
|
||||
|
||||
// the rows of the image the other way round. the script draws from the top row down; the texture takes the bottom row
|
||||
// first, as the python screens give it (abstractscreenrenderer flips the image before handing it over)
|
||||
std::vector<std::uint8_t> flipped(std::span<std::uint8_t const> const Pixels, int const Width, int const Height)
|
||||
{
|
||||
auto const row{static_cast<std::size_t>(Width) * 4};
|
||||
auto const height{static_cast<std::size_t>(Height)};
|
||||
std::vector<std::uint8_t> result(Pixels.size());
|
||||
for (std::size_t y = 0; y < height; ++y)
|
||||
{
|
||||
std::memcpy(result.data() + (height - 1 - y) * row, Pixels.data() + y * row, row);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// the post-processing of the image done on the cpu. false when the kernels refused a size
|
||||
bool post_on_cpu(std::vector<std::uint8_t> &Pixels, int &Width, int &Height, std::vector<script_screens::post_step> &Steps)
|
||||
{
|
||||
script_screens::post_image image{std::move(Pixels), Width, Height};
|
||||
auto const done{script_screens::apply_post(image, Steps)};
|
||||
Pixels = std::move(image.pixels);
|
||||
Width = image.width;
|
||||
Height = image.height;
|
||||
Steps.clear();
|
||||
return done;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
class script_screen
|
||||
@@ -159,11 +186,13 @@ class script_screen
|
||||
// the worker's, and the queue's once the worker stopped
|
||||
std::unique_ptr<script_screens::program> program;
|
||||
|
||||
// the image, handed from the worker to the main thread, bottom row first as the texture takes it
|
||||
// the image, handed from the worker to the main thread: bottom row first as the texture takes it, or, while it has
|
||||
// post-processing to run on the gpu, from the top row down as the script drew it
|
||||
std::mutex image_mutex;
|
||||
std::vector<std::uint8_t> image;
|
||||
int image_width{0};
|
||||
int image_height{0};
|
||||
std::vector<script_screens::post_step> post;
|
||||
bool image_pending{false}; // drawn and not yet uploaded
|
||||
std::size_t uploaded_texture{no_texture}; // the gl texture the image went to, to upload it again when it's made anew
|
||||
|
||||
@@ -188,7 +217,9 @@ struct script_screen_queue::implementation
|
||||
return Language == script_language::lua ? lua.get() : wasm.get();
|
||||
}
|
||||
static void present(script_screen &Screen, script_screens::render_call &Call);
|
||||
static void upload(script_screen &Screen);
|
||||
void upload(script_screen &Screen);
|
||||
// the post-processing of the image the gpu couldn't do, done on the cpu. false when it failed too
|
||||
static bool finish_on_cpu(script_screen &Screen);
|
||||
static void execute(script_screens::pending_command const &Command);
|
||||
|
||||
std::unique_ptr<script_screens::runtime> lua;
|
||||
@@ -196,6 +227,11 @@ struct script_screen_queue::implementation
|
||||
bool initialized{false};
|
||||
std::atomic<std::uint64_t> frame{0};
|
||||
|
||||
// the main thread's: the gpu post-processing, none when the renderer has none, or it failed
|
||||
script_screens::post_processor *post_processor{nullptr};
|
||||
// read by the worker: the post-processing is left to the main thread, for the gpu
|
||||
std::atomic<bool> post_on_gpu{false};
|
||||
|
||||
std::mutex screens_mutex;
|
||||
std::unordered_map<std::string, std::weak_ptr<script_screen>> screens;
|
||||
|
||||
@@ -273,6 +309,12 @@ void script_screen_queue::implementation::process(task const &Task)
|
||||
screen.failed = true;
|
||||
return;
|
||||
}
|
||||
if (false == call.post.empty() && false == post_on_gpu && false == post_on_cpu(call.pixels, call.width, call.height, call.post))
|
||||
{
|
||||
ErrorLog(std::format("{} screen \"{}\": the post-processing of the image failed", language_name(screen.language), screen.script));
|
||||
screen.failed = true;
|
||||
return;
|
||||
}
|
||||
present(screen, call);
|
||||
}
|
||||
if (false == call.commands.empty())
|
||||
@@ -282,23 +324,37 @@ void script_screen_queue::implementation::process(task const &Task)
|
||||
}
|
||||
}
|
||||
|
||||
// keeps the image the screen drew. the script draws from the top row down; the texture takes the bottom row first, as
|
||||
// the python screens give it (abstractscreenrenderer flips the image before handing it over)
|
||||
// keeps the image the screen drew, turned bottom row first for the texture; with post-processing left for the gpu, as
|
||||
// it was drawn
|
||||
void script_screen_queue::implementation::present(script_screen &Screen, script_screens::render_call &Call)
|
||||
{
|
||||
auto const row{static_cast<std::size_t>(Call.width) * 4};
|
||||
auto const height{static_cast<std::size_t>(Call.height)};
|
||||
std::scoped_lock lock{Screen.image_mutex};
|
||||
Screen.image.resize(Call.pixels.size());
|
||||
for (std::size_t y = 0; y < height; ++y)
|
||||
if (Call.post.empty())
|
||||
{
|
||||
std::memcpy(Screen.image.data() + (height - 1 - y) * row, Call.pixels.data() + y * row, row);
|
||||
Screen.image = flipped(Call.pixels, Call.width, Call.height);
|
||||
}
|
||||
else
|
||||
{
|
||||
Screen.image = std::move(Call.pixels);
|
||||
}
|
||||
Screen.post = std::move(Call.post);
|
||||
Screen.image_width = Call.width;
|
||||
Screen.image_height = Call.height;
|
||||
Screen.image_pending = true;
|
||||
}
|
||||
|
||||
bool script_screen_queue::implementation::finish_on_cpu(script_screen &Screen)
|
||||
{
|
||||
if (false == post_on_cpu(Screen.image, Screen.image_width, Screen.image_height, Screen.post))
|
||||
{
|
||||
ErrorLog(std::format("{} screen \"{}\": the post-processing of the image failed", language_name(Screen.language), Screen.script));
|
||||
Screen.image.clear();
|
||||
return false;
|
||||
}
|
||||
Screen.image = flipped(Screen.image, Screen.image_width, Screen.image_height);
|
||||
return true;
|
||||
}
|
||||
|
||||
void script_screen_queue::implementation::upload(script_screen &Screen)
|
||||
{
|
||||
if (Screen.texture == nullptr)
|
||||
@@ -315,7 +371,26 @@ void script_screen_queue::implementation::upload(script_screen &Screen)
|
||||
{
|
||||
return;
|
||||
}
|
||||
Screen.texture->update_from_memory(Screen.image_width, Screen.image_height, Screen.image.data());
|
||||
auto const on_gpu{false == Screen.post.empty() && post_processor != nullptr &&
|
||||
post_processor->apply(*Screen.texture, Screen.image, Screen.image_width, Screen.image_height, Screen.post)};
|
||||
if (false == on_gpu)
|
||||
{
|
||||
if (false == Screen.post.empty())
|
||||
{
|
||||
// the gpu can't do it: the cpu does, here this time, and on the worker from now on
|
||||
if (post_processor != nullptr)
|
||||
{
|
||||
WriteLog("Script screens: the post-processing of the images goes to the cpu");
|
||||
}
|
||||
post_processor = nullptr;
|
||||
post_on_gpu = false;
|
||||
if (false == finish_on_cpu(Screen))
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
Screen.texture->update_from_memory(Screen.image_width, Screen.image_height, Screen.image.data());
|
||||
}
|
||||
Screen.uploaded_texture = Screen.texture->get_id();
|
||||
Screen.image_pending = false;
|
||||
}
|
||||
@@ -481,7 +556,7 @@ void script_screen_queue::update()
|
||||
{
|
||||
if (auto const live{screen.lock()}; live != nullptr)
|
||||
{
|
||||
implementation::upload(*live);
|
||||
m_impl->upload(*live);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -496,6 +571,12 @@ void script_screen_queue::update()
|
||||
}
|
||||
}
|
||||
|
||||
void script_screen_queue::set_post_processor(script_screens::post_processor *Processor)
|
||||
{
|
||||
m_impl->post_processor = Processor;
|
||||
m_impl->post_on_gpu = Processor != nullptr;
|
||||
}
|
||||
|
||||
#ifndef WITH_LUA
|
||||
std::unique_ptr<script_screens::runtime> script_screens::make_lua_runtime()
|
||||
{
|
||||
|
||||
@@ -10,16 +10,37 @@ http://mozilla.org/MPL/2.0/.
|
||||
#pragma once
|
||||
|
||||
#include "scripting/scriptscreenrules.h"
|
||||
#include "scripting/scriptscreenkernels.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
struct ITexture;
|
||||
struct dictionary_source;
|
||||
|
||||
namespace script_screens
|
||||
{
|
||||
|
||||
// runs the post-processing of a screen's image (image:led_post, image:resize) on the gpu, into the screen's texture.
|
||||
// given by the renderer that can (gfx_renderer::Screen_post_processor); used on the main thread
|
||||
class post_processor
|
||||
{
|
||||
public:
|
||||
post_processor() = default;
|
||||
virtual ~post_processor() = default;
|
||||
post_processor(post_processor const &) = delete;
|
||||
post_processor &operator=(post_processor const &) = delete;
|
||||
// Pixels: the image drawn, Width x Height, RGBA from the top row down. the texture gets the result bottom row first,
|
||||
// as update_from_memory takes it, with its mipmaps. false when it can't; the screen then does the steps on the cpu
|
||||
virtual bool apply(ITexture &Target, std::span<std::uint8_t const> Pixels, int Width, int Height, std::span<post_step const> Steps) = 0;
|
||||
};
|
||||
|
||||
} // namespace script_screens
|
||||
|
||||
// screens drawn by scripts, the counterpart of the python screens (scripting/PyInt.h): Lua scripts run by LuaJIT
|
||||
// (luascreen:, luadestinationsign:) and WebAssembly modules run by WAMR (wasmscreen:, wasmdestinationsign:). a script
|
||||
// gets the state a python screen gets, returns an RGBA image and, in the cab, may send commands. the interface is
|
||||
@@ -91,6 +112,9 @@ class script_screen_queue
|
||||
// main thread, once per rendered frame: uploads the finished images, runs the commands the screens sent and
|
||||
// counts the frame
|
||||
void update();
|
||||
// main thread: the post-processing of the images goes to the gpu through the processor; with none (the default) it
|
||||
// runs on the worker, on the cpu
|
||||
void set_post_processor(script_screens::post_processor *Processor);
|
||||
|
||||
private:
|
||||
struct implementation;
|
||||
|
||||
Reference in New Issue
Block a user