18
0
mirror of https://github.com/MaSzyna-EU07/maszyna.git synced 2026-11-18 22:16:02 +01:00

Script screens: led_post and resize of the Lua screens on the gpu

This commit is contained in:
2026-11-10 16:22:35 +01:00
parent c4924f756d
commit e433b43fef
20 changed files with 971 additions and 34 deletions

View File

@@ -145,9 +145,63 @@ template <typename Type_> Type_ &check_object(lua_State *L, int const Index, cha
return *object;
}
// an image of a script: what was drawn on it, and the post-processing (image:led_post, image:resize) asked for and not
// run yet. the steps of the image render returns run in the queue, on the gpu when the renderer can; the steps of an
// image the script goes on to read or draw on (or with) run here, on the cpu, when it does
struct lua_image
{
canvas drawn;
std::vector<post_step> steps;
int width() const
{
auto result{drawn.width()};
for (auto const &step : steps)
{
result = width_after(step, result);
}
return result;
}
int height() const
{
auto result{drawn.height()};
for (auto const &step : steps)
{
result = height_after(step, result);
}
return result;
}
// the image with its post-processing done
canvas &done()
{
if (false == steps.empty())
{
post_image image{drawn.pixels(), drawn.width(), drawn.height()};
// the sizes were checked when the steps were asked for
apply_post(image, steps);
drawn = canvas{image.width, image.height, std::move(image.pixels)};
steps.clear();
}
return drawn;
}
// the image, a step further
lua_image then(post_step const &Step) const
{
lua_image result{drawn, steps};
result.steps.push_back(Step);
return result;
}
};
lua_image &check_lua_image(lua_State *L, int const Index)
{
return check_object<lua_image>(L, Index, image_metatable);
}
// the image, to read or draw on: its post-processing is done first
canvas &check_image(lua_State *L, int const Index)
{
return check_object<canvas>(L, Index, image_metatable);
return check_lua_image(L, Index).done();
}
int check_int(lua_State *L, int const Index)
@@ -226,7 +280,7 @@ int api_image(lua_State *L)
luaL_argcheck(L, valid_image_side(width), 1, "between 1 and 4096");
luaL_argcheck(L, valid_image_side(height), 2, "between 1 and 4096");
auto const color{lua_isnoneornil(L, 3) ? rgb{0, 0, 0} : check_color(L, 3)};
push_object(L, canvas{width, height, color}, image_metatable);
push_object(L, lua_image{canvas{width, height, color}, {}}, image_metatable);
return 1;
}
@@ -388,13 +442,13 @@ int api_include(lua_State *L)
int image_width(lua_State *L)
{
lua_pushinteger(L, check_image(L, 1).width());
lua_pushinteger(L, check_lua_image(L, 1).width());
return 1;
}
int image_height(lua_State *L)
{
lua_pushinteger(L, check_image(L, 1).height());
lua_pushinteger(L, check_lua_image(L, 1).height());
return 1;
}
@@ -498,36 +552,39 @@ int image_draw_image(lua_State *L)
return 0;
}
// image:led_post([bloom]): led_util's render_post, a new image (2w + 2) x (2h + 2)
// image:led_post([bloom]): led_util's render_post, a new image (2w + 2) x (2h + 2). done when the image is shown or used
int image_led_post(lua_State *L)
{
auto const &image{check_image(L, 1)};
auto const &image{check_lua_image(L, 1)};
auto const bloom{lua_isnoneornil(L, 2) || lua_toboolean(L, 2) != 0};
luaL_argcheck(L, valid_image_side(image.width() * 2 + 2) && valid_image_side(image.height() * 2 + 2), 1, "too large");
std::vector<std::uint8_t> pixels(image_size(image.width() * 2 + 2, image.height() * 2 + 2));
led_post(image.pixels(), image.width(), image.height(), pixels, bloom);
push_object(L, canvas{image.width() * 2 + 2, image.height() * 2 + 2, std::move(pixels)}, image_metatable);
post_step step;
step.type = post_step::kind::led;
step.bloom = bloom;
push_object(L, image.then(step), image_metatable);
return 1;
}
// image:resize(width, height): scaled as Pillow's resize(..., LANCZOS), a new image
// image:resize(width, height): scaled as Pillow's resize(..., LANCZOS), a new image. done when the image is shown or used
int image_resize(lua_State *L)
{
auto const &image{check_image(L, 1)};
auto const &image{check_lua_image(L, 1)};
auto const width{check_int(L, 2)};
auto const height{check_int(L, 3)};
luaL_argcheck(L, valid_image_side(width), 2, "between 1 and 4096");
luaL_argcheck(L, valid_image_side(height), 3, "between 1 and 4096");
std::vector<std::uint8_t> pixels(image_size(width, height));
resize_lanczos(image.pixels(), image.width(), image.height(), pixels, width, height);
push_object(L, canvas{width, height, std::move(pixels)}, image_metatable);
post_step step;
step.type = post_step::kind::resize;
step.width = width;
step.height = height;
push_object(L, image.then(step), image_metatable);
return 1;
}
int image_copy(lua_State *L)
{
auto const &image{check_image(L, 1)};
push_object(L, canvas{image}, image_metatable);
auto const &image{check_lua_image(L, 1)};
push_object(L, lua_image{image}, image_metatable);
return 1;
}
@@ -695,7 +752,7 @@ void lua_program::register_api()
lua_pushlightuserdata(L, this);
lua_rawset(L, LUA_REGISTRYINDEX);
make_metatable(L, image_metatable, image_methods, collect<canvas>);
make_metatable(L, image_metatable, image_methods, collect<lua_image>);
make_metatable(L, font_metatable, font_methods, collect<bitmap_font>);
make_metatable(L, sprite_metatable, sprite_methods, collect<glyph>);
@@ -806,11 +863,12 @@ bool lua_program::render(render_call &Call)
}
// nil: the screen didn't change, keep the image there is
auto succeeded{true};
if (auto const *image{test_object<canvas>(m_state, -1, image_metatable)}; image != nullptr)
if (auto const *image{test_object<lua_image>(m_state, -1, image_metatable)}; image != nullptr)
{
Call.pixels = image->pixels();
Call.width = image->width();
Call.height = image->height();
Call.pixels = image->drawn.pixels();
Call.width = image->drawn.width();
Call.height = image->drawn.height();
Call.post = image->steps;
}
else if (false == lua_isnil(m_state, -1))
{

View File

@@ -14,6 +14,7 @@ http://mozilla.org/MPL/2.0/.
#include <array>
#include <cmath>
#include <numbers>
#include <utility>
#include <vector>
namespace script_screens
@@ -239,4 +240,42 @@ bool resize_lanczos(std::span<std::uint8_t const> const Source, int const Width,
return true;
}
int width_after(post_step const &Step, int const Width)
{
return Step.type == post_step::kind::led ? Width * 2 + 2 : Step.width;
}
int height_after(post_step const &Step, int const Height)
{
return Step.type == post_step::kind::led ? Height * 2 + 2 : Step.height;
}
bool apply_post(post_image &Image, std::span<post_step const> const Steps)
{
if (Steps.empty())
{
return true;
}
auto result{Image};
for (auto const &step : Steps)
{
auto const width{width_after(step, result.width)};
auto const height{height_after(step, result.height)};
if (width <= 0 || height <= 0)
{
return false;
}
std::vector<std::uint8_t> pixels(image_size(width, height));
auto const done{step.type == post_step::kind::led ? led_post(result.pixels, result.width, result.height, pixels, step.bloom)
: resize_lanczos(result.pixels, result.width, result.height, pixels, width, height)};
if (false == done)
{
return false;
}
result = post_image{std::move(pixels), width, height};
}
Image = std::move(result);
return true;
}
} // namespace script_screens

View File

@@ -9,8 +9,10 @@ http://mozilla.org/MPL/2.0/.
#pragma once
#include <cstddef>
#include <cstdint>
#include <span>
#include <vector>
// image filters the script screens call through the host (led_post, resize_lanczos), run natively rather than
// in the interpreter. they give the very pixels of the Python screens: led_util.render_post() and Pillow's
@@ -33,4 +35,32 @@ bool led_post(std::span<std::uint8_t const> Source, int Width, int Height, std::
// for sizes that don't fit the spans
bool resize_lanczos(std::span<std::uint8_t const> Source, int Width, int Height, std::span<std::uint8_t> Target, int TargetWidth, int TargetHeight);
// one step of the post-processing a script asks for on its image (image:led_post, image:resize). the steps of the image
// render returns run on the gpu when the renderer can (script_screens::post_processor), otherwise on the cpu
struct post_step
{
enum class kind
{
led, // led_post
resize // resize_lanczos
};
kind type{kind::led};
bool bloom{true}; // led: with the glow
int width{0}; // resize: the size it gives
int height{0};
};
struct post_image
{
std::vector<std::uint8_t> pixels; // RGBA, from the top row down
int width{0};
int height{0};
};
// the size of an image Width wide, or Height high, after the step
int width_after(post_step const &Step, int Width);
int height_after(post_step const &Step, int Height);
// runs the steps on the cpu, one after another. false for sizes the kernels refuse; the image is then as it was
bool apply_post(post_image &Image, std::span<post_step const> Steps);
} // namespace script_screens

View File

@@ -47,6 +47,8 @@ struct render_call
std::vector<std::uint8_t> pixels;
int width{0};
int height{0};
// the post-processing still to run on the image (image:led_post, image:resize of the Lua screens)
std::vector<post_step> post;
};
// a running script: a Lua state, a WebAssembly instance. made, called and destroyed on the worker thread, or by the

View File

@@ -31,6 +31,7 @@ http://mozilla.org/MPL/2.0/.
#include <iterator>
#include <limits>
#include <mutex>
#include <span>
#include <stop_token>
#include <string_view>
#include <thread>
@@ -129,6 +130,32 @@ std::string_view language_name(script_language const Language)
return Language == script_language::lua ? "Lua" : "WASM";
}
// the rows of the image the other way round. the script draws from the top row down; the texture takes the bottom row
// first, as the python screens give it (abstractscreenrenderer flips the image before handing it over)
std::vector<std::uint8_t> flipped(std::span<std::uint8_t const> const Pixels, int const Width, int const Height)
{
auto const row{static_cast<std::size_t>(Width) * 4};
auto const height{static_cast<std::size_t>(Height)};
std::vector<std::uint8_t> result(Pixels.size());
for (std::size_t y = 0; y < height; ++y)
{
std::memcpy(result.data() + (height - 1 - y) * row, Pixels.data() + y * row, row);
}
return result;
}
// the post-processing of the image done on the cpu. false when the kernels refused a size
bool post_on_cpu(std::vector<std::uint8_t> &Pixels, int &Width, int &Height, std::vector<script_screens::post_step> &Steps)
{
script_screens::post_image image{std::move(Pixels), Width, Height};
auto const done{script_screens::apply_post(image, Steps)};
Pixels = std::move(image.pixels);
Width = image.width;
Height = image.height;
Steps.clear();
return done;
}
} // namespace
class script_screen
@@ -159,11 +186,13 @@ class script_screen
// the worker's, and the queue's once the worker stopped
std::unique_ptr<script_screens::program> program;
// the image, handed from the worker to the main thread, bottom row first as the texture takes it
// the image, handed from the worker to the main thread: bottom row first as the texture takes it, or, while it has
// post-processing to run on the gpu, from the top row down as the script drew it
std::mutex image_mutex;
std::vector<std::uint8_t> image;
int image_width{0};
int image_height{0};
std::vector<script_screens::post_step> post;
bool image_pending{false}; // drawn and not yet uploaded
std::size_t uploaded_texture{no_texture}; // the gl texture the image went to, to upload it again when it's made anew
@@ -188,7 +217,9 @@ struct script_screen_queue::implementation
return Language == script_language::lua ? lua.get() : wasm.get();
}
static void present(script_screen &Screen, script_screens::render_call &Call);
static void upload(script_screen &Screen);
void upload(script_screen &Screen);
// the post-processing of the image the gpu couldn't do, done on the cpu. false when it failed too
static bool finish_on_cpu(script_screen &Screen);
static void execute(script_screens::pending_command const &Command);
std::unique_ptr<script_screens::runtime> lua;
@@ -196,6 +227,11 @@ struct script_screen_queue::implementation
bool initialized{false};
std::atomic<std::uint64_t> frame{0};
// the main thread's: the gpu post-processing, none when the renderer has none, or it failed
script_screens::post_processor *post_processor{nullptr};
// read by the worker: the post-processing is left to the main thread, for the gpu
std::atomic<bool> post_on_gpu{false};
std::mutex screens_mutex;
std::unordered_map<std::string, std::weak_ptr<script_screen>> screens;
@@ -273,6 +309,12 @@ void script_screen_queue::implementation::process(task const &Task)
screen.failed = true;
return;
}
if (false == call.post.empty() && false == post_on_gpu && false == post_on_cpu(call.pixels, call.width, call.height, call.post))
{
ErrorLog(std::format("{} screen \"{}\": the post-processing of the image failed", language_name(screen.language), screen.script));
screen.failed = true;
return;
}
present(screen, call);
}
if (false == call.commands.empty())
@@ -282,23 +324,37 @@ void script_screen_queue::implementation::process(task const &Task)
}
}
// keeps the image the screen drew. the script draws from the top row down; the texture takes the bottom row first, as
// the python screens give it (abstractscreenrenderer flips the image before handing it over)
// keeps the image the screen drew, turned bottom row first for the texture; with post-processing left for the gpu, as
// it was drawn
void script_screen_queue::implementation::present(script_screen &Screen, script_screens::render_call &Call)
{
auto const row{static_cast<std::size_t>(Call.width) * 4};
auto const height{static_cast<std::size_t>(Call.height)};
std::scoped_lock lock{Screen.image_mutex};
Screen.image.resize(Call.pixels.size());
for (std::size_t y = 0; y < height; ++y)
if (Call.post.empty())
{
std::memcpy(Screen.image.data() + (height - 1 - y) * row, Call.pixels.data() + y * row, row);
Screen.image = flipped(Call.pixels, Call.width, Call.height);
}
else
{
Screen.image = std::move(Call.pixels);
}
Screen.post = std::move(Call.post);
Screen.image_width = Call.width;
Screen.image_height = Call.height;
Screen.image_pending = true;
}
bool script_screen_queue::implementation::finish_on_cpu(script_screen &Screen)
{
if (false == post_on_cpu(Screen.image, Screen.image_width, Screen.image_height, Screen.post))
{
ErrorLog(std::format("{} screen \"{}\": the post-processing of the image failed", language_name(Screen.language), Screen.script));
Screen.image.clear();
return false;
}
Screen.image = flipped(Screen.image, Screen.image_width, Screen.image_height);
return true;
}
void script_screen_queue::implementation::upload(script_screen &Screen)
{
if (Screen.texture == nullptr)
@@ -315,7 +371,26 @@ void script_screen_queue::implementation::upload(script_screen &Screen)
{
return;
}
Screen.texture->update_from_memory(Screen.image_width, Screen.image_height, Screen.image.data());
auto const on_gpu{false == Screen.post.empty() && post_processor != nullptr &&
post_processor->apply(*Screen.texture, Screen.image, Screen.image_width, Screen.image_height, Screen.post)};
if (false == on_gpu)
{
if (false == Screen.post.empty())
{
// the gpu can't do it: the cpu does, here this time, and on the worker from now on
if (post_processor != nullptr)
{
WriteLog("Script screens: the post-processing of the images goes to the cpu");
}
post_processor = nullptr;
post_on_gpu = false;
if (false == finish_on_cpu(Screen))
{
return;
}
}
Screen.texture->update_from_memory(Screen.image_width, Screen.image_height, Screen.image.data());
}
Screen.uploaded_texture = Screen.texture->get_id();
Screen.image_pending = false;
}
@@ -481,7 +556,7 @@ void script_screen_queue::update()
{
if (auto const live{screen.lock()}; live != nullptr)
{
implementation::upload(*live);
m_impl->upload(*live);
}
}
}
@@ -496,6 +571,12 @@ void script_screen_queue::update()
}
}
void script_screen_queue::set_post_processor(script_screens::post_processor *Processor)
{
m_impl->post_processor = Processor;
m_impl->post_on_gpu = Processor != nullptr;
}
#ifndef WITH_LUA
std::unique_ptr<script_screens::runtime> script_screens::make_lua_runtime()
{

View File

@@ -10,16 +10,37 @@ http://mozilla.org/MPL/2.0/.
#pragma once
#include "scripting/scriptscreenrules.h"
#include "scripting/scriptscreenkernels.h"
#include <cstdint>
#include <memory>
#include <optional>
#include <span>
#include <string>
#include <string_view>
struct ITexture;
struct dictionary_source;
namespace script_screens
{
// runs the post-processing of a screen's image (image:led_post, image:resize) on the gpu, into the screen's texture.
// given by the renderer that can (gfx_renderer::Screen_post_processor); used on the main thread
class post_processor
{
public:
post_processor() = default;
virtual ~post_processor() = default;
post_processor(post_processor const &) = delete;
post_processor &operator=(post_processor const &) = delete;
// Pixels: the image drawn, Width x Height, RGBA from the top row down. the texture gets the result bottom row first,
// as update_from_memory takes it, with its mipmaps. false when it can't; the screen then does the steps on the cpu
virtual bool apply(ITexture &Target, std::span<std::uint8_t const> Pixels, int Width, int Height, std::span<post_step const> Steps) = 0;
};
} // namespace script_screens
// screens drawn by scripts, the counterpart of the python screens (scripting/PyInt.h): Lua scripts run by LuaJIT
// (luascreen:, luadestinationsign:) and WebAssembly modules run by WAMR (wasmscreen:, wasmdestinationsign:). a script
// gets the state a python screen gets, returns an RGBA image and, in the cab, may send commands. the interface is
@@ -91,6 +112,9 @@ class script_screen_queue
// main thread, once per rendered frame: uploads the finished images, runs the commands the screens sent and
// counts the frame
void update();
// main thread: the post-processing of the images goes to the gpu through the processor; with none (the default) it
// runs on the worker, on the cpu
void set_post_processor(script_screens::post_processor *Processor);
private:
struct implementation;