CI shader validation

.gitignore

@@ -74,3 +74,5 @@ builds/
 
 CMakeLists.txt.user
 version_info.h
+
+ci_shadervalidator/validateshaders

azure-pipelines.yml

@@ -1,4 +1,25 @@
 jobs:
+  - job: validateshaders
+    pool:
+      vmImage: 'ubuntu-20.04'
+    displayName: 'Validate shaders'
+    steps:
+    - script: |
+        sudo apt-get update -y
+        sudo apt-get install -y glslang-tools
+        cd ci_shadervalidator
+        ./build.sh
+      displayName: 'Install dependencies'
+    - script: |
+        cd ci_shadervalidator
+        ./validateshaders
+      displayName: 'Validate shaders'
+    - task: PublishBuildArtifacts@1
+      inputs:
+        pathtoPublish: 'shaders'
+        artifactName: shaders
+      displayName: 'Publish shaders'
+
   - job: ubuntu1604
     pool:
       vmImage: 'Ubuntu-16.04'
@@ -12,7 +33,7 @@ jobs:
         sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 100
         sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 100
         cd ref
-        git clone "https://github.com/chriskohlhoff/asio" --depth 1 --branch asio-1-16-1 -q
+        git clone "https://github.com/chriskohlhoff/asio" --depth 1 --branch asio-1-12-2 -q
       displayName: 'Install dependencies'
     - script: |
         mkdir build
@@ -123,11 +144,6 @@ jobs:
       vmImage: 'vs2017-win2016'
     displayName: 'Windows VS2017 Debug'
     steps:
-    - task: PublishBuildArtifacts@1
-      inputs:
-        pathtoPublish: 'shaders'
-        artifactName: shaders
-      displayName: 'Publish shaders'
     - script: |
         cd ref
         git clone "https://github.com/chriskohlhoff/asio" --depth 1 --branch asio-1-12-2 -q

ci_shadervalidator/Globals.h

@@ -0,0 +1,14 @@
+#pragma once
+struct _global
+{
+        bool gfx_shadowmap_enabled;
+        bool gfx_envmap_enabled;
+        bool gfx_postfx_motionblur_enabled;
+        bool gfx_skippipeline;
+        bool gfx_extraeffects;
+        bool gfx_shadergamma;
+        bool gfx_usegles;
+};
+
+extern _global Global;
+

ci_shadervalidator/Logs.h

@@ -0,0 +1,8 @@
+#pragma once
+enum class logtype : unsigned int {
+	shader
+};
+
+inline void ErrorLog(const std::string &, logtype const)
+{
+}

ci_shadervalidator/build.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+g++ -std=c++17 -O2 -DSHADERVALIDATOR_STANDALONE ../ref/glad/src/glad.c ../gl/glsl_common.cpp ../gl/shader.cpp main.cpp -I../ref/glad/include -I../ref/glm -I. -o validateshaders

ci_shadervalidator/main.cpp

@@ -0,0 +1,112 @@
+#include "stdafx.h"
+#include "../gl/shader.h"
+#include "../gl/glsl_common.h"
+#include "Globals.h"
+#include <iostream>
+#include <fstream>
+#include <cstdlib>
+#include <filesystem>
+
+_global Global;
+
+bool check_shader(std::string filename)
+{
+	gl::shader shader_processor;
+
+	gl::glsl_common_setup();
+	std::pair<GLuint, std::string> source = shader_processor.process_source(filename, "../shaders/");
+
+	std::string stage;
+	if (source.first == GL_VERTEX_SHADER)
+		stage = "vert";
+	else if (source.first == GL_FRAGMENT_SHADER)
+		stage = "frag";
+	else if (source.first == GL_GEOMETRY_SHADER)
+		stage = "geom";
+	else
+		return false;
+
+	std::ofstream file("/tmp/shader");
+	file << source.second;
+	file.close();
+
+	int ret = system(("glslangValidator -S " + stage + " /tmp/shader").c_str());
+
+	return ret == 0;
+}
+
+bool check_shader_allopts(std::string filename)
+{
+	bool suc = true;
+
+	std::cout << "features on..." << std::flush;
+	Global.gfx_shadowmap_enabled = true;	
+	Global.gfx_envmap_enabled = true;
+	Global.gfx_postfx_motionblur_enabled = true;
+	Global.gfx_skippipeline = false;
+	Global.gfx_extraeffects = true;
+	Global.gfx_shadergamma = true;
+	if (!check_shader(filename))
+		suc = false;
+
+	std::cout << "done, features off..." << std::flush;
+	Global.gfx_shadowmap_enabled = false;
+	Global.gfx_envmap_enabled = false;
+	Global.gfx_postfx_motionblur_enabled = false;
+	Global.gfx_skippipeline = true;
+	Global.gfx_extraeffects = false;
+	Global.gfx_shadergamma = false;
+	if (!check_shader(filename))
+		suc = false;
+
+	std::cout << "done" << std::flush;
+
+	return suc;
+}
+
+
+bool check_shader_confs(std::string filename)
+{
+	bool suc = true;
+
+	std::cout << "desktop GL: " << std::flush;
+	Global.gfx_usegles = false;
+	if (!check_shader_allopts(filename))
+		suc = false;
+
+	std::cout << "; GLES: " << std::flush;
+	GLAD_GL_ES_VERSION_3_1 = 1;
+	Global.gfx_usegles = true;
+	if (!check_shader_allopts(filename))
+		suc = false;
+
+	std::cout << std::endl;
+
+	return suc;
+}
+
+int main()
+{
+	bool suc = true;
+	for (auto &f : std::filesystem::recursive_directory_iterator("../shaders/")) {
+		if (!f.is_regular_file())
+			continue;
+
+		std::string relative = std::filesystem::relative(f.path(), "../shaders/").string();
+		if (f.path().extension() != ".frag" && f.path().extension() != ".geom" && f.path().extension() != ".vert")
+			continue;
+
+		std::cout << "== validating shader " << relative << ": " << std::flush;
+		if (!check_shader_confs(relative))
+			suc = false;
+	}
+
+	if (suc)
+		std::cout << "==== all checks ok" << std::endl;
+	else {
+		std::cout << "==== checks failed" << std::endl;
+		return 1;
+	}
+
+	return 0;
+}

ci_shadervalidator/stdafx.h

@@ -0,0 +1,6 @@
+#pragma once
+#define GLM_ENABLE_EXPERIMENTAL
+#define GLM_FORCE_CTOR_INIT
+#include <glm/glm.hpp>
+#include <string>
+#include <stdexcept>

gl/glsl_common.cpp

@@ -11,7 +11,6 @@ void gl::glsl_common_setup()
     "#define MOTIONBLUR_ENABLED " + std::to_string((int)Global.gfx_postfx_motionblur_enabled) + "\n" +
     "#define POSTFX_ENABLED " + std::to_string((int)!Global.gfx_skippipeline) + "\n" +
     "#define EXTRAEFFECTS_ENABLED " + std::to_string((int)Global.gfx_extraeffects) + "\n" +
-    "#define USE_GLES " + std::to_string((int)Global.gfx_usegles) + "\n" +
     "#define MAX_LIGHTS " + std::to_string(MAX_LIGHTS) + "U\n" +
     "#define MAX_CASCADES " + std::to_string(MAX_CASCADES) + "U\n" +
     "#define MAX_PARAMS " + std::to_string(MAX_PARAMS) + "U\n" +

gl/shader.cpp

@@ -17,18 +17,19 @@ std::string gl::shader::read_file(const std::string &filename)
 {
     std::stringstream stream;
     std::ifstream f;
-    f.exceptions(std::ifstream::badbit);
 
-    f.open("shaders/" + filename);
+    f.open(filename);
     stream << f.rdbuf();
     f.close();
 
     std::string str = stream.str();
+    if (str.empty())
+        throw shader_exception("cannot read shader: " + filename);
 
     return str;
 }
 
-void gl::shader::expand_includes(std::string &str)
+void gl::shader::expand_includes(std::string &str, const std::string &basedir)
 {
     size_t start_pos = 0;
 
@@ -43,7 +44,7 @@ void gl::shader::expand_includes(std::string &str)
         std::string filename = str.substr(fp + 1, fe - fp - 1);
         std::string content;
         if (filename != "common")
-            content = read_file(filename);
+            content = read_file(basedir + filename);
         else
             content = glsl_common;
 
@@ -73,11 +74,47 @@ std::unordered_map<std::string, gl::shader::defaultparam_e> gl::shader::defaultp
     { "glossiness", defaultparam_e::glossiness }
 };
 
-void gl::shader::process_source(std::string &str)
+std::pair<GLuint, std::string> gl::shader::process_source(const std::string &filename, const std::string &basedir)
 {
-    expand_includes(str);
+    std::string str;
+
+    GLuint type;
+    if (strcend(filename, ".vert"))
+        type = GL_VERTEX_SHADER;
+    else if (strcend(filename, ".frag"))
+        type = GL_FRAGMENT_SHADER;
+    else if (strcend(filename, ".geom"))
+        type = GL_GEOMETRY_SHADER;
+    else
+        throw shader_exception("unknown shader " + filename);
+
+    if (!Global.gfx_usegles)
+    {
+        str += "#version 330 core\n";
+    }
+    else
+    {
+        if (GLAD_GL_ES_VERSION_3_1) {
+            str += "#version 310 es\n";
+            if (type == GL_GEOMETRY_SHADER)
+                str += "#extension GL_EXT_geometry_shader : require\n";
+        } else {
+            str += "#version 300 es\n";
+        }
+        str += "precision highp float;\n";
+        str += "precision highp int;\n";
+        str += "precision highp sampler2DShadow;\n";
+        str += "precision highp sampler2DArrayShadow;\n";
+    }
+    str += "vec4 FBOUT(vec4 x) { return " + (Global.gfx_shadergamma ? std::string("vec4(pow(x.rgb, vec3(1.0 / 2.2)), x.a)") : std::string("x")) + "; }\n";
+
+    str += read_file(basedir + filename);
+
+    expand_includes(str, basedir);
     parse_texture_entries(str);
     parse_param_entries(str);
+
+    return std::make_pair(type, str);
 }
 
 void gl::shader::parse_texture_entries(std::string &str)
@@ -204,46 +241,11 @@ gl::shader::shader(const std::string &filename)
 {
     name = filename;
 
-	GLuint type;
+    std::pair<GLuint, std::string> source = process_source(filename, "shaders/");
-	if (strcend(filename, ".vert"))
-		type = GL_VERTEX_SHADER;
-	else if (strcend(filename, ".frag"))
-		type = GL_FRAGMENT_SHADER;
-	else if (strcend(filename, ".geom"))
-		type = GL_GEOMETRY_SHADER;
-	else
-		throw shader_exception("unknown shader " + filename);
 
-    std::string str;
+    const GLchar *cstr = source.second.c_str();
-    if (!Global.gfx_usegles)
-    {
-        str += "#version 330 core\n";
-    }
-    else
-    {
-		if (GLAD_GL_ES_VERSION_3_1) {
-			str += "#version 310 es\n";
-			if (type == GL_GEOMETRY_SHADER)
-				str += "#extension GL_EXT_geometry_shader : require\n";
-		} else {
-			str += "#version 300 es\n";
-		}
-        str += "precision highp float;\n";
-        str += "precision highp int;\n";
-        str += "precision highp sampler2DShadow;\n";
-        str += "precision highp sampler2DArrayShadow;\n";
-    }
-    str += "vec4 FBOUT(vec4 x) { return " + (Global.gfx_shadergamma ? std::string("vec4(pow(x.rgb, vec3(1.0 / 2.2)), x.a)") : std::string("x")) + "; }\n";
 
-    str += read_file(filename);
+    **this = glCreateShader(source.first);
-    process_source(str);
-
-    const GLchar *cstr = str.c_str();
-
-    if (!cstr[0])
-        throw shader_exception("cannot read shader: " + filename);
-
-    **this = glCreateShader(type);
     glShaderSource(*this, 1, &cstr, 0);
     glCompileShader(*this);
 
@@ -253,7 +255,7 @@ gl::shader::shader(const std::string &filename)
     {
         GLchar info[512];
         glGetShaderInfoLog(*this, 512, 0, info);
-        std::cerr << std::string(info) << std::endl;
+        log_error(std::string(info));
 
         throw shader_exception("failed to compile " + filename + ": " + std::string(info));
     }
@@ -261,7 +263,9 @@ gl::shader::shader(const std::string &filename)
 
 gl::shader::~shader()
 {
+#ifndef SHADERVALIDATOR_STANDALONE
     glDeleteShader(*this);
+#endif
 }
 
 void gl::program::init()

gl/shader.h

@@ -3,6 +3,7 @@
 #include <string>
 #include <vector>
 #include <functional>
+#include <unordered_map>
 
 #include "object.h"
 #include "bindable.h"
@@ -17,6 +18,9 @@ namespace gl
     class shader : public object
     {
     public:
+#ifdef SHADERVALIDATOR_STANDALONE
+        shader() = default;
+#endif
         shader(const std::string &filename);
         ~shader();
 
@@ -60,10 +64,12 @@ namespace gl
         std::unordered_map<std::string, param_entry> param_conf;
         std::string name;
 
+#ifndef SHADERVALIDATOR_STANDALONE
     private:
-        void process_source(std::string &str);
+#endif
+        std::pair<GLuint, std::string> process_source(const std::string &filename, const std::string &basedir);
 
-        void expand_includes(std::string &str);
+        void expand_includes(std::string &str, const std::string &basedir);
         void parse_texture_entries(std::string &str);
         void parse_param_entries(std::string &str);