Reworked PackCompiler

PackCompiler now processes everything in memory to reduce file writes. Will also be able to expand error handling later

KomplexHubPlugin/packcompiler.h

@@ -36,6 +36,8 @@
 #include <qobject.h>
 #include <qtmetamacros.h>
 #include <QPainter>
+#include <QtShaderTools>
+#include <QtShaderTools/rhi/qshaderbaker.h>
 #include <QtConcurrent/qtconcurrentrun.h>
 #include <qcontainerfwd.h>
 #include <qmargins.h>
@@ -43,21 +45,42 @@
 #include <qtmetamacros.h>
 #include <qurl.h>
 #include "common/komplex_global.h"
+
+#if __has_include("vulkan/vulkan_core.h")
+#define HAS_VULKAN
+#include <vulkan/vulkan_core.h>
+#endif
+
+#if __has_include("GL/gl.h")
+#define HAS_OPENGL
+#include <GL/gl.h>
+#endif
+
+struct KOMPLEX_EXPORT BuildContext
+{
+    QMap<QByteArray, QByteArray> commonData;
+    QPromise<QUrl> promise;
+};
+
 /**
  * @brief The PackCompiler class
  * [1] Extract Pack (GZip)
  * [2] Verify directory is a pack and has shaders
- * [3] Load shader code
+ * [3] Load Common shader code
- * [4] Append Common and Global files, if they exist
+ * [4] Walk the shader directory
- * [5] Add uniform and replace variable use
+ * [5] Load shader code
- * [6] Run C Preprocessor (cpp -P) to process macros
+ * [6] Append type specific common code and global code
- * [7] Append version macro
+ * [7] Replace uniform variable use
- * [8] Compile with Qt Shader Baker
+ * [8] Add uniform header
+ * [9] Add uniform footer
+ * [10] Run preprocessor to process macros
+ * [11] Append version macro
+ * [12] Compile with QShaderBaker
+ * [13] Save
  */
-class KOMPLEX_EXPORT PackCompiler : public QObject
+class PackCompiler : public QObject
 {
     Q_OBJECT
-    QML_ELEMENT
 public:
 
     enum State
@@ -70,7 +93,6 @@ public:
     Q_ENUM(State)
 
     explicit PackCompiler(QObject *parent = nullptr);
-    ~PackCompiler() override;
 
     auto compilerOutput() const -> const QString & { return m_compilerOutput; }
     auto progress() -> qreal { return m_progress; }
@@ -78,65 +100,11 @@ public:
     auto totalSteps() -> qint64 { return m_totalSteps; }
     auto errorTitle() const -> const QString & { return m_errorTitle; }
     auto errorMessage() const -> const QString & { return m_errorMessage; }
-    auto reset() -> void;
-
-    auto process(const QUrl &uri) -> QFuture<QUrl>;
-
     auto status() const -> const QString & { return m_status; }
     auto state() const -> State { return m_state; }
+    auto reset() -> void;
 
-protected:
+    auto build(const QUrl &uri) -> QUrl;
-    /**
-     * @brief prepareShaders
-     * Scans the pack's shader directory and sends each file to
-     * prepareFile()
-     * @param uri
-     * Directory to prepare
-     */
-    auto prepareShaders(const QUrl &uri) noexcept(false) -> void;
-
-    /**
-     * @brief prepareFile
-     * Prepares the individual file by appending the header, footer,
-     * common files and replacing standard variable names
-     * @param uri
-     */
-    auto prepareFile(const QUrl &uri) noexcept(false) -> void;
-
-    /**
-     * @brief validateDirectory
-     *
-     * @param uri
-     * @return true
-     * @return false
-     */
-    auto validateDirectory(const QUrl &uri) -> bool;
-
-    auto createDirectory(const QUrl &uri) -> void;
-
-    auto validateUri(const QUrl &uri) noexcept(false) -> void;
-    auto extract(const QUrl &sourceUri) noexcept(false) -> QUrl;
-    auto copyFile(const QUrl &sourceUri, const QUrl &destinationUri) noexcept(false) -> void;
-    auto compile(const QUrl &uri) noexcept(false) -> void;
-    auto preprocess(const QUrl &uri) noexcept(false) -> void;
-    auto appendVersion(const QUrl &uri) noexcept(false) -> void;
-    auto compileShader(const QUrl &uri) noexcept(false) -> void;
-    auto loadCommonFragmentData(const QUrl &uri) -> QByteArray;
-    auto loadCommonVertexData(const QUrl &uri) -> QByteArray;
-    auto loadGlobalData(const QUrl &uri) -> QByteArray;
-
-    auto setError(const QString &title, const QString &message, const QUrl &uri = {}) -> void;
-    auto setCompilerOutput(const QString &compilerOutput) -> void;
-    auto setState(State state) -> void;
-    auto setProgress(qreal progress) -> void;
-    auto setTotalSteps(qint64 steps) -> void;
-    auto setCurrentStep(qint64 step) -> void;
-
-    auto setStatus(const QString &status) -> void;
-
-    auto incrementCompileStep() -> void;
-
-    auto removeDirectory(const QUrl &uri) -> bool;
 
 signals:
     auto compilerOutputChanged() -> void;
@@ -147,38 +115,213 @@ signals:
     auto progressChanged() -> void;
     auto currentStepChanged() -> void;
     auto totalStepsChanged() -> void;
-
     auto compileComplete(const QUrl &uri) -> void;
-
     auto statusChanged() -> void;
     auto packReady(const QString &) -> void;
     auto errorStepChanged() -> void;
     auto errorRatingChanged() -> void;
 
 private:
+    //[1]
+    /**
+     * @brief extract
+     * Extracts the packfile and deletes the source
+     * @param uri of the tar.gz pack
+     * @return uri of the extracted directory
+     */
+    auto extract(const QUrl &uri) noexcept(false) -> const QUrl;
+
+    //[2]
+    /**
+     * @brief validatePack
+     * Validates that the directory is a valid pack and has shaders to compile
+     * @param uri of the directory provided by extract
+     */
+    auto validatePack(const QUrl &uri) const noexcept(false) -> void;
+
+    //[3]
+    /**
+     * @brief loadCommonData
+     * Loads the common and global shader code from the shader directory of the uri
+     * @param uri
+     */
+    auto loadCommonData(const QUrl &uri) -> QMap<QByteArray,QByteArray>;
+
+    //[4-5]
+    /**
+     * @brief processShaders
+     * Walks the shader directory, loads the code and calls 6-13 for each
+     * @param uri
+     * @param context
+     */
+    auto processShaders(const QUrl &uri, BuildContext *context) -> void;
+
+    //[6]
+    /**
+     * @brief appendCommonData
+     * Appends the type specific common code to the shader data
+     * @param data
+     * Pointer to shader data
+     * @param suffix
+     * Shader file suffix
+     * @param context
+     * Pointer to build context object
+     */
+    auto appendCommonData(QByteArray *data, const QByteArray &suffix, BuildContext *context) noexcept(true) -> void;
+
+    //[7]
+    /**
+     * @brief replaceUniformVariables
+     * Replaces standard variables such as iFrame with their uniform buffer name
+     * @param data
+     * Pointer to the shader data
+     */
+    auto replaceUniformVariables(QByteArray *data) noexcept(true) -> void;
+
+    //[8]
+    /**
+     * @brief appendUniformHeader
+     * Adds header and uniform buffer to the shader data
+     * @param data
+     * Pointer to the shader data
+     */
+    auto appendUniformHeader(QByteArray *data) noexcept(true) -> void;
+
+    //[9]
+    /**
+     * @brief appendUniformFooter
+     * Adds the uniform footer to the shader data
+     * @param data
+     * Pointer to the shader data
+     */
+    auto appendUniformFooter(QByteArray *data) noexcept(true) -> void;
+
+    //[10]
+    /**
+     * @brief preprocess
+     * I have had issues with defines in code so passing it to a c/c++
+     * preprocessor is required. I am now using clang for this as clang
+     * is more modern and can use stdin
+     * @param data
+     */
+    auto preprocess(QByteArray *data) noexcept(false) -> void;
+
+    //[11]
+    /**
+     * @brief appendVersion
+     * The version header confuses the preprocessor, so add it at the end
+     * @param data
+     */
+    auto appendVersion(QByteArray *data) noexcept(true) -> void;
+
+    //[12]
+    /**
+     * @brief compile
+     * Compiles the shader data in memory and returns the shader object.
+     * @param data
+     * Shader data
+     * @param filename
+     * Filename of the shader
+     * @param stage
+     * Type of shader, based on suffix of original file
+     * @return
+     * Compiled shader object
+     */
+    auto compile(const QByteArray &data, const QString &filename, const QShader::Stage stage) noexcept(false) -> QShader;
+
+    //[13]
+    auto save(const QUrl &uri, const QShader &shader) noexcept(false) -> void;
+
+    //helpers
+    auto validateUri(const QUrl &uri) const noexcept(false) -> void;
+    auto run(const QString &command, const QStringList &arguments, const QByteArray &data = {}) noexcept(false) -> QByteArray;
+    auto readFile(const QUrl &uri) noexcept(false) -> QByteArray;
+    auto writeFile(const QUrl &uri, const QByteArray &data) noexcept(false) -> void;
+    auto copyFile(const QUrl &sourceUri, const QUrl &destinationUri) noexcept(false) -> void;
+    auto createDirectory(const QUrl &uri) noexcept(false) -> void;
+    auto removeDirectory(const QUrl &uri) noexcept(false) -> void;
+    auto getStageFromSuffix(const QString &suffix) const -> QShader::Stage;
+
+    auto setError(const QString &title, const QString &message, const QUrl &uri = {}) -> void;
+    auto setCompilerOutput(const QString &compilerOutput) -> void;
+    auto setState(State state) -> void;
+    auto setProgress(qreal progress) -> void;
+    auto setTotalSteps(qint64 steps) -> void;
+    auto setCurrentStep(qint64 step) -> void;
+    auto setStatus(const QString &status) -> void;
+    auto incrementCompileStep() -> void;
+
+#ifdef HAS_VULKAN
+    auto vulkanVersion() const -> qint32;
+#endif
+
+#ifdef HAS_OPENGL
+    auto openGlVersion() const -> qint32;
+#endif
+
+    //members
     static inline const QStringList m_updateVariables
+    {
+        QStringLiteral("iTime"),
+        QStringLiteral("iTimeDelta"),
+        QStringLiteral("iFrameRate"),
+        QStringLiteral("iSampleRate"),
+        QStringLiteral("iFrame"),
+        QStringLiteral("iDate"),
+        QStringLiteral("iMouse"),
+        QStringLiteral("iResolution"),
+        QStringLiteral("iColorTheme"),
+        QStringLiteral("iChannelTime"),
+        QStringLiteral("iChannelResolution")
+    };
+
+    static inline const QStringList m_shaderNameFilters
+    {
+        QStringLiteral("*.frag"),
+        QStringLiteral("*.vert"),
+        QStringLiteral("*.tese"),
+        QStringLiteral("*.tesc"),
+        QStringLiteral("*.geom"),
+        QStringLiteral("*.comp"),
+        QStringLiteral("*.glsl")
+    };
+
+    static inline const QMap<QString, QShader::Stage> m_shaderStages
+    {
         {
-            QStringLiteral("iTime"),
+            QStringLiteral("frag"),
-            QStringLiteral("iTimeDelta"),
+            QShader::FragmentStage
-            QStringLiteral("iFrameRate"),
+        },
-            QStringLiteral("iSampleRate"),
+        {
-            QStringLiteral("iFrame"),
+            QStringLiteral("vert"),
-            QStringLiteral("iDate"),
+            QShader::VertexStage
-            QStringLiteral("iMouse"),
+        },
-            QStringLiteral("iResolution"),
+        {
-            QStringLiteral("iColorTheme"),
+            QStringLiteral("tese"),
-            QStringLiteral("iChannelTime"),
+            QShader::TessellationEvaluationStage
-            QStringLiteral("iChannelResolution")
+        },
-        };
+        {
+            QStringLiteral("tesc"),
+            QShader::TessellationControlStage
+        },
+        {
+            QStringLiteral("geom"),
+            QShader::GeometryStage
+        },
+        {
+            QStringLiteral("comp"),
+            QShader::ComputeStage
+        }
+    };
 
     static inline const QByteArray m_version
-        {
+    {
-            R"(#version 450)"
+        R"(#version 450)"
-        };
+    };
 
-    static inline const QString m_header
+    static inline const QByteArray m_header
-        {
+    {
-            R"(layout(location = 0) in vec2 qt_TexCoord0;
+        R"(layout(location = 0) in vec2 qt_TexCoord0;
 layout(location = 0) out vec4 fragColor;
 
 layout(std140, binding = 0) uniform buf {
@@ -203,16 +346,20 @@ layout(binding = 3) uniform sampler2D iChannel2;
 layout(binding = 4) uniform sampler2D iChannel3;
 
 vec2 fragCoord = vec2(qt_TexCoord0.x, 1.0 - qt_TexCoord0.y) * ubuf.iResolution.xy;)"
-        };
+    };
 
-    static inline const QString m_footer
+    static inline const QByteArray m_footer
-        {
+    {
-            R"(void main() {
+        R"(
+void main() {
     vec4 color = vec4(0.0);
     mainImage(color, fragCoord);
     fragColor = color;
 })"
-        };
+    };
+
+    QMap<QByteArray, QByteArray> m_commonData;
+    QMap<QByteArray, QShaderBaker> m_shaders;
 
     QString m_compilerOutput;
     QString m_errorMessage;
@@ -227,34 +374,10 @@ vec2 fragCoord = vec2(qt_TexCoord0.x, 1.0 - qt_TexCoord0.y) * ubuf.iResolution.x
 
     QMutex m_downloadMutex;
 
+    QFuture<QUrl> m_buildFuture;
+
     static inline QList<QRegularExpression> m_variableExpressions;
 
-    static inline const QRegularExpression m_commonFragmentExpression = QRegularExpression
-    (
-        QString("^common\\.frag$"),
-        QRegularExpression::CaseInsensitiveOption
-    );
-
-    static inline const QRegularExpression m_commonVertexExpression = QRegularExpression
-    (
-        QString("^common\\.vert$"),
-        QRegularExpression::CaseInsensitiveOption
-    );
-
-    static inline const QRegularExpression m_fragmentExpression = QRegularExpression
-    (
-        QString("^.{1,}\\.frag$"),
-        QRegularExpression::CaseInsensitiveOption |
-            QRegularExpression::DotMatchesEverythingOption
-    );
-
-    static inline const QRegularExpression m_vertexExpression = QRegularExpression
-    (
-        QString("^.{1,}\\.vert$"),
-        QRegularExpression::CaseInsensitiveOption |
-            QRegularExpression::DotMatchesEverythingOption
-    );
-
     Q_PROPERTY(QString compilerOutput READ compilerOutput WRITE setCompilerOutput NOTIFY compilerOutputChanged FINAL)
     Q_PROPERTY(QString errorTitle READ errorTitle NOTIFY errorTitleChanged FINAL)
     Q_PROPERTY(QString errorMessage READ errorMessage NOTIFY errorMessageChanged FINAL)