Added PackCompiler

KomplexHubContent/MainScreen.qml

@@ -87,7 +87,8 @@ Rectangle {
                     icon.width: 32
                     icon.url: "qrc:/images/icons/icons8-famous.svg"
 
-                    onTriggered: () => {
+                    onTriggered: () =>
+                    {
                         pageLoader.page = "pages/HomePage.qml"
 
                         if(currentMenuButton !== null)
@@ -98,10 +99,12 @@ Rectangle {
                         currentMenuButton = this
 
                         searchContainer.preferredHeight = 0
+                        searchContainer.reset()
                     }
                 }
 
-                MenuButton {
+                MenuButton
+                {
                     Layout.alignment: Qt.AlignTop
 
                     height: 64
@@ -119,10 +122,12 @@ Rectangle {
                         currentMenuButton = this
 
                         searchContainer.preferredHeight = 50
+                        searchContainer.reset()
                     }
                 }
 
-                MenuButton {
+                MenuButton
+                {
                     Layout.alignment: Qt.AlignTop
 
                     height: 64
@@ -140,10 +145,12 @@ Rectangle {
                         currentMenuButton = this
 
                         searchContainer.preferredHeight = 50
+                        searchContainer.reset()
                     }
                 }
 
-                MenuButton {
+                MenuButton
+                {
                     Layout.alignment: Qt.AlignTop
 
                     height: 64
@@ -161,10 +168,12 @@ Rectangle {
                         currentMenuButton = this
 
                         searchContainer.preferredHeight = 50
+                        searchContainer.reset()
                     }
                 }
 
-                MenuButton {
+                MenuButton
+                {
                     Layout.alignment: Qt.AlignTop
 
                     height: 64
@@ -182,15 +191,18 @@ Rectangle {
                         currentMenuButton = this
 
                         searchContainer.preferredHeight = 0
+                        searchContainer.reset()
                     }
                 }
 
-                Item {
+                Item
+                {
                     Layout.fillHeight: true
                     Layout.minimumHeight: 100
                 }
 
-                MenuButton {
+                MenuButton
+                {
                     Layout.alignment: Qt.AlignBottom
 
                     height: 64
@@ -208,10 +220,12 @@ Rectangle {
                         currentMenuButton = this
 
                         searchContainer.preferredHeight = 0
+                        searchContainer.reset()
                     }
                 }
 
-                Behavior on opacity {
+                Behavior on opacity
+                {
                     NumberAnimation
                     {
                         duration: 250
@@ -229,12 +243,14 @@ Rectangle {
 
             color: "transparent"
 
-            ColumnLayout {
+            ColumnLayout
+            {
                 id: windowContainerLayout
                 anchors.fill: parent
                 spacing: 0
 
-                SearchBar {
+                SearchBar
+                {
                     property int preferredHeight: 50
                     id: searchContainer
                     color: palette.base.lighter()
@@ -245,7 +261,8 @@ Rectangle {
 
                     icon: "qrc:/images/icons/icons8-search.svg"
 
-                    onSearchTermChanged: () => {
+                    onSearchTermChanged: () =>
+                    {
                         pageLoader.setSearchTerm()
                     }
 
@@ -254,7 +271,8 @@ Rectangle {
                     }
                 }
 
-                Rectangle {
+                Rectangle
+                {
                     id: windowContent
 
                     Layout.fillHeight: true
@@ -264,20 +282,23 @@ Rectangle {
 
                     color: palette.base.darker()
 
-                    Loader {
+                    Loader
+                    {
                         property bool loading: false
                         property string page
 
                         anchors.fill: parent
 
-                        onPageChanged: () => {
+                        onPageChanged: () =>
+                        {
                             loading = true
                             fadeOutPage()
                         }
 
                         id: pageLoader
 
-                        OpacityAnimator on opacity {
+                        OpacityAnimator on opacity
+                        {
                             id: pageLoaderAnimation
 
                             target: pageLoader

KomplexHubModule/Controls/LineEdit.qml

@@ -62,10 +62,12 @@ Item {
 
     id: lineEditRoot
 
-    ColumnLayout {
+    ColumnLayout
+    {
         anchors.fill: parent
 
-        Rectangle {
+        Rectangle
+        {
             id: lineEditBackground
 
             border.color: lineEditRoot.border.color
@@ -87,7 +89,8 @@ Item {
             topLeftRadius: lineEditRoot.radius.topLeft
             topRightRadius: lineEditRoot.radius.topRight
 
-            TextInput {
+            TextInput
+            {
                 id: lineEditInput
                 anchors.fill: parent
                 color: lineEditRoot.color
@@ -101,6 +104,14 @@ Item {
                 onAccepted: () => lineEditRoot.accepted()
                 onTextChanged: () => lineEditRoot.text = lineEditInput.text
                 onEditingFinished: () => lineEditRoot.editingFinished()
+
+                MouseArea
+                {
+                    anchors.fill: parent
+                    cursorShape: Qt.IBeamCursor
+
+                    onClicked: () => lineEditInput.focus = true
+                }
             }
 
             states: [

KomplexHubModule/Controls/SearchBar.qml

@@ -9,7 +9,8 @@ import QtQuick.Controls
 import QtQuick.Layouts
 import KomplexHub
 
-Item {
+Item
+{
     property string searchTerm: ""
     property string icon: ""
     property color color: "transparent"
@@ -17,15 +18,18 @@ Item {
     id: searchBarRootItem
     clip: true
 
-    Rectangle {
+    Rectangle
+    {
         color: searchBarRootItem.color
         anchors.fill: parent
 
-        RowLayout {
+        RowLayout
+        {
             id: searchContainerLayout
             anchors.fill: parent
 
-            LineEdit {
+            LineEdit
+            {
                 id: searchEdit
                 Layout.fillHeight: true
                 Layout.fillWidth: true
@@ -40,7 +44,8 @@ Item {
                 onAccepted: () => searchBarRootItem.searchTerm = text
             }
 
-            SquareButton {
+            SquareButton
+            {
                 id: searchButton
 
                 Layout.fillHeight: searchEdit.height
@@ -67,4 +72,9 @@ Item {
             }
         }
     ]
+
+    function reset()
+    {
+        searchEdit.text = ""
+    }
 }

KomplexHubPlugin/CMakeLists.txt

@@ -60,6 +60,7 @@ qt_add_qml_module(
         common/exceptions.h
         common/shadertoymetadata.h
         common/shaderpackmetadata.h
+        SOURCES packcompiler.h packcompiler.cpp
 )
 
 target_compile_definitions(

KomplexHubPlugin/common/exceptions.h

@@ -25,4 +25,10 @@ struct SqlException : Exception
 {
     SqlException(const QString &message, qsizetype errorCode = 0) : Exception(message, errorCode) {}
 };
+
+
+struct KOMPLEX_EXPORT ShaderCompilerException : Exception
+{
+    ShaderCompilerException(const QString &message, qsizetype errorCode = 0) : Exception(message, errorCode) {}
+};
 #endif // EXCEPTIONS_H

KomplexHubPlugin/downloadmanager.cpp

@@ -7,9 +7,39 @@
 DownloadManager::DownloadManager(QObject *parent)
     : QObject{parent}
 {
+    m_compiler = new PackCompiler(this);
 
+    QObject::connect
+    (
+        m_compiler,
+        &PackCompiler::progressChanged,
+        this,
+        &DownloadManager::compileProgressChanged
+    );
+
+    QObject::connect
+    (
+        m_compiler,
+        &PackCompiler::currentStepChanged,
+        this,
+        &DownloadManager::compileStepsCompletedChanged
+    );
+
+    QObject::connect
+    (
+        m_compiler,
+        &PackCompiler::totalStepsChanged,
+        this,
+        &DownloadManager::compileStepsChanged
+    );
 }
 
+/**
+ * [1] Download image from endpoint
+ * [2] Apply generic single-media template, with an image source
+ * [3] Optionally ask user if they would like to apply manipulative shader
+ * [4] Install locally
+ */
 auto DownloadManager::downloadImage(const QString &author, const QString &authorId, const QString &description, const QUrl &url) noexcept(false) -> void
 {
     QFuture future = QtConcurrent::run
@@ -22,15 +52,29 @@ auto DownloadManager::downloadImage(const QString &author, const QString &author
 
             try
             {
-                QUrl downloadUri = download(request, id);
+                QFuture<QUrl> downloadUri = download(request, id);
 
-                ShaderPackMetadata metadata;
-                metadata.setAuthor(author);
-                metadata.setDescription(description);
-                metadata.setName(QStringLiteral("Pexels Image (%1)").arg(id));
+                downloadUri
+                    .then
+                    (
+                        [this, author, description, id](QUrl result) -> QUrl
+                        {
+                            ShaderPackMetadata metadata;
+                            metadata.setAuthor(author);
+                            metadata.setDescription(description);
+                            metadata.setName(QStringLiteral("Pexels Image (%1)").arg(id));
 
-                setState(Complete);
-                Q_EMIT downloadComplete(downloadUri.toString());
+                            setState(Complete);
+                            Q_EMIT downloadComplete(result.toString());
+                        }
+                    )
+                    .onFailed
+                    (
+                        [] ()
+                        {
+
+                        }
+                    );
             }
             catch (FileException e)
             {
@@ -48,51 +92,48 @@ auto DownloadManager::downloadImage(const QString &author, const QString &author
 
 auto DownloadManager::downloadPack(const QString &id) noexcept(false) -> void
 {
-    QFuture future = QtConcurrent::run
+    QFuture aether = QtConcurrent::run
     (
         [this, id]()
         {
-            QUrl uri = QUrl
+            QUrl downloadUrl = QUrl
             (
                 QStringLiteral("%1/%2/%3").arg
                 (
                     KOMPLEX_API_HOST,
                     KOMPLEX_API_VERSION,
-                    KOMPLEX_ENDPOINT_PACKS_ITEM,
-                    id
+                    KOMPLEX_ENDPOINT_PACKS_ITEM
                 )
             );
 
-            QNetworkRequest request(uri);
+            QNetworkRequest request(downloadUrl);
             request.setRawHeader(QByteArray("uuid"), id.toUtf8());
+            //QUrl localUri = download(request, id, Post);
 
-            QUrl localUri;
+            QFuture<QUrl> compiledUri = download(request, id, Post);
 
-            try
-            {
-                QUrl tempUri = download(request, id);
-                QUrl decompressedUri = decompress(tempUri);
-
-                ShaderToyEntry entry = readShaderToyEntry(decompressedUri);
-                //scan and replace media via user choice. resave with decompressedUri
-
-                QUrl compiledUri = compile(decompressedUri);
-                QUrl installedUri = install(compiledUri);
-
-                setState(Complete);
-
-                Q_EMIT downloadComplete(installedUri.toString());
-            }
-            catch (FileException e)
-            {
-                setError(QStringLiteral("File Exception"), e.message);
-                localUri.clear();
-            }
-            catch (NetworkException e)
-            {
-                setError(QStringLiteral("Network Exception"), e.message);
-                localUri.clear();
-            }
+            compiledUri
+                .then
+                (
+                    [this](QUrl result) -> QUrl
+                    {
+                        m_compiler->process(result);
+                    }
+                )
+                .then
+                (
+                    [this](QUrl result) -> QUrl
+                    {
+                        install(result);
+                    }
+                )
+                .onFailed
+                (
+                    [this]()
+                    {
+                        setError(m_compiler->errorTitle(), m_compiler->errorMessage());
+                    }
+                );
         }
     );
 }
@@ -103,6 +144,36 @@ auto DownloadManager::reset() -> void
     setState(Idle);
 }
 
+auto DownloadManager::compileProgress() -> qreal
+{
+    if(m_compiler == nullptr)
+    {
+        return 0;
+    }
+
+    return m_compiler->progress();
+}
+
+auto DownloadManager::compileSteps() -> qint64
+{
+    if(m_compiler == nullptr)
+    {
+        return 0;
+    }
+
+    return m_compiler->totalSteps();
+}
+
+auto DownloadManager::compileStepsCompleted() -> qint64
+{
+    if(nullptr == m_compiler)
+    {
+        return 0;
+    }
+
+    return m_compiler->currentStep();
+}
+
 auto DownloadManager::setError(const QString &title, const QString &message) -> void
 {
     if(title == m_errorTitle && message == m_errorMessage)
@@ -115,109 +186,6 @@ auto DownloadManager::setError(const QString &title, const QString &message) ->
     Q_EMIT errorChanged();
 }
 
-auto DownloadManager::compile(const QUrl &uri) noexcept(false) -> QUrl
-{
-    if(!uri.isLocalFile())
-    {
-        throw FileException(QStringLiteral("Uri needs to be a local file"), 0);
-    }
-
-    setState(Compiling);
-
-    QDir localToolsDirectory
-    (
-        QStringLiteral("%1/.local/share/komplex/tools").arg
-        (
-            QStandardPaths::writableLocation(QStandardPaths::HomeLocation)
-        )
-    );
-
-    QDir outputDirectory = QStringLiteral("%1/komplex/build").arg
-    (
-        QStandardPaths::writableLocation(QStandardPaths::TempLocation)
-    );
-
-    QStringList arguments =
-    {
-        localToolsDirectory.absoluteFilePath(QStringLiteral("stc.py")),
-        QStringLiteral("-i"),
-        uri.toLocalFile(),
-        QStringLiteral("-o"),
-        outputDirectory.absolutePath()
-    };
-
-    if(!QFile::exists(localToolsDirectory.absoluteFilePath(QStringLiteral("stc.py"))))
-    {
-        throw FileException
-        (
-            QStringLiteral("Shader Compiler is not installed at %1").arg
-            (
-                localToolsDirectory.absoluteFilePath(QStringLiteral("stc.py"))
-            )
-        );
-    }
-
-    QProcess *process = new QProcess(this);
-
-    QObject::connect
-    (
-        process,
-        &QProcess::readyReadStandardOutput,
-        this,
-        [this, process]()
-        {
-            QByteArray processData = process->readAllStandardOutput();
-            setCompilerOutput(m_compilerOutput + processData);
-        }
-    );
-
-    QObject::connect
-    (
-        process,
-        &QProcess::readyReadStandardError,
-        this,
-        [this, process]()
-        {
-            QByteArray processData = process->readAllStandardError();
-
-            if(!processData.isValidUtf8())
-            {
-                qWarning() << QStringLiteral("Process output not valid UTF8 data");
-                return;
-            }
-
-            setCompilerOutput(m_compilerOutput + processData);
-        }
-    );
-
-    process->start(QStringLiteral("python3"), arguments);
-
-    if(!process->waitForStarted(3000))
-    {
-        process->deleteLater();
-        throw FileException(QStringLiteral("Could not start shader compiler"));
-    }
-
-    if(!process->waitForFinished())
-    {
-        process->deleteLater();
-        throw FileException(QStringLiteral("Shader compiler timeout"));
-    }
-
-    if(process->exitCode() != 0)
-    {
-        process->deleteLater();
-        throw FileException(QStringLiteral("Shader compiler error"));
-    }
-
-    process->deleteLater();
-
-    QUrl outputUri = outputDirectory.absolutePath();
-    outputUri.setScheme(QStringLiteral("file://"));
-
-    return outputUri;
-}
-
 void DownloadManager::setCompilerOutput(const QString &compilerOutput)
 {
     if (m_compilerOutput == compilerOutput)
@@ -253,241 +221,6 @@ auto DownloadManager::setDownloadProgress(qreal progress) -> void
     Q_EMIT downloadProgressChanged();
 }
 
-auto DownloadManager::decompress(const QUrl &uri) noexcept(false) -> QUrl
-{
-    return {};
-}
-
-// void DownloadManager::save(ShaderToyEntry entry)
-// {
-//     QString directoryLocation = QStringLiteral("%1/komplex/src/%2").arg(QStandardPaths::writableLocation(QStandardPaths::TempLocation), entry.metadata.id);
-//     QDir directory(directoryLocation);
-
-//     if(!directory.exists())
-//     {
-//         directory.mkpath(directoryLocation + QStringLiteral("/shaders"));
-//         directory.mkpath(directoryLocation + QStringLiteral("/images"));
-//         directory.mkpath(directoryLocation + QStringLiteral("/videos"));
-//     }
-
-//     QDir shaderDirectory(directoryLocation + QStringLiteral("/shaders"));
-//     QDir imageDirectory(directoryLocation + QStringLiteral("/images"));
-//     // QDir videoDirectory(directoryLocation + QStringLiteral("/videos"));
-
-//     QJsonObject rootObject;
-//     rootObject[QStringLiteral("author")] = entry.metadata.username;
-//     rootObject[QStringLiteral("name")] = entry.metadata.name;
-//     rootObject[QStringLiteral("version")] = entry.metadata.version;
-//     rootObject[QStringLiteral("engine")] = QStringLiteral("shadertoy");
-//     rootObject[QStringLiteral("description")] = entry.metadata.description;
-//     rootObject[QStringLiteral("id")] = entry.metadata.id;
-//     rootObject[QStringLiteral("tags")] = QJsonArray::fromStringList(entry.metadata.tags);
-//     QMap<QString,QString> externalMedia;
-
-//     externalMedia.insert
-//     (
-//         directory.absoluteFilePath(QStringLiteral("thumbnail.jpg")),
-//         QStringLiteral("/media/shaders/%1.jpg").arg(entry.metadata.id)
-//     );
-
-//     for(const ShaderToyRenderPass &pass : std::as_const(entry.renderPasses))
-//     {
-//         // skip tone generators
-//         if(pass.type == QStringLiteral("sound"))
-//             continue;
-
-//         QString passName = pass.name;
-
-//         if(passName.contains(QStringLiteral("Buf")) && !passName.contains(QStringLiteral("Buffer")))
-//             passName.replace(QStringLiteral("Buf"), QStringLiteral("Buffer"));
-
-//         QFile shaderFile(shaderDirectory.absoluteFilePath(passName + QStringLiteral(".frag")));
-
-//         if(!shaderFile.open(QFile::WriteOnly))
-//         {
-//             qWarning() << QStringLiteral("Could not open shader file for saving");
-//             return;
-//         }
-
-//         if(shaderFile.write(pass.code) != pass.code.length())
-//         {
-//             qWarning() << QStringLiteral("Could not write shader file data");
-//             shaderFile.close();
-//             return;
-//         }
-
-//         shaderFile.close();
-
-//         //this is the common file
-//         if(pass.type == QStringLiteral("common"))
-//             continue; // wont have any inputs
-
-//         const ShaderToyRenderOutput *channelOutput = nullptr;
-
-//         for(const ShaderToyRenderOutput &output : std::as_const(pass.outputs))
-//         {
-//             if(output.channel == 0)
-//             {
-//                 channelOutput = &output;
-//                 break;
-//             }
-//         }
-
-//         QList<QJsonObject> channels(4);
-
-//         QJsonObject *passObject = nullptr;
-
-//         //this is the root shader
-//         if(pass.type == QStringLiteral("image"))
-//         {
-//             rootObject[QStringLiteral("source")] = QStringLiteral("./shaders/%1.frag.qsb").arg(pass.name);
-//             passObject = &rootObject;
-//         }
-//         else
-//             passObject = new QJsonObject;
-
-//         for(const ShaderToyRenderInput &input : std::as_const(pass.inputs))
-//         {
-//             /*
-//             * Only recursive buffers, images, videos and shader buffers are currently supported.
-//             * audio will default to audio capture
-//             */
-
-//             if(!m_supportedChannelTypes.contains(input.ctype))
-//             {
-//                 qWarning() << input.ctype << QStringLiteral(" is not a valid channel type");
-//                 continue;
-//             }
-
-//             // recursive buffer reference
-//             if(channelOutput && input.id == channelOutput->id)
-//             {
-//                 passObject->insert(QStringLiteral("frame_buffer_channel"), input.channel);
-//                 continue;
-//             }
-
-//             if(input.ctype == QStringLiteral("buffer"))
-//             {
-//                 // get input reference by id
-//                 const ShaderToyRenderPass *inputPass = nullptr;
-
-//                 for(const ShaderToyRenderPass &passSubScan : std::as_const(entry.renderPasses))
-//                 {
-//                     for(const ShaderToyRenderOutput &output : std::as_const(passSubScan.outputs))
-//                     {
-//                         if(output.id == input.id && output.channel == 0)
-//                         {
-//                             inputPass = &passSubScan;
-//                             break;
-//                         }
-
-//                         if(inputPass)
-//                             break;
-//                     }
-//                 }
-
-//                 //whoopsie
-//                 if(!inputPass)
-//                     continue;
-
-//                 QString name = inputPass->name.toCaseFolded();
-//                 name.replace(name.length() - 1, 1, name.right(1).toUpper());
-//                 name.remove(QLatin1Char(' '));
-//                 name.replace(QStringLiteral("buf"), QStringLiteral("buffer"));
-
-//                 channels[input.channel][QStringLiteral("source")] = QStringLiteral("{%1}").arg(name);
-//             }
-
-//             else if(input.ctype == QStringLiteral("audio"))
-//                 channels[input.channel][QStringLiteral("type")] = 4;
-
-//             else if(input.ctype == QStringLiteral("texture"))
-//             {
-//                 QString filename = input.source;
-//                 filename = filename.mid(filename.lastIndexOf(QLatin1Char('/')) + 1);
-
-//                 channels[input.channel][QStringLiteral("type")] = 0;
-//                 channels[input.channel][QStringLiteral("source")] = QStringLiteral("./images/%1").arg(filename);
-
-//                 externalMedia.insert(imageDirectory.absoluteFilePath(filename), input.source);
-//             }
-
-//             //select video file after compilation
-//             else if(input.ctype == QStringLiteral("video"))
-//             {
-//                 //set the channel source to a uuid then add that uuid to the video
-//                 // selection stringlist
-//                 QString sourceName = QUuid::createUuidV7().toString();
-//                 channels[input.channel][QStringLiteral("type")] = 1;
-//                 channels[input.channel][QStringLiteral("source")] = sourceName;
-
-//                 QStringList newSelections = m_videoSelections;
-//                 newSelections += sourceName;
-
-//                 setVideoSelections(newSelections);
-//             }
-
-//             channels[input.channel][QStringLiteral("filter")] = input.filter;
-//             channels[input.channel][QStringLiteral("wrap")] = input.wrap;
-//             channels[input.channel][QStringLiteral("invert")] = input.verticalFlip;
-//             channels[input.channel][QStringLiteral("srgb")] = input.srgb;
-//             channels[input.channel][QStringLiteral("internal")] = input.internal;
-//         }
-
-//         for(int i = 0; i < 4; ++i)
-//         {
-//             if(channels[i].isEmpty())
-//                 continue;
-
-//             passObject->insert(QStringLiteral("channel%1").arg(i), channels[i]);
-//         }
-
-//         //this is a buffer
-//         if(pass.type == QStringLiteral("buffer"))
-//         {
-//             QString name = pass.name.toCaseFolded();
-//             name.replace(name.length() - 1, 1, name.right(1).toUpper());
-//             name.remove(QLatin1Char(' '));
-//             name.replace(QStringLiteral("buf"), QStringLiteral("buffer"));
-//             passObject->insert(QStringLiteral("source"), QStringLiteral("./shaders/%1.frag.qsb").arg(passName));
-
-//             rootObject[name] = *passObject;
-//         }
-
-//         if(*passObject != rootObject)
-//             delete passObject;
-//     }
-
-//     QFile shaderPackFile(directory.absoluteFilePath(QStringLiteral("pack.json")));
-
-//     if(!shaderPackFile.open(QFile::WriteOnly))
-//     {
-//         qWarning() << QStringLiteral("Could not open pack file");
-//         return;
-//     }
-
-//     QJsonDocument packDocument;
-//     packDocument.setObject(rootObject);
-
-//     QByteArray jsonData = packDocument.toJson(QJsonDocument::Indented);
-
-//     if(shaderPackFile.write(jsonData) != jsonData.length())
-//     {
-//         qWarning() << QStringLiteral("Could not write pack data");
-//         return;
-//     }
-
-//     const QStringList keys = externalMedia.keys();
-
-//     // qWarning() << QStringLiteral("Downloading %1 Images").arg(externalMedia.count());
-//     // setStatus(Compiling, QStringLiteral("Downloading images"));
-
-//     setTotalDownloads(externalMedia.count());
-
-//     for(const QString &key : keys)
-//         downloadMedia(key, externalMedia[key]);
-// }
-
 auto DownloadManager::install(const QUrl &uri) noexcept(false) -> QUrl
 {
     if(!uri.isLocalFile())
@@ -575,116 +308,129 @@ auto DownloadManager::install(const QUrl &uri) noexcept(false) -> QUrl
     return installUri;
 }
 
-auto DownloadManager::download(const QNetworkRequest &request, const QString &id) noexcept(false) -> QUrl
+auto DownloadManager::download(const QNetworkRequest &request, const QString &id, RequestType type) -> QFuture<QUrl>
 {
-    QEventLoop loop;
-
-    QUrl downloadUri = QStringLiteral("%1/%2").arg
+    return QtConcurrent::run
     (
-        QStandardPaths::writableLocation(QStandardPaths::TempLocation),
-        request.url().path().split('/', Qt::SkipEmptyParts).last()
-    );
-
-    QWeakPointer<QNetworkAccessManager> reference = CoreServices::networkAccessManager();
-    QSharedPointer<QNetworkAccessManager> manager = reference.toStrongRef();
-
-    if(manager == nullptr)
-    {
-        throw NetworkException
-        (
-            QStringLiteral("Network Manager reference has already been deleted")
-        );
-    }
-
-    QFile downloadFile(downloadUri.toLocalFile());
-
-    if(!downloadFile.open(QFile::ReadWrite))
-    {
-        throw FileException(QStringLiteral("Could not open temp file location"));
-    }
-
-    QNetworkReply *reply = manager->post(request, nullptr);
-
-    QObject::connect
-    (
-        reply,
-        &QNetworkReply::finished,
-        &loop,
-        &QEventLoop::quit
-    );
-
-    QObject::connect
-    (
-        reply,
-        &QNetworkReply::errorOccurred,
-        &loop,
-        [&loop](QNetworkReply::NetworkError error) -> void
+        [this, request, id]() -> QUrl
         {
-            loop.quit();
+            QEventLoop loop;
+            QString filename = id;
 
-            throw NetworkException
+            if(filename.isEmpty())
+            {
+                filename = QUuid::createUuidV7().toString();
+            }
+
+            QUrl downloadUri = QStringLiteral("%1/%2").arg
             (
-                QStringLiteral("Network Error %1").arg
-                (
-                    QString::number(static_cast<qint64>(error))
-                )
+                QStandardPaths::writableLocation(QStandardPaths::TempLocation),
+                filename
             );
-        }
-    );
 
-    QObject::connect
-    (
-        reply,
-        &QNetworkReply::downloadProgress,
-        this,
-        [this](qint64 bytesDownloaded, qint64 bytesTotal)
-        {
-            setDownloadProgress(static_cast<qreal>(bytesDownloaded) / bytesTotal);
-        }
-    );
+            auto manager = CoreServices::networkAccessManager().toStrongRef();
 
-    QObject::connect
-    (
-        reply,
-        &QNetworkReply::readyRead,
-        this,
-        [this, &downloadFile, &reply]()
-        {
-            if(!downloadFile.isOpen())
+            if(manager == nullptr)
             {
-                return;
+                throw NetworkException
+                {
+                    QStringLiteral("Network Manager reference has already been deleted")
+                };
             }
 
-            quint64 bytes = reply->bytesAvailable();
-            quint64 bytesWritten = downloadFile.write(reply->read(bytes));
+            QFile downloadFile(downloadUri.toLocalFile());
 
-            if(bytesWritten != bytes)
+            if(!downloadFile.open(QFile::ReadWrite))
             {
-                throw FileException(QStringLiteral("Could not write temp file data"));
+                throw FileException(QStringLiteral("Could not open temp file location"));
             }
+
+            QNetworkReply *reply = manager->post(request, nullptr);
+
+            QObject::connect
+            (
+                reply,
+                &QNetworkReply::finished,
+                &loop,
+                &QEventLoop::quit
+            );
+
+            QObject::connect
+            (
+                reply,
+                &QNetworkReply::errorOccurred,
+                &loop,
+                [&loop](QNetworkReply::NetworkError error) -> void
+                {
+                    loop.quit();
+
+                    throw NetworkException
+                    (
+                        QStringLiteral("Network Error %1").arg
+                        (
+                            QString::number(static_cast<qint64>(error))
+                        )
+                    );
+                }
+            );
+
+            QObject::connect
+            (
+                reply,
+                &QNetworkReply::downloadProgress,
+                this,
+                [this](qint64 bytesDownloaded, qint64 bytesTotal)
+                {
+                    setDownloadSize(bytesTotal);
+                    setDownloadedBytes(bytesDownloaded);
+                    setDownloadProgress(static_cast<qreal>(bytesDownloaded) / bytesTotal);
+                }
+            );
+
+            QObject::connect
+            (
+                reply,
+                &QNetworkReply::readyRead,
+                this,
+                [this, &downloadFile, &reply]()
+                {
+                    if(!downloadFile.isOpen())
+                    {
+                        return;
+                    }
+
+                    quint64 bytes = reply->bytesAvailable();
+                    quint64 bytesWritten = downloadFile.write(reply->read(bytes));
+
+                    if(bytesWritten != bytes)
+                    {
+                        throw FileException(QStringLiteral("Could not write temp file data"));
+                    }
+                }
+            );
+
+            if(!reply->isFinished())
+            {
+                loop.exec();
+            }
+
+            if(reply->bytesAvailable() > 0)
+            {
+                quint64 bytes = reply->bytesAvailable();
+                quint64 bytesWritten = downloadFile.write(reply->read(bytes));
+
+                if(bytesWritten != bytes)
+                {
+                    throw FileException(QStringLiteral("Could not write temp file data"));
+                }
+            }
+
+            downloadFile.close();
+            manager.clear();
+
+            return downloadUri;
         }
     );
-
-    if(!reply->isFinished())
-    {
-        loop.exec();
-    }
-
-    if(reply->bytesAvailable() > 0)
-    {
-        quint64 bytes = reply->bytesAvailable();
-        quint64 bytesWritten = downloadFile.write(reply->read(bytes));
-
-        if(bytesWritten != bytes)
-        {
-            throw FileException(QStringLiteral("Could not write temp file data"));
-        }
-    }
-
-    downloadFile.close();
-    manager.clear();
-
-    return downloadUri;
 }
 
 auto DownloadManager::readShaderToyEntry(const QUrl &uri) noexcept(false) -> ShaderToyEntry
@@ -816,4 +562,22 @@ auto DownloadManager::readShaderToyEntry(const QUrl &uri) noexcept(false) -> Sha
     }
 
     return std::move(entry);
+}
+
+auto DownloadManager::setDownloadedBytes(qint64 downloadedBytes) -> void
+{
+    if (m_downloadedBytes == downloadedBytes)
+        return;
+
+    m_downloadedBytes = downloadedBytes;
+    emit downloadedBytesChanged();
+}
+
+auto DownloadManager::setDownloadSize(qint64 downloadSize) -> void
+{
+    if (m_downloadSize == downloadSize)
+        return;
+
+    m_downloadSize = downloadSize;
+    emit downloadSizeChanged();
 }

KomplexHubPlugin/downloadmanager.h

@@ -17,6 +17,7 @@
 #include "common/komplex_global.h"
 #include "common/shadertoymetadata.h"
 #include "common/shaderpackmetadata.h"
+#include "packcompiler.h"
 
 class KOMPLEX_EXPORT DownloadManager : public QObject
 {
@@ -35,6 +36,12 @@ public:
     };
     Q_ENUM(State)
 
+    enum RequestType
+    {
+        Get,
+        Post
+    };
+
     explicit DownloadManager(QObject *parent = nullptr);
 
     /**
@@ -59,7 +66,15 @@ public:
     auto state() const -> State { return m_state; }
     auto reset() -> void;
     auto downloadProgress() -> qreal { return m_downloadProgress; }
-    auto compileProgress() -> qreal { return m_compileProgress; }
+    auto compileProgress() -> qreal;
+    auto compileSteps() -> qint64;
+    auto compileStepsCompleted() -> qint64;
+
+    auto downloadSize() const -> qint64 { return m_downloadSize; }
+    auto setDownloadSize(qint64 downloadSize) -> void;
+
+    auto downloadedBytes() const -> qint64 { return m_downloadedBytes; }
+    auto setDownloadedBytes(qint64 downloadedBytes) -> void;
 
 protected:
     auto setError(const QString &title, const QString &message) -> void;
@@ -74,12 +89,15 @@ signals:
     auto downloadProgressChanged() -> void;
     auto compileProgressChanged() -> void;
     auto downloadComplete(const QString &uri) -> void;
+    auto downloadSizeChanged() -> void;
+    auto downloadedBytesChanged() -> void;
+    auto compileStepsChanged() -> void;
+    auto compileStepsCompletedChanged() -> void;
 
 private:
-    auto decompress(const QUrl &uri) noexcept(false) -> QUrl;
     auto compile(const QUrl &uri) noexcept(false) -> QUrl;
     auto install(const QUrl &uri) noexcept(false) -> QUrl;
-    auto download(const QNetworkRequest &request, const QString &id) noexcept(false) -> QUrl;
+    auto download(const QNetworkRequest &request, const QString &id, RequestType type = Get) -> QFuture<QUrl>;
     auto readShaderToyEntry(const QUrl &uri) noexcept(false) -> ShaderToyEntry;
 
     QString m_compilerOutput;
@@ -87,17 +105,25 @@ private:
     QString m_errorMessage;
 
     qreal m_downloadProgress = 0;
+    qint64 m_downloadSize = 0;
+    qint64 m_downloadedBytes = 0;
     qreal m_compileProgress = 0;
     State m_state;
 
     QMutex m_downloadMutex;
 
+    PackCompiler *m_compiler = nullptr;
+
     Q_PROPERTY(QString compilerOutput READ compilerOutput WRITE setCompilerOutput NOTIFY compilerOutputChanged FINAL)
     Q_PROPERTY(QString errorTitle READ errorTitle NOTIFY errorChanged FINAL)
     Q_PROPERTY(QString errorMessage READ errorMessage NOTIFY errorChanged FINAL)
     Q_PROPERTY(qreal downloadProgress READ downloadProgress NOTIFY downloadProgressChanged FINAL)
     Q_PROPERTY(qreal compileProgress READ compileProgress NOTIFY compileProgressChanged FINAL)
     Q_PROPERTY(State state READ state NOTIFY stateChanged FINAL)
+    Q_PROPERTY(qint64 downloadSize READ downloadSize WRITE setDownloadSize NOTIFY downloadSizeChanged FINAL)
+    Q_PROPERTY(qint64 downloadedBytes READ downloadedBytes WRITE setDownloadedBytes NOTIFY downloadedBytesChanged FINAL)
+    Q_PROPERTY(qint64 compileSteps READ compileSteps NOTIFY compileStepsChanged FINAL)
+    Q_PROPERTY(qint64 compileStepsCompleted READ compileStepsCompleted NOTIFY compileStepsCompletedChanged FINAL)
 };
 Q_DECLARE_METATYPE(DownloadManager)
 

KomplexHubPlugin/packcompiler.cpp

@@ -0,0 +1,942 @@
+#include "packcompiler.h"
+#include "common/exceptions.h"
+#include <utility>
+
+PackCompiler::PackCompiler(QObject *parent)
+    : QObject(parent)
+{
+    if(m_variableExpressions.isEmpty())
+    {
+        for(const QString &variable : m_updateVariables)
+        {
+            m_variableExpressions.append
+            (
+                QRegularExpression
+                (
+                    QStringLiteral("\\b(?<!ubuf\\.)%1\\b").arg
+                    (
+                        variable
+                    )
+                )
+            );
+        }
+    }
+}
+
+PackCompiler::~PackCompiler() = default;
+
+auto PackCompiler::reset() -> void
+{
+    setError({}, {});
+    setProgress(std::numeric_limits<qreal>::infinity());
+    setTotalSteps(0);
+    setCurrentStep(0);
+    setCompilerOutput({});
+    setStatus({});
+    setState(Idle);
+}
+
+auto PackCompiler::process(const QUrl &uri) -> QFuture<QUrl>
+{
+    return QtConcurrent::run
+    (
+        [this, uri] () -> QUrl
+        {
+            if(!uri.isLocalFile())
+            {
+                setError
+                (
+                    QStringLiteral("File Error"),
+                    QStringLiteral("URI %1 is not a local file").arg
+                    (
+                        uri.toString()
+                    )
+                );
+
+                return {};
+            }
+
+            reset();
+            setStatus(QStringLiteral("Compiling %1").arg(uri.fileName()));
+            setState(Compiling);
+
+            if(!validateDirectory(uri))
+            {
+                throw FileException
+                (
+                    QStringLiteral("Source directory is invalid")
+                );
+            }
+
+            QUrl buildUri
+            (
+                QStringLiteral("file://%1/komplex/build/%2").arg
+                (
+                    QStandardPaths::writableLocation
+                    (
+                        QStandardPaths::TempLocation
+                    ),
+                    uri.fileName()
+                )
+            );
+
+            try
+            {
+                extract(uri, buildUri);
+                prepareShaders(buildUri);
+                compile(buildUri);
+            }
+            catch (FileException e)
+            {
+                removeDirectory(buildUri);
+                setError(QStringLiteral("File Error"), e.message);
+                throw e;
+            }
+            catch (ShaderCompilerException e)
+            {
+                removeDirectory(buildUri);
+                setError(QStringLiteral("Compiler Error"), e.message);
+                throw e;
+            }
+
+            return buildUri;
+        }
+    );
+}
+
+auto PackCompiler::prepareShaders(const QUrl &uri) noexcept(false) -> void
+{
+    if(!uri.isLocalFile())
+    {
+        throw FileException
+        (
+            QStringLiteral("URI %1 is not a local file uri").arg
+            (
+                uri.toString()
+            )
+        );
+    }
+
+    QDir sourceDirectory(uri.toLocalFile());
+    QDir shaderDirectory
+    (
+        sourceDirectory.absoluteFilePath
+        (
+            QStringLiteral("shaders")
+        )
+    );
+
+    QStringList entries = shaderDirectory.entryList
+    (
+        QDir::NoDotAndDotDot | QDir::Files
+    );
+
+    setTotalSteps(entries.count() * 2);
+    setProgress(0);
+
+    QByteArray commonFragmentData = loadCommonFragmentData(uri);
+
+    if(commonFragmentData.length() > 0)
+    {
+        setTotalSteps(m_totalSteps -= 1);
+        incrementCompileStep();
+    }
+
+    QByteArray commonVertexData = loadCommonVertexData(uri);
+
+    if(commonVertexData.length() > 0)
+    {
+        setTotalSteps(m_totalSteps -= 1);
+        incrementCompileStep();
+    }
+
+    QByteArray globalData = loadGlobalData(uri);
+
+    if(globalData.length() > 0)
+    {
+        setTotalSteps(m_totalSteps -= 1);
+        incrementCompileStep();
+    }
+
+    for(const QString &entry : std::as_const(entries))
+    {
+        auto commonFragMatch = m_commonFragmentExpression.match(entry);
+        auto commonVertMatch = m_commonVertexExpression.match(entry);
+
+        if(commonFragMatch.hasMatch() || commonVertMatch.hasMatch())
+        {
+            continue;
+        }
+
+        auto fragMatch = m_fragmentExpression.match(entry);
+        auto vertMatch = m_vertexExpression.match(entry);
+
+        QFile file(shaderDirectory.absoluteFilePath(entry));
+
+        if(!file.open(QFile::ReadWrite))
+        {
+            throw FileException
+            (
+                QStringLiteral("Could not open file %1 for preperation").arg
+                (
+                    shaderDirectory.absoluteFilePath(entry)
+                )
+            );
+        }
+
+        QByteArray fileData = file.readAll();
+
+        if(fileData.length() != file.size())
+        {
+            throw FileException
+            (
+                QStringLiteral("Could not read file %1 for preperation").arg
+                (
+                    shaderDirectory.absoluteFilePath(entry)
+                )
+            );
+        }
+
+        file.close();
+
+        QByteArray commonData;
+
+        if(fragMatch.hasMatch())
+        {
+            commonData = commonFragmentData;
+        }
+        else if(vertMatch.hasMatch())
+        {
+            commonData = commonVertexData;
+        }
+        else
+        {
+            throw FileException
+            (
+                QStringLiteral("File %1 is an unrecognized shader type (frag/vert)").arg
+                (
+                    shaderDirectory.absoluteFilePath(entry)
+                )
+            );
+        }
+
+        QByteArray combinedData = QByteArray
+        (
+            globalData + QByteArray("\n") +
+            commonData + QByteArray("\n") +
+            fileData
+        );
+
+        for(const QRegularExpression &expression : std::as_const(m_variableExpressions))
+        {
+            auto match = expression.match(combinedData);
+
+            if(match.hasMatch())
+            {
+                QByteArray replacement = QByteArray("ubuf.") + match.captured().toLocal8Bit();
+                combinedData.replace(match.captured().toLocal8Bit(), replacement);
+            }
+        }
+
+        QByteArray preparedData = QByteArray
+        (
+            m_header.toLocal8Bit() + QByteArray("\n") +
+            combinedData + QByteArray("\n") +
+            m_footer.toLocal8Bit()
+        );
+
+        if(!file.open(QFile::ReadWrite | QFile::Truncate))
+        {
+            throw FileException
+            (
+                QStringLiteral("Could not open file %1 for writing preperation").arg
+                (
+                    shaderDirectory.absoluteFilePath(entry)
+                )
+            );
+        }
+
+        if(file.write(preparedData) != preparedData.length())
+        {
+            throw FileException
+            (
+                QStringLiteral("Could not write file %1 preperation").arg
+                (
+                    shaderDirectory.absoluteFilePath(entry)
+                )
+            );
+        }
+
+        file.close();
+        incrementCompileStep();
+    }
+}
+
+auto PackCompiler::loadCommonFragmentData(const QUrl &uri) -> QByteArray
+{
+    if(!uri.isLocalFile())
+    {
+        throw FileException
+        (
+            QStringLiteral("URI %1 is not a local file uri").arg
+            (
+                uri.toString()
+            )
+        );
+    }
+
+    QDir sourceDirectory
+    (
+        QStringLiteral("%1/shaders/").arg
+        (
+            uri.toLocalFile()
+        )
+    );
+
+    const QStringList entries = sourceDirectory.entryList
+    (
+        QDir::NoDotAndDotDot | QDir::Files
+    );
+
+    QByteArray data;
+
+    for(const QString &entry : entries)
+    {
+        if (entry.toLower() != QStringLiteral("common.frag"))
+        {
+            continue;
+        }
+
+        QFile file(sourceDirectory.absoluteFilePath(entry));
+
+        if(!file.open(QFile::ReadWrite))
+        {
+            throw FileException(QStringLiteral("Could not open common shader data"));
+        }
+
+        data = file.readAll();
+
+        if(data.length() != file.size())
+        {
+            throw FileException(QStringLiteral("Common fragment shader file read error"));
+        }
+
+        file.close();
+        break;
+    }
+
+    return data;
+}
+
+auto PackCompiler::loadCommonVertexData(const QUrl &uri) -> QByteArray
+{
+    if(!uri.isLocalFile())
+    {
+        throw FileException
+        (
+            QStringLiteral("URI %1 is not a local file uri").arg
+            (
+                uri.toString()
+            )
+        );
+    }
+
+    QDir sourceDirectory
+    (
+        QStringLiteral("%1/shaders/").arg
+        (
+            uri.toLocalFile()
+        )
+    );
+
+    const QStringList entries = sourceDirectory.entryList
+    (
+        QDir::NoDotAndDotDot | QDir::Files
+    );
+
+    QByteArray data;
+
+    for(const QString &entry : entries)
+    {
+        if (entry.toLower() != QStringLiteral("common.vert"))
+        {
+            continue;
+        }
+
+        QFile file(sourceDirectory.absoluteFilePath(entry));
+
+        if(!file.open(QFile::ReadWrite))
+        {
+            throw FileException(QStringLiteral("Could not open common vertex shader data"));
+        }
+
+        data = file.readAll();
+
+        if(data.length() != file.size())
+        {
+            throw FileException(QStringLiteral("Common vertex shader file read error"));
+        }
+
+        file.close();
+        break;
+    }
+
+    return data;
+}
+
+auto PackCompiler::loadGlobalData(const QUrl &uri) -> QByteArray
+{
+    if(!uri.isLocalFile())
+    {
+        throw FileException
+        (
+            QStringLiteral("URI %1 is not a local file uri").arg
+            (
+                uri.toString()
+            )
+        );
+    }
+
+    QDir sourceDirectory
+    (
+        QStringLiteral("%1/shaders/").arg
+        (
+            uri.toLocalFile()
+        )
+    );
+
+    const QStringList entries = sourceDirectory.entryList
+    (
+        QDir::NoDotAndDotDot | QDir::Files
+    );
+
+    QByteArray data;
+
+    for(const QString &entry : entries)
+    {
+        if (entry.toLower() != QStringLiteral("global.glsl"))
+        {
+            continue;
+        }
+
+        QFile file(sourceDirectory.absoluteFilePath(entry));
+
+        if(!file.open(QFile::ReadWrite))
+        {
+            throw FileException
+            (
+                QStringLiteral("Could not open global shader data")
+            );
+        }
+
+        data = file.readAll();
+
+        if(data.length() != file.size())
+        {
+            throw FileException
+            (
+                QStringLiteral("Global shader file read error")
+            );
+        }
+
+        file.close();
+        break;
+    }
+
+    return data;
+}
+
+auto PackCompiler::validateDirectory(const QUrl &uri) -> bool
+{
+    QDir directory(uri.toLocalFile());
+
+    return
+    (
+        directory.exists(QStringLiteral("shaders")) &&
+        directory.exists(QStringLiteral("pack.json"))
+    );
+}
+
+auto PackCompiler::extract(const QUrl &sourceUri, const QUrl &destinationUri) noexcept(false) -> void
+{
+    if(!sourceUri.isLocalFile() || !sourceUri.isValid())
+    {
+        throw FileException(QStringLiteral("Uri needs to be a local file"), 0);
+    }
+
+    QUrl outputUri
+    (
+        QStringLiteral("file://%1_proc").arg
+        (
+            sourceUri.toLocalFile()
+        )
+    );
+
+    QStringList arguments =
+    {
+        QStringLiteral("-xzf"),
+        sourceUri.toLocalFile(),
+        QStringLiteral("-C"),
+        outputUri.toLocalFile()
+    };
+
+    QProcess *process = new QProcess(this);
+
+    QObject::connect
+    (
+        process,
+        &QProcess::readyReadStandardOutput,
+        this,
+        [this, process]()
+        {
+            QByteArray processData = process->readAllStandardOutput();
+            setCompilerOutput(m_compilerOutput + processData);
+        }
+    );
+
+    QObject::connect
+    (
+        process,
+        &QProcess::readyReadStandardError,
+        this,
+        [this, process]()
+        {
+            QByteArray processData = process->readAllStandardError();
+
+            if(!processData.isValidUtf8())
+            {
+                qWarning() << QStringLiteral("Process output not valid UTF8 data");
+                return;
+            }
+
+            setCompilerOutput(m_compilerOutput + processData);
+        }
+    );
+
+    process->start(QStringLiteral("tar"), arguments);
+
+    if(!process->waitForStarted(3000))
+    {
+        process->deleteLater();
+        throw FileException(QStringLiteral("Could not start preprocessor"));
+    }
+
+    if(!process->waitForFinished())
+    {
+        process->deleteLater();
+        throw FileException(QStringLiteral("Preprocessor timeout"));
+    }
+
+    if(process->exitCode() != 0)
+    {
+        process->deleteLater();
+        throw ShaderCompilerException(m_compilerOutput);
+    }
+
+    process->deleteLater();
+}
+
+auto PackCompiler::copyFile(const QUrl &sourceUri, const QUrl &destinationUri) noexcept(false) -> void
+{
+    if(!QFile::exists(sourceUri.toLocalFile()))
+    {
+        return;
+    }
+
+    if(!QFile::copy(sourceUri.toLocalFile(), destinationUri.toLocalFile()))
+    {
+        throw FileException(QStringLiteral("Could not copy file"));
+    }
+}
+
+auto PackCompiler::compile(const QUrl &uri) noexcept(false) -> void
+{
+    if(!uri.isLocalFile() || !uri.isValid())
+    {
+        throw FileException(QStringLiteral("Uri needs to be a local file"), 0);
+    }
+
+    setState(Compiling);
+
+    QDir packDirectory(uri.toLocalFile());
+    QDir shaderDirectory
+    (
+        packDirectory.absoluteFilePath
+        (
+            QStringLiteral("shaders")
+        )
+    );
+
+    shaderDirectory.setNameFilters
+    (
+        {
+            QStringLiteral("*.frag"),
+            QStringLiteral("*.vert")
+        }
+    );
+
+    const QStringList shaders = shaderDirectory.entryList
+    (
+        QDir::Files | QDir::NoDotAndDotDot
+    );
+
+    for(const QString &shader : shaders)
+    {
+        if(shader.startsWith(QStringLiteral("common."), Qt::CaseInsensitive))
+        {
+            continue;
+        }
+
+        QUrl shaderUri
+        (
+            QStringLiteral("file://%1").arg
+            (
+                shaderDirectory.absoluteFilePath(shader)
+            )
+        );
+
+        preprocess(shaderUri);
+        appendVersion(shaderUri);
+        compileShader(shaderUri);
+
+        incrementCompileStep();
+    }
+}
+
+auto PackCompiler::preprocess(const QUrl &uri) noexcept(false) -> void
+{
+    if(!uri.isLocalFile() || !uri.isValid())
+    {
+        throw FileException(QStringLiteral("Uri needs to be a local file"), 0);
+    }
+
+    QUrl outputUri
+    (
+        QStringLiteral("file://%1_proc").arg
+        (
+            uri.toLocalFile()
+        )
+    );
+
+    QStringList arguments =
+    {
+        QStringLiteral("-P"),
+        uri.toLocalFile(),
+        outputUri.toLocalFile()
+    };
+
+    QProcess *process = new QProcess(this);
+
+    QObject::connect
+    (
+        process,
+        &QProcess::readyReadStandardOutput,
+        this,
+        [this, process]()
+        {
+            QByteArray processData = process->readAllStandardOutput();
+            setCompilerOutput(m_compilerOutput + processData);
+        }
+    );
+
+    QObject::connect
+    (
+        process,
+        &QProcess::readyReadStandardError,
+        this,
+        [this, process]()
+        {
+            QByteArray processData = process->readAllStandardError();
+
+            if(!processData.isValidUtf8())
+            {
+                qWarning() << QStringLiteral("Process output not valid UTF8 data");
+                return;
+            }
+
+            setCompilerOutput(m_compilerOutput + processData);
+        }
+    );
+
+    process->start(QStringLiteral("cpp"), arguments);
+
+    if(!process->waitForStarted(3000))
+    {
+        process->deleteLater();
+        throw FileException(QStringLiteral("Could not start preprocessor"));
+    }
+
+    if(!process->waitForFinished())
+    {
+        process->deleteLater();
+        throw FileException(QStringLiteral("Preprocessor timeout"));
+    }
+
+    if(process->exitCode() != 0)
+    {
+        process->deleteLater();
+        throw ShaderCompilerException(m_compilerOutput);
+    }
+
+    process->deleteLater();
+
+    QFile::remove(uri.toLocalFile());
+    QFile::rename(outputUri.toLocalFile(), uri.toLocalFile());
+}
+
+auto PackCompiler::appendVersion(const QUrl &uri) noexcept(false) -> void
+{
+    if(!uri.isValid() || !uri.isLocalFile())
+    {
+        throw FileException(QStringLiteral("File is not a valid local file"));
+    }
+
+    QFile file(uri.toLocalFile());
+
+    if(!file.open(QFile::ReadWrite))
+    {
+        throw FileException(QStringLiteral("Could not open file for appending"));
+    }
+
+    QByteArray data = file.readAll();
+
+    file.seek(0);
+
+    file.write(m_version);
+    file.write(QByteArray("\n\n"));
+    file.write(data);
+
+    file.close();
+}
+
+auto PackCompiler::compileShader(const QUrl &uri) noexcept(false) -> void
+{
+    if(!uri.isLocalFile())
+    {
+        throw FileException
+        (
+            QStringLiteral("Uri needs to be a local file")
+        );
+    }
+
+    QString qsb = QStringLiteral("/usr/lib/qt6/bin/qsb");
+
+    if(!QFile::exists(qsb))
+    {
+        throw FileException(QStringLiteral("QSB missing"));
+    }
+
+    QUrl outputUri
+    (
+        QStringLiteral("file://%1.qsb").arg
+        (
+            uri.toLocalFile()
+        )
+    );
+
+    QStringList arguments =
+    {
+        QStringLiteral("--glsl"),
+        QStringLiteral("330es, 330, 440"),
+        QStringLiteral("-o"),
+        outputUri.toLocalFile(),
+        uri.toLocalFile()
+    };
+
+    QProcess *process = new QProcess(this);
+    QString path = uri.toDisplayString(QUrl::RemoveFilename | QUrl::RemoveScheme).remove(QStringLiteral("//"));
+    process->setWorkingDirectory(path);
+
+    QObject::connect
+    (
+        process,
+        &QProcess::readyReadStandardOutput,
+        this,
+        [this, process]()
+        {
+            QByteArray processData = process->readAllStandardOutput();
+            setCompilerOutput(m_compilerOutput + processData);
+        }
+    );
+
+    QObject::connect
+    (
+        process,
+        &QProcess::readyReadStandardError,
+        this,
+        [this, process]()
+        {
+            QByteArray processData = process->readAllStandardError();
+
+            if(!processData.isValidUtf8())
+            {
+                qWarning() << QStringLiteral("Process output not valid UTF8 data");
+                return;
+            }
+
+            setCompilerOutput(m_compilerOutput + processData);
+        }
+    );
+
+    process->start(qsb, arguments);
+
+    if(!process->waitForStarted(3000))
+    {
+        process->deleteLater();
+        throw FileException(QStringLiteral("Could not start shader compiler"));
+    }
+
+    if(!process->waitForFinished())
+    {
+        process->deleteLater();
+        throw FileException(QStringLiteral("Shader compiler timeout"));
+    }
+
+    if(process->exitCode() != 0)
+    {
+        process->deleteLater();
+        throw ShaderCompilerException(m_compilerOutput);
+    }
+
+    process->deleteLater();
+
+    if(!QFile::remove(uri.toLocalFile()))
+    {
+        throw FileException(QStringLiteral("Could not clean up"));
+    }
+}
+
+auto PackCompiler::removeDirectory(const QUrl &uri) -> bool
+{
+    QStringList arguments =
+    {
+        QStringLiteral("-rf"),
+        uri.toLocalFile()
+    };
+
+    QProcess *process = new QProcess(this);
+
+    process->start(QStringLiteral("rm"), arguments);
+
+    if(!process->waitForStarted(3000))
+    {
+        process->deleteLater();
+        return false;
+    }
+
+    if(!process->waitForFinished())
+    {
+        process->deleteLater();
+        return false;
+    }
+
+    if(process->exitCode() != 0)
+    {
+        process->deleteLater();
+        return false;
+    }
+
+    process->deleteLater();
+    return true;
+}
+
+auto PackCompiler::setProgress(qreal progress) -> void
+{
+    if(qFuzzyCompare(progress, m_progress))
+    {
+        return;
+    }
+
+    m_progress = progress;
+    Q_EMIT progressChanged();
+}
+
+void PackCompiler::setCompilerOutput(const QString &compilerOutput)
+{
+    if (m_compilerOutput == compilerOutput)
+    {
+        return;
+    }
+
+    m_compilerOutput = compilerOutput;
+    emit compilerOutputChanged();
+}
+
+auto PackCompiler::setError(const QString &title, const QString &message, const QUrl &uri) -> void
+{
+    if(title == m_errorTitle && message == m_errorMessage)
+    {
+        return;
+    }
+
+    m_errorMessage = std::move(message);
+    m_errorTitle = std::move(title);
+
+    if(uri.isValid() && uri.isLocalFile())
+    {
+        QFile errorFile(uri.toLocalFile());
+
+        if(errorFile.open(QFile::ReadWrite | QFile::Append))
+        {
+            errorFile.write(message.toLocal8Bit() + QByteArray("\n"));
+        }
+    }
+
+    Q_EMIT errorTitleChanged();
+    Q_EMIT errorMessageChanged();
+    Q_EMIT errorOcurred();
+}
+
+auto PackCompiler::setState(State state) -> void
+{
+    if(state == m_state)
+    {
+        return;
+    }
+
+    m_state = state;
+    Q_EMIT stateChanged();
+}
+
+auto PackCompiler::setStatus(const QString &status) -> void
+{
+    if (m_status == status)
+    {
+        return;
+    }
+
+    m_status = status;
+    Q_EMIT statusChanged();
+}
+
+auto PackCompiler::setTotalSteps(qint64 steps) -> void
+{
+    if(steps == m_totalSteps)
+    {
+        return;
+    }
+
+    m_totalSteps = steps;
+    Q_EMIT totalStepsChanged();
+}
+
+auto PackCompiler::setCurrentStep(qint64 step) -> void
+{
+    if(step == m_currentStep)
+    {
+        return;
+    }
+
+    m_currentStep = step;
+    Q_EMIT currentStepChanged();
+}
+
+auto PackCompiler::incrementCompileStep() -> void
+{
+    setCurrentStep(m_currentStep + 1);
+
+    setProgress
+    (
+        static_cast<qreal>(m_currentStep) / m_totalSteps
+    );
+}

KomplexHubPlugin/packcompiler.h

@@ -0,0 +1,248 @@
+#ifndef PACKCOMPILER_H
+#define PACKCOMPILER_H
+
+#include <QDir>
+#include <QFile>
+#include <QFuture>
+#include <QtConcurrent/QtConcurrentRun>
+#include <QStandardPaths>
+#include <QProcess>
+#include <QRegularExpression>
+#include <QMutex>
+#include <QMutexLocker>
+#include <QImage>
+#include <QRgb>
+#include <QColor>
+#include <QtQuick/QQuickItem>
+#include <qdir.h>
+#include <qobject.h>
+#include <qtmetamacros.h>
+#include <QPainter>
+#include <QtConcurrent/qtconcurrentrun.h>
+#include <qcontainerfwd.h>
+#include <qmargins.h>
+#include <qstandardpaths.h>
+#include <qtmetamacros.h>
+#include <qurl.h>
+#include "common/komplex_global.h"
+/**
+ * @brief The PackCompiler class
+ * [1] Extract Pack (GZip)
+ * [2] Verify directory is a pack and has shaders
+ * [3] Load shader code
+ * [4] Append Common and Global files, if they exist
+ * [5] Add uniform and replace variable use
+ * [6] Run C Preprocessor (cpp -P) to process macros
+ * [7] Append version macro
+ * [8] Compile with Qt Shader Baker
+ */
+class KOMPLEX_EXPORT PackCompiler : public QObject
+{
+    Q_OBJECT
+    QML_ELEMENT
+public:
+
+    enum State
+    {
+        Idle,
+        Compiling,
+        Complete,
+        Error
+    };
+    Q_ENUM(State)
+
+    explicit PackCompiler(QObject *parent = nullptr);
+    ~PackCompiler() override;
+
+    auto compilerOutput() const -> const QString & { return m_compilerOutput; }
+    auto progress() -> qreal { return m_progress; }
+    auto currentStep() -> qint64 { return m_currentStep; }
+    auto totalSteps() -> qint64 { return m_totalSteps; }
+    auto errorTitle() const -> const QString & { return m_errorTitle; }
+    auto errorMessage() const -> const QString & { return m_errorMessage; }
+    auto reset() -> void;
+
+    auto process(const QUrl &uri) -> QFuture<QUrl>;
+
+    auto status() const -> const QString & { return m_status; }
+    auto state() const -> State { return m_state; }
+
+protected:
+    /**
+     * @brief prepareShaders
+     * Scans the pack's shader directory and sends each file to
+     * prepareFile()
+     * @param uri
+     * Directory to prepare
+     */
+    auto prepareShaders(const QUrl &uri) noexcept(false) -> void;
+
+    /**
+     * @brief prepareFile
+     * Prepares the individual file by appending the header, footer,
+     * common files and replacing standard variable names
+     * @param uri
+     */
+    auto prepareFile(const QUrl &uri) noexcept(false) -> void;
+
+    /**
+     * @brief validateDirectory
+     *
+     * @param uri
+     * @return true
+     * @return false
+     */
+    auto validateDirectory(const QUrl &uri) -> bool;
+
+    auto extract(const QUrl &sourceUri, const QUrl &destinationUri) noexcept(false) -> void;
+    auto copyFile(const QUrl &sourceUri, const QUrl &destinationUri) noexcept(false) -> void;
+    auto compile(const QUrl &uri) noexcept(false) -> void;
+    auto preprocess(const QUrl &uri) noexcept(false) -> void;
+    auto appendVersion(const QUrl &uri) noexcept(false) -> void;
+    auto compileShader(const QUrl &uri) noexcept(false) -> void;
+    auto loadCommonFragmentData(const QUrl &uri) -> QByteArray;
+    auto loadCommonVertexData(const QUrl &uri) -> QByteArray;
+    auto loadGlobalData(const QUrl &uri) -> QByteArray;
+
+    auto setError(const QString &title, const QString &message, const QUrl &uri = {}) -> void;
+    auto setCompilerOutput(const QString &compilerOutput) -> void;
+    auto setState(State state) -> void;
+    auto setProgress(qreal progress) -> void;
+    auto setTotalSteps(qint64 steps) -> void;
+    auto setCurrentStep(qint64 step) -> void;
+
+    auto setStatus(const QString &status) -> void;
+
+    auto incrementCompileStep() -> void;
+
+    auto removeDirectory(const QUrl &uri) -> bool;
+
+signals:
+    auto compilerOutputChanged() -> void;
+    auto errorOcurred() -> void;
+    auto errorTitleChanged() -> void;
+    auto errorMessageChanged() -> void;
+    auto stateChanged() -> void;
+    auto progressChanged() -> void;
+    auto currentStepChanged() -> void;
+    auto totalStepsChanged() -> void;
+
+    auto compileComplete(const QUrl &uri) -> void;
+
+    auto statusChanged() -> void;
+    auto packReady(const QString &) -> void;
+    auto errorStepChanged() -> void;
+    auto errorRatingChanged() -> void;
+
+private:
+    static inline const QStringList m_updateVariables
+        {
+            QStringLiteral("iTime"),
+            QStringLiteral("iTimeDelta"),
+            QStringLiteral("iFrameRate"),
+            QStringLiteral("iSampleRate"),
+            QStringLiteral("iFrame"),
+            QStringLiteral("iDate"),
+            QStringLiteral("iMouse"),
+            QStringLiteral("iResolution"),
+            QStringLiteral("iColorTheme"),
+            QStringLiteral("iChannelTime"),
+            QStringLiteral("iChannelResolution")
+        };
+
+    static inline const QByteArray m_version
+        {
+            R"(#version 450)"
+        };
+
+    static inline const QString m_header
+        {
+            R"(layout(location = 0) in vec2 qt_TexCoord0;
+layout(location = 0) out vec4 fragColor;
+
+layout(std140, binding = 0) uniform buf {
+    mat4 qt_Matrix;
+    float qt_Opacity;
+    float iTime;
+    float iTimeDelta;
+    float iFrameRate;
+    float iSampleRate;
+    int iFrame;
+    vec4 iDate;
+    vec4 iMouse;
+    vec3 iResolution;
+    float iChannelTime[4];
+    vec3 iChannelResolution[4];
+    vec4 iColorTheme[4];
+} ubuf;
+
+layout(binding = 1) uniform sampler2D iChannel0;
+layout(binding = 2) uniform sampler2D iChannel1;
+layout(binding = 3) uniform sampler2D iChannel2;
+layout(binding = 4) uniform sampler2D iChannel3;
+
+vec2 fragCoord = vec2(qt_TexCoord0.x, 1.0 - qt_TexCoord0.y) * ubuf.iResolution.xy;)"
+        };
+
+    static inline const QString m_footer
+        {
+            R"(void main() {
+    vec4 color = vec4(0.0);
+    mainImage(color, fragCoord);
+    fragColor = color;
+})"
+        };
+
+    QString m_compilerOutput;
+    QString m_errorMessage;
+    QString m_errorTitle;
+    QString m_status;
+
+    State m_state;
+
+    qreal m_currentStep = 0;
+    qreal m_progress = 0;
+    qreal m_totalSteps = 0;
+
+    QMutex m_downloadMutex;
+
+    static inline QList<QRegularExpression> m_variableExpressions;
+
+    static inline const QRegularExpression m_commonFragmentExpression = QRegularExpression
+    (
+        QString("^common\\.frag$"),
+        QRegularExpression::CaseInsensitiveOption
+    );
+
+    static inline const QRegularExpression m_commonVertexExpression = QRegularExpression
+    (
+        QString("^common\\.vert$"),
+        QRegularExpression::CaseInsensitiveOption
+    );
+
+    static inline const QRegularExpression m_fragmentExpression = QRegularExpression
+    (
+        QString("^.{1,}\\.frag$"),
+        QRegularExpression::CaseInsensitiveOption |
+            QRegularExpression::DotMatchesEverythingOption
+    );
+
+    static inline const QRegularExpression m_vertexExpression = QRegularExpression
+    (
+        QString("^.{1,}\\.vert$"),
+        QRegularExpression::CaseInsensitiveOption |
+            QRegularExpression::DotMatchesEverythingOption
+    );
+
+    Q_PROPERTY(QString compilerOutput READ compilerOutput WRITE setCompilerOutput NOTIFY compilerOutputChanged FINAL)
+    Q_PROPERTY(QString errorTitle READ errorTitle NOTIFY errorTitleChanged FINAL)
+    Q_PROPERTY(QString errorMessage READ errorMessage NOTIFY errorMessageChanged FINAL)
+    Q_PROPERTY(qreal progress READ progress NOTIFY progressChanged FINAL)
+    Q_PROPERTY(qint64 totalSteps READ totalSteps NOTIFY totalStepsChanged FINAL)
+    Q_PROPERTY(qint64 currentStep READ currentStep NOTIFY currentStepChanged FINAL)
+    Q_PROPERTY(QString status READ status WRITE setStatus NOTIFY statusChanged FINAL)
+    Q_PROPERTY(State state READ state NOTIFY stateChanged FINAL)
+};
+
+Q_DECLARE_METATYPE(PackCompiler)
+#endif // PackCompiler_H